1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/ADT/StringExtras.h"
18#include "llvm/Analysis/OptimizationRemarkEmitter.h"
19#include "llvm/Analysis/StackSafetyAnalysis.h"
20#include "llvm/Analysis/TargetLibraryInfo.h"
21#include "llvm/Analysis/TargetTransformInfo.h"
22#include "llvm/Bitcode/BitcodeReader.h"
23#include "llvm/Bitcode/BitcodeWriter.h"
24#include "llvm/CodeGen/Analysis.h"
25#include "llvm/Config/llvm-config.h"
26#include "llvm/IR/AutoUpgrade.h"
27#include "llvm/IR/DiagnosticPrinter.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/LLVMRemarkStreamer.h"
30#include "llvm/IR/LegacyPassManager.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/LTO/LTOBackend.h"
34#include "llvm/LTO/SummaryBasedOptimizations.h"
35#include "llvm/Linker/IRMover.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Object/IRObjectFile.h"
38#include "llvm/Support/CommandLine.h"
39#include "llvm/Support/Error.h"
40#include "llvm/Support/FileSystem.h"
41#include "llvm/Support/ManagedStatic.h"
42#include "llvm/Support/MemoryBuffer.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SHA1.h"
45#include "llvm/Support/SourceMgr.h"
46#include "llvm/Support/ThreadPool.h"
47#include "llvm/Support/Threading.h"
48#include "llvm/Support/TimeProfiler.h"
49#include "llvm/Support/ToolOutputFile.h"
50#include "llvm/Support/VCSRevision.h"
51#include "llvm/Support/raw_ostream.h"
52#include "llvm/Target/TargetOptions.h"
53#include "llvm/Transforms/IPO.h"
54#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
55#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
56#include "llvm/Transforms/Utils/FunctionImportUtils.h"
57#include "llvm/Transforms/Utils/SplitModule.h"
58
59#include <optional>
60#include <set>
61
62using namespace llvm;
63using namespace lto;
64using namespace object;
65
66#define DEBUG_TYPE "lto"
67
68static cl::opt<bool>
69 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(Val: false), cl::Hidden,
70 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
71
72namespace llvm {
73/// Enable global value internalization in LTO.
74cl::opt<bool> EnableLTOInternalization(
75 "enable-lto-internalization", cl::init(Val: true), cl::Hidden,
76 cl::desc("Enable global value internalization in LTO"));
77
78/// Indicate we are linking with an allocator that supports hot/cold operator
79/// new interfaces.
80extern cl::opt<bool> SupportsHotColdNew;
81
82/// Enable MemProf context disambiguation for thin link.
83extern cl::opt<bool> EnableMemProfContextDisambiguation;
84} // namespace llvm
85
86// Computes a unique hash for the Module considering the current list of
87// export/import and other global analysis results.
88// The hash is produced in \p Key.
89void llvm::computeLTOCacheKey(
90 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
91 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
92 const FunctionImporter::ExportSetTy &ExportList,
93 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
94 const GVSummaryMapTy &DefinedGlobals,
95 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
96 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
97 // Compute the unique hash for this entry.
98 // This is based on the current compiler version, the module itself, the
99 // export list, the hash for every single module in the import list, the
100 // list of ResolvedODR for the module, and the list of preserved symbols.
101 SHA1 Hasher;
102
103 // Start with the compiler revision
104 Hasher.update(LLVM_VERSION_STRING);
105#ifdef LLVM_REVISION
106 Hasher.update(LLVM_REVISION);
107#endif
108
109 // Include the parts of the LTO configuration that affect code generation.
110 auto AddString = [&](StringRef Str) {
111 Hasher.update(Str);
112 Hasher.update(Data: ArrayRef<uint8_t>{0});
113 };
114 auto AddUnsigned = [&](unsigned I) {
115 uint8_t Data[4];
116 support::endian::write32le(P: Data, V: I);
117 Hasher.update(Data: ArrayRef<uint8_t>{Data, 4});
118 };
119 auto AddUint64 = [&](uint64_t I) {
120 uint8_t Data[8];
121 support::endian::write64le(P: Data, V: I);
122 Hasher.update(Data: ArrayRef<uint8_t>{Data, 8});
123 };
124 AddString(Conf.CPU);
125 // FIXME: Hash more of Options. For now all clients initialize Options from
126 // command-line flags (which is unsupported in production), but may set
127 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
128 // DataSections and DebuggerTuning via command line flags.
129 AddUnsigned(Conf.Options.RelaxELFRelocations);
130 AddUnsigned(Conf.Options.FunctionSections);
131 AddUnsigned(Conf.Options.DataSections);
132 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
133 for (auto &A : Conf.MAttrs)
134 AddString(A);
135 if (Conf.RelocModel)
136 AddUnsigned(*Conf.RelocModel);
137 else
138 AddUnsigned(-1);
139 if (Conf.CodeModel)
140 AddUnsigned(*Conf.CodeModel);
141 else
142 AddUnsigned(-1);
143 for (const auto &S : Conf.MllvmArgs)
144 AddString(S);
145 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
146 AddUnsigned(static_cast<int>(Conf.CGFileType));
147 AddUnsigned(Conf.OptLevel);
148 AddUnsigned(Conf.Freestanding);
149 AddString(Conf.OptPipeline);
150 AddString(Conf.AAPipeline);
151 AddString(Conf.OverrideTriple);
152 AddString(Conf.DefaultTriple);
153 AddString(Conf.DwoDir);
154
155 // Include the hash for the current module
156 auto ModHash = Index.getModuleHash(ModPath: ModuleID);
157 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
158
159 std::vector<uint64_t> ExportsGUID;
160 ExportsGUID.reserve(n: ExportList.size());
161 for (const auto &VI : ExportList) {
162 auto GUID = VI.getGUID();
163 ExportsGUID.push_back(x: GUID);
164 }
165
166 // Sort the export list elements GUIDs.
167 llvm::sort(C&: ExportsGUID);
168 for (uint64_t GUID : ExportsGUID) {
169 // The export list can impact the internalization, be conservative here
170 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
171 }
172
173 // Include the hash for every module we import functions from. The set of
174 // imported symbols for each module may affect code generation and is
175 // sensitive to link order, so include that as well.
176 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
177 struct ImportModule {
178 ImportMapIteratorTy ModIt;
179 const ModuleSummaryIndex::ModuleInfo *ModInfo;
180
181 StringRef getIdentifier() const { return ModIt->getFirst(); }
182 const FunctionImporter::FunctionsToImportTy &getFunctions() const {
183 return ModIt->second;
184 }
185
186 const ModuleHash &getHash() const { return ModInfo->second; }
187 };
188
189 std::vector<ImportModule> ImportModulesVector;
190 ImportModulesVector.reserve(n: ImportList.size());
191
192 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
193 ++It) {
194 ImportModulesVector.push_back(x: {.ModIt: It, .ModInfo: Index.getModule(ModPath: It->getFirst())});
195 }
196 // Order using module hash, to be both independent of module name and
197 // module order.
198 llvm::sort(C&: ImportModulesVector,
199 Comp: [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
200 return Lhs.getHash() < Rhs.getHash();
201 });
202 for (const ImportModule &Entry : ImportModulesVector) {
203 auto ModHash = Entry.getHash();
204 Hasher.update(Data: ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
205
206 AddUint64(Entry.getFunctions().size());
207 for (auto &Fn : Entry.getFunctions())
208 AddUint64(Fn);
209 }
210
211 // Include the hash for the resolved ODR.
212 for (auto &Entry : ResolvedODR) {
213 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
214 sizeof(GlobalValue::GUID)));
215 Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
216 sizeof(GlobalValue::LinkageTypes)));
217 }
218
219 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
220 // defined in this module.
221 std::set<GlobalValue::GUID> UsedCfiDefs;
222 std::set<GlobalValue::GUID> UsedCfiDecls;
223
224 // Typeids used in this module.
225 std::set<GlobalValue::GUID> UsedTypeIds;
226
227 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
228 if (CfiFunctionDefs.count(x: ValueGUID))
229 UsedCfiDefs.insert(x: ValueGUID);
230 if (CfiFunctionDecls.count(x: ValueGUID))
231 UsedCfiDecls.insert(x: ValueGUID);
232 };
233
234 auto AddUsedThings = [&](GlobalValueSummary *GS) {
235 if (!GS) return;
236 AddUnsigned(GS->getVisibility());
237 AddUnsigned(GS->isLive());
238 AddUnsigned(GS->canAutoHide());
239 for (const ValueInfo &VI : GS->refs()) {
240 AddUnsigned(VI.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
241 AddUsedCfiGlobal(VI.getGUID());
242 }
243 if (auto *GVS = dyn_cast<GlobalVarSummary>(Val: GS)) {
244 AddUnsigned(GVS->maybeReadOnly());
245 AddUnsigned(GVS->maybeWriteOnly());
246 }
247 if (auto *FS = dyn_cast<FunctionSummary>(Val: GS)) {
248 for (auto &TT : FS->type_tests())
249 UsedTypeIds.insert(x: TT);
250 for (auto &TT : FS->type_test_assume_vcalls())
251 UsedTypeIds.insert(x: TT.GUID);
252 for (auto &TT : FS->type_checked_load_vcalls())
253 UsedTypeIds.insert(x: TT.GUID);
254 for (auto &TT : FS->type_test_assume_const_vcalls())
255 UsedTypeIds.insert(x: TT.VFunc.GUID);
256 for (auto &TT : FS->type_checked_load_const_vcalls())
257 UsedTypeIds.insert(x: TT.VFunc.GUID);
258 for (auto &ET : FS->calls()) {
259 AddUnsigned(ET.first.isDSOLocal(WithDSOLocalPropagation: Index.withDSOLocalPropagation()));
260 AddUsedCfiGlobal(ET.first.getGUID());
261 }
262 }
263 };
264
265 // Include the hash for the linkage type to reflect internalization and weak
266 // resolution, and collect any used type identifier resolutions.
267 for (auto &GS : DefinedGlobals) {
268 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
269 Hasher.update(
270 Data: ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
271 AddUsedCfiGlobal(GS.first);
272 AddUsedThings(GS.second);
273 }
274
275 // Imported functions may introduce new uses of type identifier resolutions,
276 // so we need to collect their used resolutions as well.
277 for (const ImportModule &ImpM : ImportModulesVector)
278 for (auto &ImpF : ImpM.getFunctions()) {
279 GlobalValueSummary *S =
280 Index.findSummaryInModule(ValueGUID: ImpF, ModuleId: ImpM.getIdentifier());
281 AddUsedThings(S);
282 // If this is an alias, we also care about any types/etc. that the aliasee
283 // may reference.
284 if (auto *AS = dyn_cast_or_null<AliasSummary>(Val: S))
285 AddUsedThings(AS->getBaseObject());
286 }
287
288 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
289 AddString(TId);
290
291 AddUnsigned(S.TTRes.TheKind);
292 AddUnsigned(S.TTRes.SizeM1BitWidth);
293
294 AddUint64(S.TTRes.AlignLog2);
295 AddUint64(S.TTRes.SizeM1);
296 AddUint64(S.TTRes.BitMask);
297 AddUint64(S.TTRes.InlineBits);
298
299 AddUint64(S.WPDRes.size());
300 for (auto &WPD : S.WPDRes) {
301 AddUnsigned(WPD.first);
302 AddUnsigned(WPD.second.TheKind);
303 AddString(WPD.second.SingleImplName);
304
305 AddUint64(WPD.second.ResByArg.size());
306 for (auto &ByArg : WPD.second.ResByArg) {
307 AddUint64(ByArg.first.size());
308 for (uint64_t Arg : ByArg.first)
309 AddUint64(Arg);
310 AddUnsigned(ByArg.second.TheKind);
311 AddUint64(ByArg.second.Info);
312 AddUnsigned(ByArg.second.Byte);
313 AddUnsigned(ByArg.second.Bit);
314 }
315 }
316 };
317
318 // Include the hash for all type identifiers used by this module.
319 for (GlobalValue::GUID TId : UsedTypeIds) {
320 auto TidIter = Index.typeIds().equal_range(x: TId);
321 for (auto It = TidIter.first; It != TidIter.second; ++It)
322 AddTypeIdSummary(It->second.first, It->second.second);
323 }
324
325 AddUnsigned(UsedCfiDefs.size());
326 for (auto &V : UsedCfiDefs)
327 AddUint64(V);
328
329 AddUnsigned(UsedCfiDecls.size());
330 for (auto &V : UsedCfiDecls)
331 AddUint64(V);
332
333 if (!Conf.SampleProfile.empty()) {
334 auto FileOrErr = MemoryBuffer::getFile(Filename: Conf.SampleProfile);
335 if (FileOrErr) {
336 Hasher.update(Str: FileOrErr.get()->getBuffer());
337
338 if (!Conf.ProfileRemapping.empty()) {
339 FileOrErr = MemoryBuffer::getFile(Filename: Conf.ProfileRemapping);
340 if (FileOrErr)
341 Hasher.update(Str: FileOrErr.get()->getBuffer());
342 }
343 }
344 }
345
346 Key = toHex(Input: Hasher.result());
347}
348
349static void thinLTOResolvePrevailingGUID(
350 const Config &C, ValueInfo VI,
351 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
352 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
353 isPrevailing,
354 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
355 recordNewLinkage,
356 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
357 GlobalValue::VisibilityTypes Visibility =
358 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
359 : GlobalValue::DefaultVisibility;
360 for (auto &S : VI.getSummaryList()) {
361 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
362 // Ignore local and appending linkage values since the linker
363 // doesn't resolve them.
364 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
365 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
366 continue;
367 // We need to emit only one of these. The prevailing module will keep it,
368 // but turned into a weak, while the others will drop it when possible.
369 // This is both a compile-time optimization and a correctness
370 // transformation. This is necessary for correctness when we have exported
371 // a reference - we need to convert the linkonce to weak to
372 // ensure a copy is kept to satisfy the exported reference.
373 // FIXME: We may want to split the compile time and correctness
374 // aspects into separate routines.
375 if (isPrevailing(VI.getGUID(), S.get())) {
376 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage)) {
377 S->setLinkage(GlobalValue::getWeakLinkage(
378 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
379 // The kept copy is eligible for auto-hiding (hidden visibility) if all
380 // copies were (i.e. they were all linkonce_odr global unnamed addr).
381 // If any copy is not (e.g. it was originally weak_odr), then the symbol
382 // must remain externally available (e.g. a weak_odr from an explicitly
383 // instantiated template). Additionally, if it is in the
384 // GUIDPreservedSymbols set, that means that it is visibile outside
385 // the summary (e.g. in a native object or a bitcode file without
386 // summary), and in that case we cannot hide it as it isn't possible to
387 // check all copies.
388 S->setCanAutoHide(VI.canAutoHide() &&
389 !GUIDPreservedSymbols.count(V: VI.getGUID()));
390 }
391 if (C.VisibilityScheme == Config::FromPrevailing)
392 Visibility = S->getVisibility();
393 }
394 // Alias and aliasee can't be turned into available_externally.
395 else if (!isa<AliasSummary>(Val: S.get()) &&
396 !GlobalInvolvedWithAlias.count(V: S.get()))
397 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
398
399 // For ELF, set visibility to the computed visibility from summaries. We
400 // don't track visibility from declarations so this may be more relaxed than
401 // the most constraining one.
402 if (C.VisibilityScheme == Config::ELF)
403 S->setVisibility(Visibility);
404
405 if (S->linkage() != OriginalLinkage)
406 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
407 }
408
409 if (C.VisibilityScheme == Config::FromPrevailing) {
410 for (auto &S : VI.getSummaryList()) {
411 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
412 if (GlobalValue::isLocalLinkage(Linkage: OriginalLinkage) ||
413 GlobalValue::isAppendingLinkage(Linkage: S->linkage()))
414 continue;
415 S->setVisibility(Visibility);
416 }
417 }
418}
419
420/// Resolve linkage for prevailing symbols in the \p Index.
421//
422// We'd like to drop these functions if they are no longer referenced in the
423// current module. However there is a chance that another module is still
424// referencing them because of the import. We make sure we always emit at least
425// one copy.
426void llvm::thinLTOResolvePrevailingInIndex(
427 const Config &C, ModuleSummaryIndex &Index,
428 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
429 isPrevailing,
430 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
431 recordNewLinkage,
432 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
433 // We won't optimize the globals that are referenced by an alias for now
434 // Ideally we should turn the alias into a global and duplicate the definition
435 // when needed.
436 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
437 for (auto &I : Index)
438 for (auto &S : I.second.SummaryList)
439 if (auto AS = dyn_cast<AliasSummary>(Val: S.get()))
440 GlobalInvolvedWithAlias.insert(V: &AS->getAliasee());
441
442 for (auto &I : Index)
443 thinLTOResolvePrevailingGUID(C, VI: Index.getValueInfo(R: I),
444 GlobalInvolvedWithAlias, isPrevailing,
445 recordNewLinkage, GUIDPreservedSymbols);
446}
447
448static void thinLTOInternalizeAndPromoteGUID(
449 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
450 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
451 isPrevailing) {
452 auto ExternallyVisibleCopies =
453 llvm::count_if(Range: VI.getSummaryList(),
454 P: [](const std::unique_ptr<GlobalValueSummary> &Summary) {
455 return !GlobalValue::isLocalLinkage(Linkage: Summary->linkage());
456 });
457
458 for (auto &S : VI.getSummaryList()) {
459 // First see if we need to promote an internal value because it is not
460 // exported.
461 if (isExported(S->modulePath(), VI)) {
462 if (GlobalValue::isLocalLinkage(Linkage: S->linkage()))
463 S->setLinkage(GlobalValue::ExternalLinkage);
464 continue;
465 }
466
467 // Otherwise, see if we can internalize.
468 if (!EnableLTOInternalization)
469 continue;
470
471 // Non-exported values with external linkage can be internalized.
472 if (GlobalValue::isExternalLinkage(Linkage: S->linkage())) {
473 S->setLinkage(GlobalValue::InternalLinkage);
474 continue;
475 }
476
477 // Non-exported function and variable definitions with a weak-for-linker
478 // linkage can be internalized in certain cases. The minimum legality
479 // requirements would be that they are not address taken to ensure that we
480 // don't break pointer equality checks, and that variables are either read-
481 // or write-only. For functions, this is the case if either all copies are
482 // [local_]unnamed_addr, or we can propagate reference edge attributes
483 // (which is how this is guaranteed for variables, when analyzing whether
484 // they are read or write-only).
485 //
486 // However, we only get to this code for weak-for-linkage values in one of
487 // two cases:
488 // 1) The prevailing copy is not in IR (it is in native code).
489 // 2) The prevailing copy in IR is not exported from its module.
490 // Additionally, at least for the new LTO API, case 2 will only happen if
491 // there is exactly one definition of the value (i.e. in exactly one
492 // module), as duplicate defs are result in the value being marked exported.
493 // Likely, users of the legacy LTO API are similar, however, currently there
494 // are llvm-lto based tests of the legacy LTO API that do not mark
495 // duplicate linkonce_odr copies as exported via the tool, so we need
496 // to handle that case below by checking the number of copies.
497 //
498 // Generally, we only want to internalize a weak-for-linker value in case
499 // 2, because in case 1 we cannot see how the value is used to know if it
500 // is read or write-only. We also don't want to bloat the binary with
501 // multiple internalized copies of non-prevailing linkonce/weak functions.
502 // Note if we don't internalize, we will convert non-prevailing copies to
503 // available_externally anyway, so that we drop them after inlining. The
504 // only reason to internalize such a function is if we indeed have a single
505 // copy, because internalizing it won't increase binary size, and enables
506 // use of inliner heuristics that are more aggressive in the face of a
507 // single call to a static (local). For variables, internalizing a read or
508 // write only variable can enable more aggressive optimization. However, we
509 // already perform this elsewhere in the ThinLTO backend handling for
510 // read or write-only variables (processGlobalForThinLTO).
511 //
512 // Therefore, only internalize linkonce/weak if there is a single copy, that
513 // is prevailing in this IR module. We can do so aggressively, without
514 // requiring the address to be insignificant, or that a variable be read or
515 // write-only.
516 if (!GlobalValue::isWeakForLinker(Linkage: S->linkage()) ||
517 GlobalValue::isExternalWeakLinkage(Linkage: S->linkage()))
518 continue;
519
520 if (isPrevailing(VI.getGUID(), S.get()) && ExternallyVisibleCopies == 1)
521 S->setLinkage(GlobalValue::InternalLinkage);
522 }
523}
524
525// Update the linkages in the given \p Index to mark exported values
526// as external and non-exported values as internal.
527void llvm::thinLTOInternalizeAndPromoteInIndex(
528 ModuleSummaryIndex &Index,
529 function_ref<bool(StringRef, ValueInfo)> isExported,
530 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
531 isPrevailing) {
532 for (auto &I : Index)
533 thinLTOInternalizeAndPromoteGUID(VI: Index.getValueInfo(R: I), isExported,
534 isPrevailing);
535}
536
537// Requires a destructor for std::vector<InputModule>.
538InputFile::~InputFile() = default;
539
540Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
541 std::unique_ptr<InputFile> File(new InputFile);
542
543 Expected<IRSymtabFile> FOrErr = readIRSymtab(MBRef: Object);
544 if (!FOrErr)
545 return FOrErr.takeError();
546
547 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
548 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
549 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
550 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
551 File->ComdatTable = FOrErr->TheReader.getComdatTable();
552
553 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
554 size_t Begin = File->Symbols.size();
555 for (const irsymtab::Reader::SymbolRef &Sym :
556 FOrErr->TheReader.module_symbols(I))
557 // Skip symbols that are irrelevant to LTO. Note that this condition needs
558 // to match the one in Skip() in LTO::addRegularLTO().
559 if (Sym.isGlobal() && !Sym.isFormatSpecific())
560 File->Symbols.push_back(x: Sym);
561 File->ModuleSymIndices.push_back(x: {Begin, File->Symbols.size()});
562 }
563
564 File->Mods = FOrErr->Mods;
565 File->Strtab = std::move(FOrErr->Strtab);
566 return std::move(File);
567}
568
569StringRef InputFile::getName() const {
570 return Mods[0].getModuleIdentifier();
571}
572
573BitcodeModule &InputFile::getSingleBitcodeModule() {
574 assert(Mods.size() == 1 && "Expect only one bitcode module");
575 return Mods[0];
576}
577
578LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
579 const Config &Conf)
580 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
581 Ctx(Conf), CombinedModule(std::make_unique<Module>(args: "ld-temp.o", args&: Ctx)),
582 Mover(std::make_unique<IRMover>(args&: *CombinedModule)) {}
583
584LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
585 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
586 if (!Backend)
587 this->Backend =
588 createInProcessThinBackend(Parallelism: llvm::heavyweight_hardware_concurrency());
589}
590
591LTO::LTO(Config Conf, ThinBackend Backend,
592 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
593 : Conf(std::move(Conf)),
594 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
595 ThinLTO(std::move(Backend)),
596 GlobalResolutions(std::make_optional<StringMap<GlobalResolution>>()),
597 LTOMode(LTOMode) {}
598
599// Requires a destructor for MapVector<BitcodeModule>.
600LTO::~LTO() = default;
601
602// Add the symbols in the given module to the GlobalResolutions map, and resolve
603// their partitions.
604void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
605 ArrayRef<SymbolResolution> Res,
606 unsigned Partition, bool InSummary) {
607 auto *ResI = Res.begin();
608 auto *ResE = Res.end();
609 (void)ResE;
610 const Triple TT(RegularLTO.CombinedModule->getTargetTriple());
611 for (const InputFile::Symbol &Sym : Syms) {
612 assert(ResI != ResE);
613 SymbolResolution Res = *ResI++;
614
615 auto &GlobalRes = (*GlobalResolutions)[Sym.getName()];
616 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
617 if (Res.Prevailing) {
618 assert(!GlobalRes.Prevailing &&
619 "Multiple prevailing defs are not allowed");
620 GlobalRes.Prevailing = true;
621 GlobalRes.IRName = std::string(Sym.getIRName());
622 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
623 // Sometimes it can be two copies of symbol in a module and prevailing
624 // symbol can have no IR name. That might happen if symbol is defined in
625 // module level inline asm block. In case we have multiple modules with
626 // the same symbol we want to use IR name of the prevailing symbol.
627 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
628 // we can later use it to check if there is any prevailing copy in IR.
629 GlobalRes.IRName = std::string(Sym.getIRName());
630 }
631
632 // In rare occasion, the symbol used to initialize GlobalRes has a different
633 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
634 // symbol is referenced through its mangled name, say @"\01_symbol" while
635 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
636 // In that case, we have the same actual Symbol that can get two different
637 // GUID, leading to some invalid internalization. Workaround this by marking
638 // the GlobalRes external.
639
640 // FIXME: instead of this check, it would be desirable to compute GUIDs
641 // based on mangled name, but this requires an access to the Target Triple
642 // and would be relatively invasive on the codebase.
643 if (GlobalRes.IRName != Sym.getIRName()) {
644 GlobalRes.Partition = GlobalResolution::External;
645 GlobalRes.VisibleOutsideSummary = true;
646 }
647
648 // Set the partition to external if we know it is re-defined by the linker
649 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
650 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
651 // already recorded as being referenced from a different partition.
652 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
653 (GlobalRes.Partition != GlobalResolution::Unknown &&
654 GlobalRes.Partition != Partition)) {
655 GlobalRes.Partition = GlobalResolution::External;
656 } else
657 // First recorded reference, save the current partition.
658 GlobalRes.Partition = Partition;
659
660 // Flag as visible outside of summary if visible from a regular object or
661 // from a module that does not have a summary.
662 GlobalRes.VisibleOutsideSummary |=
663 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
664
665 GlobalRes.ExportDynamic |= Res.ExportDynamic;
666 }
667}
668
669static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
670 ArrayRef<SymbolResolution> Res) {
671 StringRef Path = Input->getName();
672 OS << Path << '\n';
673 auto ResI = Res.begin();
674 for (const InputFile::Symbol &Sym : Input->symbols()) {
675 assert(ResI != Res.end());
676 SymbolResolution Res = *ResI++;
677
678 OS << "-r=" << Path << ',' << Sym.getName() << ',';
679 if (Res.Prevailing)
680 OS << 'p';
681 if (Res.FinalDefinitionInLinkageUnit)
682 OS << 'l';
683 if (Res.VisibleToRegularObj)
684 OS << 'x';
685 if (Res.LinkerRedefined)
686 OS << 'r';
687 OS << '\n';
688 }
689 OS.flush();
690 assert(ResI == Res.end());
691}
692
693Error LTO::add(std::unique_ptr<InputFile> Input,
694 ArrayRef<SymbolResolution> Res) {
695 assert(!CalledGetMaxTasks);
696
697 if (Conf.ResolutionFile)
698 writeToResolutionFile(OS&: *Conf.ResolutionFile, Input: Input.get(), Res);
699
700 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
701 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
702 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
703 Conf.VisibilityScheme = Config::ELF;
704 }
705
706 const SymbolResolution *ResI = Res.begin();
707 for (unsigned I = 0; I != Input->Mods.size(); ++I)
708 if (Error Err = addModule(Input&: *Input, ModI: I, ResI, ResE: Res.end()))
709 return Err;
710
711 assert(ResI == Res.end());
712 return Error::success();
713}
714
715Error LTO::addModule(InputFile &Input, unsigned ModI,
716 const SymbolResolution *&ResI,
717 const SymbolResolution *ResE) {
718 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
719 if (!LTOInfo)
720 return LTOInfo.takeError();
721
722 if (EnableSplitLTOUnit) {
723 // If only some modules were split, flag this in the index so that
724 // we can skip or error on optimizations that need consistently split
725 // modules (whole program devirt and lower type tests).
726 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
727 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
728 } else
729 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
730
731 BitcodeModule BM = Input.Mods[ModI];
732
733 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
734 !LTOInfo->UnifiedLTO)
735 return make_error<StringError>(
736 Args: "unified LTO compilation must use "
737 "compatible bitcode modules (use -funified-lto)",
738 Args: inconvertibleErrorCode());
739
740 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
741 LTOMode = LTOK_UnifiedThin;
742
743 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
744
745 auto ModSyms = Input.module_symbols(I: ModI);
746 addModuleToGlobalRes(Syms: ModSyms, Res: {ResI, ResE},
747 Partition: IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
748 InSummary: LTOInfo->HasSummary);
749
750 if (IsThinLTO)
751 return addThinLTO(BM, Syms: ModSyms, ResI, ResE);
752
753 RegularLTO.EmptyCombinedModule = false;
754 Expected<RegularLTOState::AddedModule> ModOrErr =
755 addRegularLTO(BM, Syms: ModSyms, ResI, ResE);
756 if (!ModOrErr)
757 return ModOrErr.takeError();
758
759 if (!LTOInfo->HasSummary)
760 return linkRegularLTO(Mod: std::move(*ModOrErr), /*LivenessFromIndex=*/false);
761
762 // Regular LTO module summaries are added to a dummy module that represents
763 // the combined regular LTO module.
764 if (Error Err = BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: ""))
765 return Err;
766 RegularLTO.ModsWithSummaries.push_back(x: std::move(*ModOrErr));
767 return Error::success();
768}
769
770// Checks whether the given global value is in a non-prevailing comdat
771// (comdat containing values the linker indicated were not prevailing,
772// which we then dropped to available_externally), and if so, removes
773// it from the comdat. This is called for all global values to ensure the
774// comdat is empty rather than leaving an incomplete comdat. It is needed for
775// regular LTO modules, in case we are in a mixed-LTO mode (both regular
776// and thin LTO modules) compilation. Since the regular LTO module will be
777// linked first in the final native link, we want to make sure the linker
778// doesn't select any of these incomplete comdats that would be left
779// in the regular LTO module without this cleanup.
780static void
781handleNonPrevailingComdat(GlobalValue &GV,
782 std::set<const Comdat *> &NonPrevailingComdats) {
783 Comdat *C = GV.getComdat();
784 if (!C)
785 return;
786
787 if (!NonPrevailingComdats.count(x: C))
788 return;
789
790 // Additionally need to drop all global values from the comdat to
791 // available_externally, to satisfy the COMDAT requirement that all members
792 // are discarded as a unit. The non-local linkage global values avoid
793 // duplicate definition linker errors.
794 GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
795
796 if (auto GO = dyn_cast<GlobalObject>(Val: &GV))
797 GO->setComdat(nullptr);
798}
799
800// Add a regular LTO object to the link.
801// The resulting module needs to be linked into the combined LTO module with
802// linkRegularLTO.
803Expected<LTO::RegularLTOState::AddedModule>
804LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
805 const SymbolResolution *&ResI,
806 const SymbolResolution *ResE) {
807 RegularLTOState::AddedModule Mod;
808 Expected<std::unique_ptr<Module>> MOrErr =
809 BM.getLazyModule(Context&: RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
810 /*IsImporting*/ false);
811 if (!MOrErr)
812 return MOrErr.takeError();
813 Module &M = **MOrErr;
814 Mod.M = std::move(*MOrErr);
815
816 if (Error Err = M.materializeMetadata())
817 return std::move(Err);
818
819 // If cfi.functions is present and we are in regular LTO mode, LowerTypeTests
820 // will rename local functions in the merged module as "<function name>.1".
821 // This causes linking errors, since other parts of the module expect the
822 // original function name.
823 if (LTOMode == LTOK_UnifiedRegular)
824 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata(Name: "cfi.functions"))
825 M.eraseNamedMetadata(NMD: CfiFunctionsMD);
826
827 UpgradeDebugInfo(M);
828
829 ModuleSymbolTable SymTab;
830 SymTab.addModule(M: &M);
831
832 for (GlobalVariable &GV : M.globals())
833 if (GV.hasAppendingLinkage())
834 Mod.Keep.push_back(x: &GV);
835
836 DenseSet<GlobalObject *> AliasedGlobals;
837 for (auto &GA : M.aliases())
838 if (GlobalObject *GO = GA.getAliaseeObject())
839 AliasedGlobals.insert(V: GO);
840
841 // In this function we need IR GlobalValues matching the symbols in Syms
842 // (which is not backed by a module), so we need to enumerate them in the same
843 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
844 // matches the order of an irsymtab, but when we read the irsymtab in
845 // InputFile::create we omit some symbols that are irrelevant to LTO. The
846 // Skip() function skips the same symbols from the module as InputFile does
847 // from the symbol table.
848 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
849 auto Skip = [&]() {
850 while (MsymI != MsymE) {
851 auto Flags = SymTab.getSymbolFlags(S: *MsymI);
852 if ((Flags & object::BasicSymbolRef::SF_Global) &&
853 !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
854 return;
855 ++MsymI;
856 }
857 };
858 Skip();
859
860 std::set<const Comdat *> NonPrevailingComdats;
861 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
862 for (const InputFile::Symbol &Sym : Syms) {
863 assert(ResI != ResE);
864 SymbolResolution Res = *ResI++;
865
866 assert(MsymI != MsymE);
867 ModuleSymbolTable::Symbol Msym = *MsymI++;
868 Skip();
869
870 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Val&: Msym)) {
871 if (Res.Prevailing) {
872 if (Sym.isUndefined())
873 continue;
874 Mod.Keep.push_back(x: GV);
875 // For symbols re-defined with linker -wrap and -defsym options,
876 // set the linkage to weak to inhibit IPO. The linkage will be
877 // restored by the linker.
878 if (Res.LinkerRedefined)
879 GV->setLinkage(GlobalValue::WeakAnyLinkage);
880
881 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
882 if (GlobalValue::isLinkOnceLinkage(Linkage: OriginalLinkage))
883 GV->setLinkage(GlobalValue::getWeakLinkage(
884 ODR: GlobalValue::isLinkOnceODRLinkage(Linkage: OriginalLinkage)));
885 } else if (isa<GlobalObject>(Val: GV) &&
886 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
887 GV->hasAvailableExternallyLinkage()) &&
888 !AliasedGlobals.count(V: cast<GlobalObject>(Val: GV))) {
889 // Any of the above three types of linkage indicates that the
890 // chosen prevailing symbol will have the same semantics as this copy of
891 // the symbol, so we may be able to link it with available_externally
892 // linkage. We will decide later whether to do that when we link this
893 // module (in linkRegularLTO), based on whether it is undefined.
894 Mod.Keep.push_back(x: GV);
895 GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
896 if (GV->hasComdat())
897 NonPrevailingComdats.insert(x: GV->getComdat());
898 cast<GlobalObject>(Val: GV)->setComdat(nullptr);
899 }
900
901 // Set the 'local' flag based on the linker resolution for this symbol.
902 if (Res.FinalDefinitionInLinkageUnit) {
903 GV->setDSOLocal(true);
904 if (GV->hasDLLImportStorageClass())
905 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
906 DefaultStorageClass);
907 }
908 } else if (auto *AS =
909 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Val&: Msym)) {
910 // Collect non-prevailing symbols.
911 if (!Res.Prevailing)
912 NonPrevailingAsmSymbols.insert(V: AS->first);
913 } else {
914 llvm_unreachable("unknown symbol type");
915 }
916
917 // Common resolution: collect the maximum size/alignment over all commons.
918 // We also record if we see an instance of a common as prevailing, so that
919 // if none is prevailing we can ignore it later.
920 if (Sym.isCommon()) {
921 // FIXME: We should figure out what to do about commons defined by asm.
922 // For now they aren't reported correctly by ModuleSymbolTable.
923 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
924 CommonRes.Size = std::max(a: CommonRes.Size, b: Sym.getCommonSize());
925 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
926 CommonRes.Alignment =
927 std::max(a: Align(SymAlignValue), b: CommonRes.Alignment);
928 }
929 CommonRes.Prevailing |= Res.Prevailing;
930 }
931 }
932
933 if (!M.getComdatSymbolTable().empty())
934 for (GlobalValue &GV : M.global_values())
935 handleNonPrevailingComdat(GV, NonPrevailingComdats);
936
937 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
938 // block.
939 if (!M.getModuleInlineAsm().empty()) {
940 std::string NewIA = ".lto_discard";
941 if (!NonPrevailingAsmSymbols.empty()) {
942 // Don't dicard a symbol if there is a live .symver for it.
943 ModuleSymbolTable::CollectAsmSymvers(
944 M, AsmSymver: [&](StringRef Name, StringRef Alias) {
945 if (!NonPrevailingAsmSymbols.count(V: Alias))
946 NonPrevailingAsmSymbols.erase(V: Name);
947 });
948 NewIA += " " + llvm::join(R&: NonPrevailingAsmSymbols, Separator: ", ");
949 }
950 NewIA += "\n";
951 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
952 }
953
954 assert(MsymI == MsymE);
955 return std::move(Mod);
956}
957
958Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
959 bool LivenessFromIndex) {
960 std::vector<GlobalValue *> Keep;
961 for (GlobalValue *GV : Mod.Keep) {
962 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GUID: GV->getGUID())) {
963 if (Function *F = dyn_cast<Function>(Val: GV)) {
964 if (DiagnosticOutputFile) {
965 if (Error Err = F->materialize())
966 return Err;
967 OptimizationRemarkEmitter ORE(F, nullptr);
968 ORE.emit(OptDiag&: OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
969 << ore::NV("Function", F)
970 << " not added to the combined module ");
971 }
972 }
973 continue;
974 }
975
976 if (!GV->hasAvailableExternallyLinkage()) {
977 Keep.push_back(x: GV);
978 continue;
979 }
980
981 // Only link available_externally definitions if we don't already have a
982 // definition.
983 GlobalValue *CombinedGV =
984 RegularLTO.CombinedModule->getNamedValue(Name: GV->getName());
985 if (CombinedGV && !CombinedGV->isDeclaration())
986 continue;
987
988 Keep.push_back(x: GV);
989 }
990
991 return RegularLTO.Mover->move(Src: std::move(Mod.M), ValuesToLink: Keep, AddLazyFor: nullptr,
992 /* IsPerformingImport */ false);
993}
994
995// Add a ThinLTO module to the link.
996Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
997 const SymbolResolution *&ResI,
998 const SymbolResolution *ResE) {
999 const SymbolResolution *ResITmp = ResI;
1000 for (const InputFile::Symbol &Sym : Syms) {
1001 assert(ResITmp != ResE);
1002 SymbolResolution Res = *ResITmp++;
1003
1004 if (!Sym.getIRName().empty()) {
1005 auto GUID = GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier(
1006 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: ""));
1007 if (Res.Prevailing)
1008 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
1009 }
1010 }
1011
1012 if (Error Err =
1013 BM.readSummary(CombinedIndex&: ThinLTO.CombinedIndex, ModulePath: BM.getModuleIdentifier(),
1014 IsPrevailing: [&](GlobalValue::GUID GUID) {
1015 return ThinLTO.PrevailingModuleForGUID[GUID] ==
1016 BM.getModuleIdentifier();
1017 }))
1018 return Err;
1019 LLVM_DEBUG(dbgs() << "Module " << BM.getModuleIdentifier() << "\n");
1020
1021 for (const InputFile::Symbol &Sym : Syms) {
1022 assert(ResI != ResE);
1023 SymbolResolution Res = *ResI++;
1024
1025 if (!Sym.getIRName().empty()) {
1026 auto GUID = GlobalValue::getGUID(GlobalName: GlobalValue::getGlobalIdentifier(
1027 Name: Sym.getIRName(), Linkage: GlobalValue::ExternalLinkage, FileName: ""));
1028 if (Res.Prevailing) {
1029 assert(ThinLTO.PrevailingModuleForGUID[GUID] ==
1030 BM.getModuleIdentifier());
1031
1032 // For linker redefined symbols (via --wrap or --defsym) we want to
1033 // switch the linkage to `weak` to prevent IPOs from happening.
1034 // Find the summary in the module for this very GV and record the new
1035 // linkage so that we can switch it when we import the GV.
1036 if (Res.LinkerRedefined)
1037 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1038 ValueGUID: GUID, ModuleId: BM.getModuleIdentifier()))
1039 S->setLinkage(GlobalValue::WeakAnyLinkage);
1040 }
1041
1042 // If the linker resolved the symbol to a local definition then mark it
1043 // as local in the summary for the module we are adding.
1044 if (Res.FinalDefinitionInLinkageUnit) {
1045 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1046 ValueGUID: GUID, ModuleId: BM.getModuleIdentifier())) {
1047 S->setDSOLocal(true);
1048 }
1049 }
1050 }
1051 }
1052
1053 if (!ThinLTO.ModuleMap.insert(KV: {BM.getModuleIdentifier(), BM}).second)
1054 return make_error<StringError>(
1055 Args: "Expected at most one ThinLTO module per bitcode file",
1056 Args: inconvertibleErrorCode());
1057
1058 if (!Conf.ThinLTOModulesToCompile.empty()) {
1059 if (!ThinLTO.ModulesToCompile)
1060 ThinLTO.ModulesToCompile = ModuleMapType();
1061 // This is a fuzzy name matching where only modules with name containing the
1062 // specified switch values are going to be compiled.
1063 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1064 if (BM.getModuleIdentifier().contains(Other: Name)) {
1065 ThinLTO.ModulesToCompile->insert(KV: {BM.getModuleIdentifier(), BM});
1066 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
1067 << " to compile\n";
1068 }
1069 }
1070 }
1071
1072 return Error::success();
1073}
1074
1075unsigned LTO::getMaxTasks() const {
1076 CalledGetMaxTasks = true;
1077 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1078 : ThinLTO.ModuleMap.size();
1079 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1080}
1081
1082// If only some of the modules were split, we cannot correctly handle
1083// code that contains type tests or type checked loads.
1084Error LTO::checkPartiallySplit() {
1085 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1086 return Error::success();
1087
1088 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
1089 Name: Intrinsic::getName(Intrinsic::type_test));
1090 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
1091 Name: Intrinsic::getName(Intrinsic::type_checked_load));
1092 Function *TypeCheckedLoadRelativeFunc =
1093 RegularLTO.CombinedModule->getFunction(
1094 Name: Intrinsic::getName(Intrinsic::type_checked_load_relative));
1095
1096 // First check if there are type tests / type checked loads in the
1097 // merged regular LTO module IR.
1098 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1099 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1100 (TypeCheckedLoadRelativeFunc &&
1101 !TypeCheckedLoadRelativeFunc->use_empty()))
1102 return make_error<StringError>(
1103 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1104 Args: inconvertibleErrorCode());
1105
1106 // Otherwise check if there are any recorded in the combined summary from the
1107 // ThinLTO modules.
1108 for (auto &P : ThinLTO.CombinedIndex) {
1109 for (auto &S : P.second.SummaryList) {
1110 auto *FS = dyn_cast<FunctionSummary>(Val: S.get());
1111 if (!FS)
1112 continue;
1113 if (!FS->type_test_assume_vcalls().empty() ||
1114 !FS->type_checked_load_vcalls().empty() ||
1115 !FS->type_test_assume_const_vcalls().empty() ||
1116 !FS->type_checked_load_const_vcalls().empty() ||
1117 !FS->type_tests().empty())
1118 return make_error<StringError>(
1119 Args: "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1120 Args: inconvertibleErrorCode());
1121 }
1122 }
1123 return Error::success();
1124}
1125
1126Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
1127 // Compute "dead" symbols, we don't want to import/export these!
1128 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1129 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1130 for (auto &Res : *GlobalResolutions) {
1131 // Normally resolution have IR name of symbol. We can do nothing here
1132 // otherwise. See comments in GlobalResolution struct for more details.
1133 if (Res.second.IRName.empty())
1134 continue;
1135
1136 GlobalValue::GUID GUID = GlobalValue::getGUID(
1137 GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Res.second.IRName));
1138
1139 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1140 GUIDPreservedSymbols.insert(V: GUID);
1141
1142 if (Res.second.ExportDynamic)
1143 DynamicExportSymbols.insert(V: GUID);
1144
1145 GUIDPrevailingResolutions[GUID] =
1146 Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
1147 }
1148
1149 auto isPrevailing = [&](GlobalValue::GUID G) {
1150 auto It = GUIDPrevailingResolutions.find(Val: G);
1151 if (It == GUIDPrevailingResolutions.end())
1152 return PrevailingType::Unknown;
1153 return It->second;
1154 };
1155 computeDeadSymbolsWithConstProp(Index&: ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1156 isPrevailing, ImportEnabled: Conf.OptLevel > 0);
1157
1158 // Setup output file to emit statistics.
1159 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1160 if (!StatsFileOrErr)
1161 return StatsFileOrErr.takeError();
1162 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1163
1164 // TODO: Ideally this would be controlled automatically by detecting that we
1165 // are linking with an allocator that supports these interfaces, rather than
1166 // an internal option (which would still be needed for tests, however). For
1167 // example, if the library exported a symbol like __malloc_hot_cold the linker
1168 // could recognize that and set a flag in the lto::Config.
1169 if (SupportsHotColdNew)
1170 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1171
1172 Error Result = runRegularLTO(AddStream);
1173 if (!Result)
1174 // This will reset the GlobalResolutions optional once done with it to
1175 // reduce peak memory before importing.
1176 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1177
1178 if (StatsFile)
1179 PrintStatisticsJSON(OS&: StatsFile->os());
1180
1181 return Result;
1182}
1183
1184void lto::updateMemProfAttributes(Module &Mod,
1185 const ModuleSummaryIndex &Index) {
1186 if (Index.withSupportsHotColdNew())
1187 return;
1188
1189 // The profile matcher applies hotness attributes directly for allocations,
1190 // and those will cause us to generate calls to the hot/cold interfaces
1191 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
1192 // link then assume we don't want these calls (e.g. not linking with
1193 // the appropriate library, or otherwise trying to disable this behavior).
1194 for (auto &F : Mod) {
1195 for (auto &BB : F) {
1196 for (auto &I : BB) {
1197 auto *CI = dyn_cast<CallBase>(Val: &I);
1198 if (!CI)
1199 continue;
1200 if (CI->hasFnAttr(Kind: "memprof"))
1201 CI->removeFnAttr(Kind: "memprof");
1202 // Strip off all memprof metadata as it is no longer needed.
1203 // Importantly, this avoids the addition of new memprof attributes
1204 // after inlining propagation.
1205 // TODO: If we support additional types of MemProf metadata beyond hot
1206 // and cold, we will need to update the metadata based on the allocator
1207 // APIs supported instead of completely stripping all.
1208 CI->setMetadata(KindID: LLVMContext::MD_memprof, Node: nullptr);
1209 CI->setMetadata(KindID: LLVMContext::MD_callsite, Node: nullptr);
1210 }
1211 }
1212 }
1213}
1214
1215Error LTO::runRegularLTO(AddStreamFn AddStream) {
1216 // Setup optimization remarks.
1217 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1218 Context&: RegularLTO.CombinedModule->getContext(), RemarksFilename: Conf.RemarksFilename,
1219 RemarksPasses: Conf.RemarksPasses, RemarksFormat: Conf.RemarksFormat, RemarksWithHotness: Conf.RemarksWithHotness,
1220 RemarksHotnessThreshold: Conf.RemarksHotnessThreshold);
1221 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1222 if (!DiagFileOrErr)
1223 return DiagFileOrErr.takeError();
1224 DiagnosticOutputFile = std::move(*DiagFileOrErr);
1225
1226 // Finalize linking of regular LTO modules containing summaries now that
1227 // we have computed liveness information.
1228 for (auto &M : RegularLTO.ModsWithSummaries)
1229 if (Error Err = linkRegularLTO(Mod: std::move(M),
1230 /*LivenessFromIndex=*/true))
1231 return Err;
1232
1233 // Ensure we don't have inconsistently split LTO units with type tests.
1234 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1235 // this path both cases but eventually this should be split into two and
1236 // do the ThinLTO checks in `runThinLTO`.
1237 if (Error Err = checkPartiallySplit())
1238 return Err;
1239
1240 // Make sure commons have the right size/alignment: we kept the largest from
1241 // all the prevailing when adding the inputs, and we apply it here.
1242 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1243 for (auto &I : RegularLTO.Commons) {
1244 if (!I.second.Prevailing)
1245 // Don't do anything if no instance of this common was prevailing.
1246 continue;
1247 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(Name: I.first);
1248 if (OldGV && DL.getTypeAllocSize(Ty: OldGV->getValueType()) == I.second.Size) {
1249 // Don't create a new global if the type is already correct, just make
1250 // sure the alignment is correct.
1251 OldGV->setAlignment(I.second.Alignment);
1252 continue;
1253 }
1254 ArrayType *Ty =
1255 ArrayType::get(ElementType: Type::getInt8Ty(C&: RegularLTO.Ctx), NumElements: I.second.Size);
1256 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1257 GlobalValue::CommonLinkage,
1258 ConstantAggregateZero::get(Ty), "");
1259 GV->setAlignment(I.second.Alignment);
1260 if (OldGV) {
1261 OldGV->replaceAllUsesWith(V: GV);
1262 GV->takeName(V: OldGV);
1263 OldGV->eraseFromParent();
1264 } else {
1265 GV->setName(I.first);
1266 }
1267 }
1268
1269 updateMemProfAttributes(Mod&: *RegularLTO.CombinedModule, Index: ThinLTO.CombinedIndex);
1270
1271 bool WholeProgramVisibilityEnabledInLTO =
1272 Conf.HasWholeProgramVisibility &&
1273 // If validation is enabled, upgrade visibility only when all vtables
1274 // have typeinfos.
1275 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
1276
1277 // This returns true when the name is local or not defined. Locals are
1278 // expected to be handled separately.
1279 auto IsVisibleToRegularObj = [&](StringRef name) {
1280 auto It = GlobalResolutions->find(Key: name);
1281 return (It == GlobalResolutions->end() || It->second.VisibleOutsideSummary);
1282 };
1283
1284 // If allowed, upgrade public vcall visibility metadata to linkage unit
1285 // visibility before whole program devirtualization in the optimizer.
1286 updateVCallVisibilityInModule(
1287 M&: *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1288 DynamicExportSymbols, ValidateAllVtablesHaveTypeInfos: Conf.ValidateAllVtablesHaveTypeInfos,
1289 IsVisibleToRegularObj);
1290 updatePublicTypeTestCalls(M&: *RegularLTO.CombinedModule,
1291 WholeProgramVisibilityEnabledInLTO);
1292
1293 if (Conf.PreOptModuleHook &&
1294 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1295 return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile));
1296
1297 if (!Conf.CodeGenOnly) {
1298 for (const auto &R : *GlobalResolutions) {
1299 GlobalValue *GV =
1300 RegularLTO.CombinedModule->getNamedValue(Name: R.second.IRName);
1301 if (!R.second.isPrevailingIRSymbol())
1302 continue;
1303 if (R.second.Partition != 0 &&
1304 R.second.Partition != GlobalResolution::External)
1305 continue;
1306
1307 // Ignore symbols defined in other partitions.
1308 // Also skip declarations, which are not allowed to have internal linkage.
1309 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1310 continue;
1311
1312 // Symbols that are marked DLLImport or DLLExport should not be
1313 // internalized, as they are either externally visible or referencing
1314 // external symbols. Symbols that have AvailableExternally or Appending
1315 // linkage might be used by future passes and should be kept as is.
1316 // These linkages are seen in Unified regular LTO, because the process
1317 // of creating split LTO units introduces symbols with that linkage into
1318 // one of the created modules. Normally, only the ThinLTO backend would
1319 // compile this module, but Unified Regular LTO processes both
1320 // modules created by the splitting process as regular LTO modules.
1321 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1322 ((GV->getDLLStorageClass() != GlobalValue::DefaultStorageClass) ||
1323 GV->hasAvailableExternallyLinkage() || GV->hasAppendingLinkage()))
1324 continue;
1325
1326 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1327 : GlobalValue::UnnamedAddr::None);
1328 if (EnableLTOInternalization && R.second.Partition == 0)
1329 GV->setLinkage(GlobalValue::InternalLinkage);
1330 }
1331
1332 if (Conf.PostInternalizeModuleHook &&
1333 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1334 return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile));
1335 }
1336
1337 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1338 if (Error Err =
1339 backend(C: Conf, AddStream, ParallelCodeGenParallelismLevel: RegularLTO.ParallelCodeGenParallelismLevel,
1340 M&: *RegularLTO.CombinedModule, CombinedIndex&: ThinLTO.CombinedIndex))
1341 return Err;
1342 }
1343
1344 return finalizeOptimizationRemarks(DiagOutputFile: std::move(DiagnosticOutputFile));
1345}
1346
1347static const char *libcallRoutineNames[] = {
1348#define HANDLE_LIBCALL(code, name) name,
1349#include "llvm/IR/RuntimeLibcalls.def"
1350#undef HANDLE_LIBCALL
1351};
1352
1353ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
1354 return ArrayRef(libcallRoutineNames);
1355}
1356
1357/// This class defines the interface to the ThinLTO backend.
1358class lto::ThinBackendProc {
1359protected:
1360 const Config &Conf;
1361 ModuleSummaryIndex &CombinedIndex;
1362 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries;
1363 lto::IndexWriteCallback OnWrite;
1364 bool ShouldEmitImportsFiles;
1365
1366public:
1367 ThinBackendProc(
1368 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1369 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1370 lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
1371 : Conf(Conf), CombinedIndex(CombinedIndex),
1372 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries),
1373 OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {}
1374
1375 virtual ~ThinBackendProc() = default;
1376 virtual Error start(
1377 unsigned Task, BitcodeModule BM,
1378 const FunctionImporter::ImportMapTy &ImportList,
1379 const FunctionImporter::ExportSetTy &ExportList,
1380 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1381 MapVector<StringRef, BitcodeModule> &ModuleMap) = 0;
1382 virtual Error wait() = 0;
1383 virtual unsigned getThreadCount() = 0;
1384
1385 // Write sharded indices and (optionally) imports to disk
1386 Error emitFiles(const FunctionImporter::ImportMapTy &ImportList,
1387 llvm::StringRef ModulePath,
1388 const std::string &NewModulePath) {
1389 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1390 std::error_code EC;
1391 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1392 ImportList, ModuleToSummariesForIndex);
1393
1394 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1395 sys::fs::OpenFlags::OF_None);
1396 if (EC)
1397 return errorCodeToError(EC);
1398 writeIndexToFile(Index: CombinedIndex, Out&: OS, ModuleToSummariesForIndex: &ModuleToSummariesForIndex);
1399
1400 if (ShouldEmitImportsFiles) {
1401 EC = EmitImportsFiles(ModulePath, OutputFilename: NewModulePath + ".imports",
1402 ModuleToSummariesForIndex);
1403 if (EC)
1404 return errorCodeToError(EC);
1405 }
1406 return Error::success();
1407 }
1408};
1409
1410namespace {
1411class InProcessThinBackend : public ThinBackendProc {
1412 ThreadPool BackendThreadPool;
1413 AddStreamFn AddStream;
1414 FileCache Cache;
1415 std::set<GlobalValue::GUID> CfiFunctionDefs;
1416 std::set<GlobalValue::GUID> CfiFunctionDecls;
1417
1418 std::optional<Error> Err;
1419 std::mutex ErrMu;
1420
1421 bool ShouldEmitIndexFiles;
1422
1423public:
1424 InProcessThinBackend(
1425 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1426 ThreadPoolStrategy ThinLTOParallelism,
1427 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1428 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1429 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
1430 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1431 OnWrite, ShouldEmitImportsFiles),
1432 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1433 Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1434 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1435 CfiFunctionDefs.insert(
1436 x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name)));
1437 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1438 CfiFunctionDecls.insert(
1439 x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name)));
1440 }
1441
1442 Error runThinLTOBackendThread(
1443 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1444 ModuleSummaryIndex &CombinedIndex,
1445 const FunctionImporter::ImportMapTy &ImportList,
1446 const FunctionImporter::ExportSetTy &ExportList,
1447 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1448 const GVSummaryMapTy &DefinedGlobals,
1449 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1450 auto RunThinBackend = [&](AddStreamFn AddStream) {
1451 LTOLLVMContext BackendContext(Conf);
1452 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Context&: BackendContext);
1453 if (!MOrErr)
1454 return MOrErr.takeError();
1455
1456 return thinBackend(C: Conf, Task, AddStream, M&: **MOrErr, CombinedIndex,
1457 ImportList, DefinedGlobals, ModuleMap: &ModuleMap);
1458 };
1459
1460 auto ModuleID = BM.getModuleIdentifier();
1461
1462 if (ShouldEmitIndexFiles) {
1463 if (auto E = emitFiles(ImportList, ModulePath: ModuleID, NewModulePath: ModuleID.str()))
1464 return E;
1465 }
1466
1467 if (!Cache || !CombinedIndex.modulePaths().count(Key: ModuleID) ||
1468 all_of(Range: CombinedIndex.getModuleHash(ModPath: ModuleID),
1469 P: [](uint32_t V) { return V == 0; }))
1470 // Cache disabled or no entry for this module in the combined index or
1471 // no module hash.
1472 return RunThinBackend(AddStream);
1473
1474 SmallString<40> Key;
1475 // The module may be cached, this helps handling it.
1476 computeLTOCacheKey(Key, Conf, Index: CombinedIndex, ModuleID, ImportList,
1477 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1478 CfiFunctionDecls);
1479 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1480 if (Error Err = CacheAddStreamOrErr.takeError())
1481 return Err;
1482 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1483 if (CacheAddStream)
1484 return RunThinBackend(CacheAddStream);
1485
1486 return Error::success();
1487 }
1488
1489 Error start(
1490 unsigned Task, BitcodeModule BM,
1491 const FunctionImporter::ImportMapTy &ImportList,
1492 const FunctionImporter::ExportSetTy &ExportList,
1493 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1494 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1495 StringRef ModulePath = BM.getModuleIdentifier();
1496 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1497 const GVSummaryMapTy &DefinedGlobals =
1498 ModuleToDefinedGVSummaries.find(Val: ModulePath)->second;
1499 BackendThreadPool.async(
1500 F: [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1501 const FunctionImporter::ImportMapTy &ImportList,
1502 const FunctionImporter::ExportSetTy &ExportList,
1503 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1504 &ResolvedODR,
1505 const GVSummaryMapTy &DefinedGlobals,
1506 MapVector<StringRef, BitcodeModule> &ModuleMap) {
1507 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1508 timeTraceProfilerInitialize(TimeTraceGranularity: Conf.TimeTraceGranularity,
1509 ProcName: "thin backend");
1510 Error E = runThinLTOBackendThread(
1511 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1512 ResolvedODR, DefinedGlobals, ModuleMap);
1513 if (E) {
1514 std::unique_lock<std::mutex> L(ErrMu);
1515 if (Err)
1516 Err = joinErrors(E1: std::move(*Err), E2: std::move(E));
1517 else
1518 Err = std::move(E);
1519 }
1520 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1521 timeTraceProfilerFinishThread();
1522 },
1523 ArgList&: BM, ArgList: std::ref(t&: CombinedIndex), ArgList: std::ref(t: ImportList), ArgList: std::ref(t: ExportList),
1524 ArgList: std::ref(t: ResolvedODR), ArgList: std::ref(t: DefinedGlobals), ArgList: std::ref(t&: ModuleMap));
1525
1526 if (OnWrite)
1527 OnWrite(std::string(ModulePath));
1528 return Error::success();
1529 }
1530
1531 Error wait() override {
1532 BackendThreadPool.wait();
1533 if (Err)
1534 return std::move(*Err);
1535 else
1536 return Error::success();
1537 }
1538
1539 unsigned getThreadCount() override {
1540 return BackendThreadPool.getThreadCount();
1541 }
1542};
1543} // end anonymous namespace
1544
1545ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,
1546 lto::IndexWriteCallback OnWrite,
1547 bool ShouldEmitIndexFiles,
1548 bool ShouldEmitImportsFiles) {
1549 return
1550 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1551 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1552 AddStreamFn AddStream, FileCache Cache) {
1553 return std::make_unique<InProcessThinBackend>(
1554 args: Conf, args&: CombinedIndex, args: Parallelism, args: ModuleToDefinedGVSummaries,
1555 args&: AddStream, args&: Cache, args: OnWrite, args: ShouldEmitIndexFiles,
1556 args: ShouldEmitImportsFiles);
1557 };
1558}
1559
1560// Given the original \p Path to an output file, replace any path
1561// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1562// resulting directory if it does not yet exist.
1563std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,
1564 StringRef NewPrefix) {
1565 if (OldPrefix.empty() && NewPrefix.empty())
1566 return std::string(Path);
1567 SmallString<128> NewPath(Path);
1568 llvm::sys::path::replace_path_prefix(Path&: NewPath, OldPrefix, NewPrefix);
1569 StringRef ParentPath = llvm::sys::path::parent_path(path: NewPath.str());
1570 if (!ParentPath.empty()) {
1571 // Make sure the new directory exists, creating it if necessary.
1572 if (std::error_code EC = llvm::sys::fs::create_directories(path: ParentPath))
1573 llvm::errs() << "warning: could not create directory '" << ParentPath
1574 << "': " << EC.message() << '\n';
1575 }
1576 return std::string(NewPath);
1577}
1578
1579namespace {
1580class WriteIndexesThinBackend : public ThinBackendProc {
1581 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1582 raw_fd_ostream *LinkedObjectsFile;
1583
1584public:
1585 WriteIndexesThinBackend(
1586 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1587 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1588 std::string OldPrefix, std::string NewPrefix,
1589 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1590 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1591 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1592 OnWrite, ShouldEmitImportsFiles),
1593 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1594 NativeObjectPrefix(NativeObjectPrefix),
1595 LinkedObjectsFile(LinkedObjectsFile) {}
1596
1597 Error start(
1598 unsigned Task, BitcodeModule BM,
1599 const FunctionImporter::ImportMapTy &ImportList,
1600 const FunctionImporter::ExportSetTy &ExportList,
1601 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1602 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1603 StringRef ModulePath = BM.getModuleIdentifier();
1604 std::string NewModulePath =
1605 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix);
1606
1607 if (LinkedObjectsFile) {
1608 std::string ObjectPrefix =
1609 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1610 std::string LinkedObjectsFilePath =
1611 getThinLTOOutputFile(Path: ModulePath, OldPrefix, NewPrefix: ObjectPrefix);
1612 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1613 }
1614
1615 if (auto E = emitFiles(ImportList, ModulePath, NewModulePath))
1616 return E;
1617
1618 if (OnWrite)
1619 OnWrite(std::string(ModulePath));
1620 return Error::success();
1621 }
1622
1623 Error wait() override { return Error::success(); }
1624
1625 // WriteIndexesThinBackend should always return 1 to prevent module
1626 // re-ordering and avoid non-determinism in the final link.
1627 unsigned getThreadCount() override { return 1; }
1628};
1629} // end anonymous namespace
1630
1631ThinBackend lto::createWriteIndexesThinBackend(
1632 std::string OldPrefix, std::string NewPrefix,
1633 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1634 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1635 return
1636 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1637 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1638 AddStreamFn AddStream, FileCache Cache) {
1639 return std::make_unique<WriteIndexesThinBackend>(
1640 args: Conf, args&: CombinedIndex, args: ModuleToDefinedGVSummaries, args: OldPrefix,
1641 args: NewPrefix, args: NativeObjectPrefix, args: ShouldEmitImportsFiles,
1642 args: LinkedObjectsFile, args: OnWrite);
1643 };
1644}
1645
1646Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
1647 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1648 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
1649 ThinLTO.CombinedIndex.releaseTemporaryMemory();
1650 timeTraceProfilerBegin(Name: "ThinLink", Detail: StringRef(""));
1651 auto TimeTraceScopeExit = llvm::make_scope_exit(F: []() {
1652 if (llvm::timeTraceProfilerEnabled())
1653 llvm::timeTraceProfilerEnd();
1654 });
1655 if (ThinLTO.ModuleMap.empty())
1656 return Error::success();
1657
1658 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1659 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1660 return Error::success();
1661 }
1662
1663 if (Conf.CombinedIndexHook &&
1664 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1665 return Error::success();
1666
1667 // Collect for each module the list of function it defines (GUID ->
1668 // Summary).
1669 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
1670 ThinLTO.ModuleMap.size());
1671 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1672 ModuleToDefinedGVSummaries);
1673 // Create entries for any modules that didn't have any GV summaries
1674 // (either they didn't have any GVs to start with, or we suppressed
1675 // generation of the summaries because they e.g. had inline assembly
1676 // uses that couldn't be promoted/renamed on export). This is so
1677 // InProcessThinBackend::start can still launch a backend thread, which
1678 // is passed the map of summaries for the module, without any special
1679 // handling for this case.
1680 for (auto &Mod : ThinLTO.ModuleMap)
1681 if (!ModuleToDefinedGVSummaries.count(Val: Mod.first))
1682 ModuleToDefinedGVSummaries.try_emplace(Key: Mod.first);
1683
1684 // Synthesize entry counts for functions in the CombinedIndex.
1685 computeSyntheticCounts(Index&: ThinLTO.CombinedIndex);
1686
1687 DenseMap<StringRef, FunctionImporter::ImportMapTy> ImportLists(
1688 ThinLTO.ModuleMap.size());
1689 DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(
1690 ThinLTO.ModuleMap.size());
1691 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1692
1693 if (DumpThinCGSCCs)
1694 ThinLTO.CombinedIndex.dumpSCCs(OS&: outs());
1695
1696 std::set<GlobalValue::GUID> ExportedGUIDs;
1697
1698 bool WholeProgramVisibilityEnabledInLTO =
1699 Conf.HasWholeProgramVisibility &&
1700 // If validation is enabled, upgrade visibility only when all vtables
1701 // have typeinfos.
1702 (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);
1703 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
1704 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
1705
1706 // If we're validating, get the vtable symbols that should not be
1707 // upgraded because they correspond to typeIDs outside of index-based
1708 // WPD info.
1709 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
1710 if (WholeProgramVisibilityEnabledInLTO &&
1711 Conf.ValidateAllVtablesHaveTypeInfos) {
1712 // This returns true when the name is local or not defined. Locals are
1713 // expected to be handled separately.
1714 auto IsVisibleToRegularObj = [&](StringRef name) {
1715 auto It = GlobalResolutions->find(Key: name);
1716 return (It == GlobalResolutions->end() ||
1717 It->second.VisibleOutsideSummary);
1718 };
1719
1720 getVisibleToRegularObjVtableGUIDs(Index&: ThinLTO.CombinedIndex,
1721 VisibleToRegularObjSymbols,
1722 IsVisibleToRegularObj);
1723 }
1724
1725 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1726 // the summaries before whole program devirtualization below.
1727 updateVCallVisibilityInIndex(
1728 Index&: ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
1729 DynamicExportSymbols, VisibleToRegularObjSymbols);
1730
1731 // Perform index-based WPD. This will return immediately if there are
1732 // no index entries in the typeIdMetadata map (e.g. if we are instead
1733 // performing IR-based WPD in hybrid regular/thin LTO mode).
1734 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1735 runWholeProgramDevirtOnIndex(Summary&: ThinLTO.CombinedIndex, ExportedGUIDs,
1736 LocalWPDTargetsMap);
1737
1738 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1739 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1740 };
1741 if (EnableMemProfContextDisambiguation) {
1742 MemProfContextDisambiguation ContextDisambiguation;
1743 ContextDisambiguation.run(Index&: ThinLTO.CombinedIndex, isPrevailing);
1744 }
1745
1746 // Figure out which symbols need to be internalized. This also needs to happen
1747 // at -O0 because summary-based DCE is implemented using internalization, and
1748 // we must apply DCE consistently with the full LTO module in order to avoid
1749 // undefined references during the final link.
1750 for (auto &Res : *GlobalResolutions) {
1751 // If the symbol does not have external references or it is not prevailing,
1752 // then not need to mark it as exported from a ThinLTO partition.
1753 if (Res.second.Partition != GlobalResolution::External ||
1754 !Res.second.isPrevailingIRSymbol())
1755 continue;
1756 auto GUID = GlobalValue::getGUID(
1757 GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Res.second.IRName));
1758 // Mark exported unless index-based analysis determined it to be dead.
1759 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1760 ExportedGUIDs.insert(x: GUID);
1761 }
1762
1763 // Reset the GlobalResolutions to deallocate the associated memory, as there
1764 // are no further accesses. We specifically want to do this before computing
1765 // cross module importing, which adds to peak memory via the computed import
1766 // and export lists.
1767 GlobalResolutions.reset();
1768
1769 if (Conf.OptLevel > 0)
1770 ComputeCrossModuleImport(Index: ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1771 isPrevailing, ImportLists, ExportLists);
1772
1773 // Any functions referenced by the jump table in the regular LTO object must
1774 // be exported.
1775 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1776 ExportedGUIDs.insert(
1777 x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Def)));
1778 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1779 ExportedGUIDs.insert(
1780 x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: Decl)));
1781
1782 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1783 const auto &ExportList = ExportLists.find(Val: ModuleIdentifier);
1784 return (ExportList != ExportLists.end() && ExportList->second.count(V: VI)) ||
1785 ExportedGUIDs.count(x: VI.getGUID());
1786 };
1787
1788 // Update local devirtualized targets that were exported by cross-module
1789 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1790 updateIndexWPDForExports(Summary&: ThinLTO.CombinedIndex, isExported,
1791 LocalWPDTargetsMap);
1792
1793 thinLTOInternalizeAndPromoteInIndex(Index&: ThinLTO.CombinedIndex, isExported,
1794 isPrevailing);
1795
1796 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1797 GlobalValue::GUID GUID,
1798 GlobalValue::LinkageTypes NewLinkage) {
1799 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1800 };
1801 thinLTOResolvePrevailingInIndex(C: Conf, Index&: ThinLTO.CombinedIndex, isPrevailing,
1802 recordNewLinkage, GUIDPreservedSymbols);
1803
1804 thinLTOPropagateFunctionAttrs(Index&: ThinLTO.CombinedIndex, isPrevailing);
1805
1806 generateParamAccessSummary(Index&: ThinLTO.CombinedIndex);
1807
1808 if (llvm::timeTraceProfilerEnabled())
1809 llvm::timeTraceProfilerEnd();
1810
1811 TimeTraceScopeExit.release();
1812
1813 std::unique_ptr<ThinBackendProc> BackendProc =
1814 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1815 AddStream, Cache);
1816
1817 auto &ModuleMap =
1818 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1819
1820 auto ProcessOneModule = [&](int I) -> Error {
1821 auto &Mod = *(ModuleMap.begin() + I);
1822 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1823 // combined module and parallel code generation partitions.
1824 return BackendProc->start(Task: RegularLTO.ParallelCodeGenParallelismLevel + I,
1825 BM: Mod.second, ImportList: ImportLists[Mod.first],
1826 ExportList: ExportLists[Mod.first], ResolvedODR: ResolvedODR[Mod.first],
1827 ModuleMap&: ThinLTO.ModuleMap);
1828 };
1829
1830 if (BackendProc->getThreadCount() == 1) {
1831 // Process the modules in the order they were provided on the command-line.
1832 // It is important for this codepath to be used for WriteIndexesThinBackend,
1833 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1834 // order as the inputs, which otherwise would affect the final link order.
1835 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1836 if (Error E = ProcessOneModule(I))
1837 return E;
1838 } else {
1839 // When executing in parallel, process largest bitsize modules first to
1840 // improve parallelism, and avoid starving the thread pool near the end.
1841 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1842 // of 100 sec).
1843 std::vector<BitcodeModule *> ModulesVec;
1844 ModulesVec.reserve(n: ModuleMap.size());
1845 for (auto &Mod : ModuleMap)
1846 ModulesVec.push_back(x: &Mod.second);
1847 for (int I : generateModulesOrdering(R: ModulesVec))
1848 if (Error E = ProcessOneModule(I))
1849 return E;
1850 }
1851 return BackendProc->wait();
1852}
1853
1854Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
1855 LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
1856 StringRef RemarksFormat, bool RemarksWithHotness,
1857 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
1858 std::string Filename = std::string(RemarksFilename);
1859 // For ThinLTO, file.opt.<format> becomes
1860 // file.opt.<format>.thin.<num>.<format>.
1861 if (!Filename.empty() && Count != -1)
1862 Filename =
1863 (Twine(Filename) + ".thin." + llvm::utostr(X: Count) + "." + RemarksFormat)
1864 .str();
1865
1866 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1867 Context, RemarksFilename: Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,
1868 RemarksHotnessThreshold);
1869 if (Error E = ResultOrErr.takeError())
1870 return std::move(E);
1871
1872 if (*ResultOrErr)
1873 (*ResultOrErr)->keep();
1874
1875 return ResultOrErr;
1876}
1877
1878Expected<std::unique_ptr<ToolOutputFile>>
1879lto::setupStatsFile(StringRef StatsFilename) {
1880 // Setup output file to emit statistics.
1881 if (StatsFilename.empty())
1882 return nullptr;
1883
1884 llvm::EnableStatistics(DoPrintOnExit: false);
1885 std::error_code EC;
1886 auto StatsFile =
1887 std::make_unique<ToolOutputFile>(args&: StatsFilename, args&: EC, args: sys::fs::OF_None);
1888 if (EC)
1889 return errorCodeToError(EC);
1890
1891 StatsFile->keep();
1892 return std::move(StatsFile);
1893}
1894
1895// Compute the ordering we will process the inputs: the rough heuristic here
1896// is to sort them per size so that the largest module get schedule as soon as
1897// possible. This is purely a compile-time optimization.
1898std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {
1899 auto Seq = llvm::seq<int>(Begin: 0, End: R.size());
1900 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
1901 llvm::sort(C&: ModulesOrdering, Comp: [&](int LeftIndex, int RightIndex) {
1902 auto LSize = R[LeftIndex]->getBuffer().size();
1903 auto RSize = R[RightIndex]->getBuffer().size();
1904 return LSize > RSize;
1905 });
1906 return ModulesOrdering;
1907}
1908

source code of llvm/lib/LTO/LTO.cpp