1	//===- bolt/Rewrite/RewriteInstance.cpp - ELF rewriter --------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "bolt/Rewrite/RewriteInstance.h"
10	#include "bolt/Core/AddressMap.h"
11	#include "bolt/Core/BinaryContext.h"
12	#include "bolt/Core/BinaryEmitter.h"
13	#include "bolt/Core/BinaryFunction.h"
14	#include "bolt/Core/DebugData.h"
15	#include "bolt/Core/Exceptions.h"
16	#include "bolt/Core/FunctionLayout.h"
17	#include "bolt/Core/MCPlusBuilder.h"
18	#include "bolt/Core/ParallelUtilities.h"
19	#include "bolt/Core/Relocation.h"
20	#include "bolt/Passes/BinaryPasses.h"
21	#include "bolt/Passes/CacheMetrics.h"
22	#include "bolt/Passes/IdenticalCodeFolding.h"
23	#include "bolt/Passes/PAuthGadgetScanner.h"
24	#include "bolt/Passes/ReorderFunctions.h"
25	#include "bolt/Profile/BoltAddressTranslation.h"
26	#include "bolt/Profile/DataAggregator.h"
27	#include "bolt/Profile/DataReader.h"
28	#include "bolt/Profile/YAMLProfileReader.h"
29	#include "bolt/Profile/YAMLProfileWriter.h"
30	#include "bolt/Rewrite/BinaryPassManager.h"
31	#include "bolt/Rewrite/DWARFRewriter.h"
32	#include "bolt/Rewrite/ExecutableFileMemoryManager.h"
33	#include "bolt/Rewrite/JITLinkLinker.h"
34	#include "bolt/Rewrite/MetadataRewriters.h"
35	#include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h"
36	#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
37	#include "bolt/Utils/CommandLineOpts.h"
38	#include "bolt/Utils/Utils.h"
39	#include "llvm/ADT/AddressRanges.h"
40	#include "llvm/ADT/STLExtras.h"
41	#include "llvm/DebugInfo/DWARF/DWARFContext.h"
42	#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
43	#include "llvm/MC/MCAsmBackend.h"
44	#include "llvm/MC/MCAsmInfo.h"
45	#include "llvm/MC/MCDisassembler/MCDisassembler.h"
46	#include "llvm/MC/MCObjectStreamer.h"
47	#include "llvm/MC/MCStreamer.h"
48	#include "llvm/MC/MCSymbol.h"
49	#include "llvm/MC/TargetRegistry.h"
50	#include "llvm/Object/ObjectFile.h"
51	#include "llvm/Support/Alignment.h"
52	#include "llvm/Support/Casting.h"
53	#include "llvm/Support/CommandLine.h"
54	#include "llvm/Support/DataExtractor.h"
55	#include "llvm/Support/Errc.h"
56	#include "llvm/Support/Error.h"
57	#include "llvm/Support/FileSystem.h"
58	#include "llvm/Support/ManagedStatic.h"
59	#include "llvm/Support/Timer.h"
60	#include "llvm/Support/ToolOutputFile.h"
61	#include "llvm/Support/raw_ostream.h"
62	#include <algorithm>
63	#include <fstream>
64	#include <memory>
65	#include <optional>
66	#include <system_error>
67
68	#undef DEBUG_TYPE
69	#define DEBUG_TYPE "bolt"
70
71	using namespace llvm;
72	using namespace object;
73	using namespace bolt;
74
75	extern cl::opt<uint32_t> X86AlignBranchBoundary;
76	extern cl::opt<bool> X86AlignBranchWithin32BBoundaries;
77
78	namespace opts {
79
80	extern cl::list<std::string> HotTextMoveSections;
81	extern cl::opt<bool> Hugify;
82	extern cl::opt<bool> Instrument;
83	extern cl::opt<bool> KeepNops;
84	extern cl::opt<bool> Lite;
85	extern cl::list<std::string> ReorderData;
86	extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
87	extern cl::opt<bool> TerminalTrap;
88	extern cl::opt<bool> TimeBuild;
89	extern cl::opt<bool> TimeRewrite;
90	extern cl::opt<bolt::IdenticalCodeFolding::ICFLevel, false,
91	llvm::bolt::DeprecatedICFNumericOptionParser>
92	ICF;
93
94	static cl::opt<bool>
95	AllowStripped("allow-stripped",
96	cl::desc("allow processing of stripped binaries"), cl::Hidden,
97	cl::cat(BoltCategory));
98
99	static cl::opt<bool> ForceToDataRelocations(
100	"force-data-relocations",
101	cl::desc("force relocations to data sections to always be processed"),
102
103	cl::Hidden, cl::cat(BoltCategory));
104
105	static cl::opt<std::string>
106	BoltID("bolt-id",
107	cl::desc("add any string to tag this execution in the "
108	"output binary via bolt info section"),
109	cl::cat(BoltCategory));
110
111	cl::opt<bool> DumpDotAll(
112	"dump-dot-all",
113	cl::desc("dump function CFGs to graphviz format after each stage;"
114	"enable '-print-loops' for color-coded blocks"),
115	cl::Hidden, cl::cat(BoltCategory));
116
117	static cl::list<std::string>
118	ForceFunctionNames("funcs",
119	cl::CommaSeparated,
120	cl::desc("limit optimizations to functions from the list"),
121	cl::value_desc("func1,func2,func3,..."),
122	cl::Hidden,
123	cl::cat(BoltCategory));
124
125	static cl::opt<std::string>
126	FunctionNamesFile("funcs-file",
127	cl::desc("file with list of functions to optimize"),
128	cl::Hidden,
129	cl::cat(BoltCategory));
130
131	static cl::list<std::string> ForceFunctionNamesNR(
132	"funcs-no-regex", cl::CommaSeparated,
133	cl::desc("limit optimizations to functions from the list (non-regex)"),
134	cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));
135
136	static cl::opt<std::string> FunctionNamesFileNR(
137	"funcs-file-no-regex",
138	cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,
139	cl::cat(BoltCategory));
140
141	cl::opt<bool>
142	KeepTmp("keep-tmp",
143	cl::desc("preserve intermediate .o file"),
144	cl::Hidden,
145	cl::cat(BoltCategory));
146
147	static cl::opt<unsigned>
148	LiteThresholdPct("lite-threshold-pct",
149	cl::desc("threshold (in percent) for selecting functions to process in lite "
150	"mode. Higher threshold means fewer functions to process. E.g "
151	"threshold of 90 means only top 10 percent of functions with "
152	"profile will be processed."),
153	cl::init(Val: 0),
154	cl::ZeroOrMore,
155	cl::Hidden,
156	cl::cat(BoltOptCategory));
157
158	static cl::opt<unsigned> LiteThresholdCount(
159	"lite-threshold-count",
160	cl::desc("similar to '-lite-threshold-pct' but specify threshold using "
161	"absolute function call count. I.e. limit processing to functions "
162	"executed at least the specified number of times."),
163	cl::init(Val: 0), cl::Hidden, cl::cat(BoltOptCategory));
164
165	static cl::opt<unsigned>
166	MaxFunctions("max-funcs",
167	cl::desc("maximum number of functions to process"), cl::Hidden,
168	cl::cat(BoltCategory));
169
170	static cl::opt<unsigned> MaxDataRelocations(
171	"max-data-relocations",
172	cl::desc("maximum number of data relocations to process"), cl::Hidden,
173	cl::cat(BoltCategory));
174
175	cl::opt<bool> PrintAll("print-all",
176	cl::desc("print functions after each stage"), cl::Hidden,
177	cl::cat(BoltCategory));
178
179	static cl::opt<bool>
180	PrintProfile("print-profile",
181	cl::desc("print functions after attaching profile"),
182	cl::Hidden, cl::cat(BoltCategory));
183
184	cl::opt<bool> PrintCFG("print-cfg",
185	cl::desc("print functions after CFG construction"),
186	cl::Hidden, cl::cat(BoltCategory));
187
188	cl::opt<bool> PrintDisasm("print-disasm",
189	cl::desc("print function after disassembly"),
190	cl::Hidden, cl::cat(BoltCategory));
191
192	static cl::opt<bool>
193	PrintGlobals("print-globals",
194	cl::desc("print global symbols after disassembly"), cl::Hidden,
195	cl::cat(BoltCategory));
196
197	extern cl::opt<bool> PrintSections;
198
199	static cl::opt<bool> PrintLoopInfo("print-loops",
200	cl::desc("print loop related information"),
201	cl::Hidden, cl::cat(BoltCategory));
202
203	static cl::opt<cl::boolOrDefault> RelocationMode(
204	"relocs", cl::desc("use relocations in the binary (default=autodetect)"),
205	cl::cat(BoltCategory));
206
207	extern cl::opt<std::string> SaveProfile;
208
209	static cl::list<std::string>
210	SkipFunctionNames("skip-funcs",
211	cl::CommaSeparated,
212	cl::desc("list of functions to skip"),
213	cl::value_desc("func1,func2,func3,..."),
214	cl::Hidden,
215	cl::cat(BoltCategory));
216
217	static cl::opt<std::string>
218	SkipFunctionNamesFile("skip-funcs-file",
219	cl::desc("file with list of functions to skip"),
220	cl::Hidden,
221	cl::cat(BoltCategory));
222
223	static cl::opt<bool> TrapOldCode(
224	"trap-old-code",
225	cl::desc("insert traps in old function bodies (relocation mode)"),
226	cl::Hidden, cl::cat(BoltCategory));
227
228	static cl::opt<std::string> DWPPathName("dwp",
229	cl::desc("Path and name to DWP file."),
230	cl::Hidden, cl::init(Val: ""),
231	cl::cat(BoltCategory));
232
233	static cl::opt<bool>
234	UseGnuStack("use-gnu-stack",
235	cl::desc("use GNU_STACK program header for new segment (workaround for "
236	"issues with strip/objcopy)"),
237	cl::ZeroOrMore,
238	cl::cat(BoltCategory));
239
240	static cl::opt<uint64_t> CustomAllocationVMA(
241	"custom-allocation-vma",
242	cl::desc("use a custom address at which new code will be put, "
243	"bypassing BOLT's logic to detect where to put code"),
244	cl::Hidden, cl::cat(BoltCategory));
245
246	static cl::opt<bool>
247	SequentialDisassembly("sequential-disassembly",
248	cl::desc("performs disassembly sequentially"),
249	cl::init(Val: false),
250	cl::cat(BoltOptCategory));
251
252	static cl::opt<bool> WriteBoltInfoSection(
253	"bolt-info", cl::desc("write bolt info section in the output binary"),
254	cl::init(Val: true), cl::Hidden, cl::cat(BoltOutputCategory));
255
256	cl::bits<GadgetScannerKind> GadgetScannersToRun(
257	"scanners", cl::desc("which gadget scanners to run"),
258	cl::values(
259	clEnumValN(GS_PACRET, "pacret",
260	"pac-ret: return address protection (subset of \"pauth\")"),
261	clEnumValN(GS_PAUTH, "pauth", "All Pointer Authentication scanners"),
262	clEnumValN(GS_ALL, "all", "All implemented scanners")),
263	cl::ZeroOrMore, cl::CommaSeparated, cl::cat(BinaryAnalysisCategory));
264
265	} // namespace opts
266
267	// FIXME: implement a better way to mark sections for replacement.
268	constexpr const char *RewriteInstance::SectionsToOverwrite[];
269	std::vector<std::string> RewriteInstance::DebugSectionsToOverwrite = {
270	".debug_abbrev", ".debug_aranges", ".debug_line", ".debug_line_str",
271	".debug_loc", ".debug_loclists", ".debug_ranges", ".debug_rnglists",
272	".gdb_index", ".debug_addr", ".debug_abbrev", ".debug_info",
273	".debug_types", ".pseudo_probe"};
274
275	const char RewriteInstance::TimerGroupName[] = "rewrite";
276	const char RewriteInstance::TimerGroupDesc[] = "Rewrite passes";
277
278	namespace llvm {
279	namespace bolt {
280
281	extern const char *BoltRevision;
282
283	// Weird location for createMCPlusBuilder, but this is here to avoid a
284	// cyclic dependency of libCore (its natural place) and libTarget. libRewrite
285	// can depend on libTarget, but not libCore. Since libRewrite is the only
286	// user of this function, we define it here.
287	MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
288	const MCInstrAnalysis *Analysis,
289	const MCInstrInfo *Info,
290	const MCRegisterInfo *RegInfo,
291	const MCSubtargetInfo *STI) {
292	#ifdef X86_AVAILABLE
293	if (Arch == Triple::x86_64)
294	return createX86MCPlusBuilder(Analysis, Info, RegInfo, STI);
295	#endif
296
297	#ifdef AARCH64_AVAILABLE
298	if (Arch == Triple::aarch64)
299	return createAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
300	#endif
301
302	#ifdef RISCV_AVAILABLE
303	if (Arch == Triple::riscv64)
304	return createRISCVMCPlusBuilder(Analysis, Info, RegInfo, STI);
305	#endif
306
307	llvm_unreachable("architecture unsupported by MCPlusBuilder");
308	}
309
310	} // namespace bolt
311	} // namespace llvm
312
313	using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;
314
315	namespace {
316
317	bool refersToReorderedSection(ErrorOr<BinarySection &> Section) {
318	return llvm::any_of(Range&: opts::ReorderData, P: [&](const std::string &SectionName) {
319	return Section && Section->getName() == SectionName;
320	});
321	}
322
323	} // anonymous namespace
324
325	Expected<std::unique_ptr<RewriteInstance>>
326	RewriteInstance::create(ELFObjectFileBase *File, const int Argc,
327	const char *const *Argv, StringRef ToolPath,
328	raw_ostream &Stdout, raw_ostream &Stderr) {
329	Error Err = Error::success();
330	auto RI = std::make_unique<RewriteInstance>(args&: File, args: Argc, args&: Argv, args&: ToolPath,
331	args&: Stdout, args&: Stderr, args&: Err);
332	if (Err)
333	return std::move(Err);
334	return std::move(RI);
335	}
336
337	RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,
338	const char *const *Argv, StringRef ToolPath,
339	raw_ostream &Stdout, raw_ostream &Stderr,
340	Error &Err)
341	: InputFile(File), Argc(Argc), Argv(Argv), ToolPath(ToolPath),
342	SHStrTab(StringTableBuilder::ELF) {
343	ErrorAsOutParameter EAO(&Err);
344	auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(Val: InputFile);
345	if (!ELF64LEFile) {
346	Err = createStringError(EC: errc::not_supported,
347	S: "Only 64-bit LE ELF binaries are supported");
348	return;
349	}
350
351	bool IsPIC = false;
352	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
353	if (Obj.getHeader().e_type != ELF::ET_EXEC) {
354	Stdout << "BOLT-INFO: shared object or position-independent executable "
355	"detected\n";
356	IsPIC = true;
357	}
358
359	// Make sure we don't miss any output on core dumps.
360	Stdout.SetUnbuffered();
361	Stderr.SetUnbuffered();
362	LLVM_DEBUG(dbgs().SetUnbuffered());
363
364	// Read RISCV subtarget features from input file
365	std::unique_ptr<SubtargetFeatures> Features;
366	Triple TheTriple = File->makeTriple();
367	if (TheTriple.getArch() == llvm::Triple::riscv64) {
368	Expected<SubtargetFeatures> FeaturesOrErr = File->getFeatures();
369	if (auto E = FeaturesOrErr.takeError()) {
370	Err = std::move(E);
371	return;
372	} else {
373	Features.reset(p: new SubtargetFeatures(*FeaturesOrErr));
374	}
375	}
376
377	Relocation::Arch = TheTriple.getArch();
378	auto BCOrErr = BinaryContext::createBinaryContext(
379	TheTriple, SSP: std::make_shared<orc::SymbolStringPool>(), InputFileName: File->getFileName(),
380	Features: Features.get(), IsPIC,
381	DwCtx: DWARFContext::create(Obj: *File, RelocAction: DWARFContext::ProcessDebugRelocations::Ignore,
382	L: nullptr, DWPName: opts::DWPPathName,
383	RecoverableErrorHandler: WithColor::defaultErrorHandler,
384	WarningHandler: WithColor::defaultWarningHandler),
385	Logger: JournalingStreams{.Out: Stdout, .Err: Stderr});
386	if (Error E = BCOrErr.takeError()) {
387	Err = std::move(E);
388	return;
389	}
390	BC = std::move(BCOrErr.get());
391	BC->initializeTarget(TargetBuilder: std::unique_ptr<MCPlusBuilder>(
392	createMCPlusBuilder(Arch: BC->TheTriple->getArch(), Analysis: BC->MIA.get(),
393	Info: BC->MII.get(), RegInfo: BC->MRI.get(), STI: BC->STI.get())));
394
395	BAT = std::make_unique<BoltAddressTranslation>();
396
397	if (opts::UpdateDebugSections)
398	DebugInfoRewriter = std::make_unique<DWARFRewriter>(args&: *BC);
399
400	if (opts::Instrument)
401	BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
402	else if (opts::Hugify)
403	BC->setRuntimeLibrary(std::make_unique<HugifyRuntimeLibrary>());
404	}
405
406	RewriteInstance::~RewriteInstance() {}
407
408	Error RewriteInstance::setProfile(StringRef Filename) {
409	if (!sys::fs::exists(Path: Filename))
410	return errorCodeToError(EC: make_error_code(E: errc::no_such_file_or_directory));
411
412	if (ProfileReader) {
413	// Already exists
414	return make_error<StringError>(Args: Twine("multiple profiles specified: ") +
415	ProfileReader->getFilename() + " and " +
416	Filename,
417	Args: inconvertibleErrorCode());
418	}
419
420	// Spawn a profile reader based on file contents.
421	if (DataAggregator::checkPerfDataMagic(FileName: Filename))
422	ProfileReader = std::make_unique<DataAggregator>(args&: Filename);
423	else if (YAMLProfileReader::isYAML(Filename))
424	ProfileReader = std::make_unique<YAMLProfileReader>(args&: Filename);
425	else
426	ProfileReader = std::make_unique<DataReader>(args&: Filename);
427
428	return Error::success();
429	}
430
431	/// Return true if the function \p BF should be disassembled.
432	static bool shouldDisassemble(const BinaryFunction &BF) {
433	if (BF.isPseudo())
434	return false;
435
436	if (opts::processAllFunctions())
437	return true;
438
439	return !BF.isIgnored();
440	}
441
442	// Return if a section stored in the image falls into a segment address space.
443	// If not, Set \p Overlap to true if there's a partial overlap.
444	template <class ELFT>
445	static bool checkOffsets(const typename ELFT::Phdr &Phdr,
446	const typename ELFT::Shdr &Sec, bool &Overlap) {
447	// SHT_NOBITS sections don't need to have an offset inside the segment.
448	if (Sec.sh_type == ELF::SHT_NOBITS)
449	return true;
450
451	// Only non-empty sections can be at the end of a segment.
452	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
453	AddressRange SectionAddressRange((uint64_t)Sec.sh_offset,
454	Sec.sh_offset + SectionSize);
455	AddressRange SegmentAddressRange(Phdr.p_offset,
456	Phdr.p_offset + Phdr.p_filesz);
457	if (SegmentAddressRange.contains(R: SectionAddressRange))
458	return true;
459
460	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
461	return false;
462	}
463
464	// Check that an allocatable section belongs to a virtual address
465	// space of a segment.
466	template <class ELFT>
467	static bool checkVMA(const typename ELFT::Phdr &Phdr,
468	const typename ELFT::Shdr &Sec, bool &Overlap) {
469	// Only non-empty sections can be at the end of a segment.
470	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
471	AddressRange SectionAddressRange((uint64_t)Sec.sh_addr,
472	Sec.sh_addr + SectionSize);
473	AddressRange SegmentAddressRange(Phdr.p_vaddr, Phdr.p_vaddr + Phdr.p_memsz);
474
475	if (SegmentAddressRange.contains(R: SectionAddressRange))
476	return true;
477	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
478	return false;
479	}
480
481	void RewriteInstance::markGnuRelroSections() {
482	using ELFT = ELF64LE;
483	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
484	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
485	const ELFFile<ELFT> &Obj = ELF64LEFile->getELFFile();
486
487	auto handleSection = [&](const ELFT::Phdr &Phdr, SectionRef SecRef) {
488	BinarySection *BinarySection = BC->getSectionForSectionRef(Section: SecRef);
489	// If the section is non-allocatable, ignore it for GNU_RELRO purposes:
490	// it can't be made read-only after runtime relocations processing.
491	if (!BinarySection || !BinarySection->isAllocatable())
492	return;
493	const ELFShdrTy *Sec = cantFail(ValOrErr: Obj.getSection(Index: SecRef.getIndex()));
494	bool ImageOverlap{false}, VMAOverlap{false};
495	bool ImageContains = checkOffsets<ELFT>(Phdr, Sec: *Sec, Overlap&: ImageOverlap);
496	bool VMAContains = checkVMA<ELFT>(Phdr, Sec: *Sec, Overlap&: VMAOverlap);
497	if (ImageOverlap) {
498	if (opts::Verbosity >= 1)
499	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial file offset "
500	<< "overlap with section " << BinarySection->getName()
501	<< '\n';
502	return;
503	}
504	if (VMAOverlap) {
505	if (opts::Verbosity >= 1)
506	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial VMA overlap "
507	<< "with section " << BinarySection->getName() << '\n';
508	return;
509	}
510	if (!ImageContains || !VMAContains)
511	return;
512	BinarySection->setRelro();
513	if (opts::Verbosity >= 1)
514	BC->outs() << "BOLT-INFO: marking " << BinarySection->getName()
515	<< " as GNU_RELRO\n";
516	};
517
518	for (const ELFT::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers()))
519	if (Phdr.p_type == ELF::PT_GNU_RELRO)
520	for (SectionRef SecRef : InputFile->sections())
521	handleSection(Phdr, SecRef);
522	}
523
524	Error RewriteInstance::discoverStorage() {
525	NamedRegionTimer T("discoverStorage", "discover storage", TimerGroupName,
526	TimerGroupDesc, opts::TimeRewrite);
527
528	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
529	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
530
531	BC->StartFunctionAddress = Obj.getHeader().e_entry;
532
533	NextAvailableAddress = 0;
534	uint64_t NextAvailableOffset = 0;
535	Expected<ELF64LE::PhdrRange> PHsOrErr = Obj.program_headers();
536	if (Error E = PHsOrErr.takeError())
537	return E;
538
539	ELF64LE::PhdrRange PHs = PHsOrErr.get();
540	for (const ELF64LE::Phdr &Phdr : PHs) {
541	switch (Phdr.p_type) {
542	case ELF::PT_LOAD:
543	BC->FirstAllocAddress = std::min(a: BC->FirstAllocAddress,
544	b: static_cast<uint64_t>(Phdr.p_vaddr));
545	NextAvailableAddress = std::max(a: NextAvailableAddress,
546	b: Phdr.p_vaddr + Phdr.p_memsz);
547	NextAvailableOffset = std::max(a: NextAvailableOffset,
548	b: Phdr.p_offset + Phdr.p_filesz);
549
550	BC->SegmentMapInfo[Phdr.p_vaddr] =
551	SegmentInfo{.Address: Phdr.p_vaddr,
552	.Size: Phdr.p_memsz,
553	.FileOffset: Phdr.p_offset,
554	.FileSize: Phdr.p_filesz,
555	.Alignment: Phdr.p_align,
556	.IsExecutable: (Phdr.p_flags & ELF::PF_X) != 0,
557	.IsWritable: (Phdr.p_flags & ELF::PF_W) != 0};
558	if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
559	Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
560	BC->IsLinuxKernel = true;
561	break;
562	case ELF::PT_INTERP:
563	BC->HasInterpHeader = true;
564	break;
565	}
566	}
567
568	if (BC->IsLinuxKernel)
569	BC->outs() << "BOLT-INFO: Linux kernel binary detected\n";
570
571	for (const SectionRef &Section : InputFile->sections()) {
572	Expected<StringRef> SectionNameOrErr = Section.getName();
573	if (Error E = SectionNameOrErr.takeError())
574	return E;
575	StringRef SectionName = SectionNameOrErr.get();
576	if (SectionName == BC->getMainCodeSectionName()) {
577	BC->OldTextSectionAddress = Section.getAddress();
578	BC->OldTextSectionSize = Section.getSize();
579
580	Expected<StringRef> SectionContentsOrErr = Section.getContents();
581	if (Error E = SectionContentsOrErr.takeError())
582	return E;
583	StringRef SectionContents = SectionContentsOrErr.get();
584	BC->OldTextSectionOffset =
585	SectionContents.data() - InputFile->getData().data();
586	}
587
588	if (!opts::HeatmapMode &&
589	!(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&
590	(SectionName.starts_with(Prefix: getOrgSecPrefix()) ||
591	SectionName == getBOLTTextSectionName()))
592	return createStringError(
593	EC: errc::function_not_supported,
594	S: "BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize");
595	}
596
597	if (!NextAvailableAddress || !NextAvailableOffset)
598	return createStringError(EC: errc::executable_format_error,
599	S: "no PT_LOAD pheader seen");
600
601	BC->outs() << "BOLT-INFO: first alloc address is 0x"
602	<< Twine::utohexstr(Val: BC->FirstAllocAddress) << '\n';
603
604	FirstNonAllocatableOffset = NextAvailableOffset;
605
606	if (opts::CustomAllocationVMA) {
607	// If user specified a custom address where we should start writing new
608	// data, honor that.
609	NextAvailableAddress = opts::CustomAllocationVMA;
610	// Sanity check the user-supplied address and emit warnings if something
611	// seems off.
612	for (const ELF64LE::Phdr &Phdr : PHs) {
613	switch (Phdr.p_type) {
614	case ELF::PT_LOAD:
615	if (NextAvailableAddress >= Phdr.p_vaddr &&
616	NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) {
617	BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x"
618	<< Twine::utohexstr(Val: NextAvailableAddress)
619	<< " conflicts with ELF segment at 0x"
620	<< Twine::utohexstr(Val: Phdr.p_vaddr) << "\n";
621	}
622	}
623	}
624	}
625	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->PageAlign);
626	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: BC->PageAlign);
627
628	// Hugify: Additional huge page from left side due to
629	// weird ASLR mapping addresses (4KB aligned)
630	if (opts::Hugify && !BC->HasFixedLoadAddress) {
631	NextAvailableAddress += BC->PageAlign;
632	}
633
634	NewTextSegmentAddress = NextAvailableAddress;
635	NewTextSegmentOffset = NextAvailableOffset;
636
637	if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
638	// This is where the black magic happens. Creating PHDR table in a segment
639	// other than that containing ELF header is tricky. Some loaders and/or
640	// parts of loaders will apply e_phoff from ELF header assuming both are in
641	// the same segment, while others will do the proper calculation.
642	// We create the new PHDR table in such a way that both of the methods
643	// of loading and locating the table work. There's a slight file size
644	// overhead because of that.
645	//
646	// NB: bfd's strip command cannot do the above and will corrupt the
647	// binary during the process of stripping non-allocatable sections.
648	if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress)
649	NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress;
650	else
651	NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress;
652
653	assert(NextAvailableOffset ==
654	NextAvailableAddress - BC->FirstAllocAddress &&
655	"PHDR table address calculation error");
656
657	BC->outs() << "BOLT-INFO: creating new program header table at address 0x"
658	<< Twine::utohexstr(Val: NextAvailableAddress) << ", offset 0x"
659	<< Twine::utohexstr(Val: NextAvailableOffset) << '\n';
660
661	PHDRTableAddress = NextAvailableAddress;
662	PHDRTableOffset = NextAvailableOffset;
663	NewTextSegmentAddress = NextAvailableAddress;
664	NewTextSegmentOffset = NextAvailableOffset;
665
666	// Reserve space for 3 extra pheaders.
667	unsigned Phnum = Obj.getHeader().e_phnum;
668	Phnum += 3;
669
670	// Reserve two more pheaders to avoid having writeable and executable
671	// segment in instrumented binary.
672	if (opts::Instrument)
673	Phnum += 2;
674
675	NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
676	NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
677
678	// Align at cache line.
679	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 64);
680	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: 64);
681	}
682
683	BC->LayoutStartAddress = NextAvailableAddress;
684
685	// Tools such as objcopy can strip section contents but leave header
686	// entries. Check that at least .text is mapped in the file.
687	if (!getFileOffsetForAddress(Address: BC->OldTextSectionAddress))
688	return createStringError(EC: errc::executable_format_error,
689	S: "BOLT-ERROR: input binary is not a valid ELF "
690	"executable as its text section is not "
691	"mapped to a valid segment");
692	return Error::success();
693	}
694
695	Error RewriteInstance::run() {
696	assert(BC && "failed to create a binary context");
697
698	BC->outs() << "BOLT-INFO: Target architecture: "
699	<< Triple::getArchTypeName(
700	Kind: (llvm::Triple::ArchType)InputFile->getArch())
701	<< "\n";
702	BC->outs() << "BOLT-INFO: BOLT version: " << BoltRevision << "\n";
703
704	if (Error E = discoverStorage())
705	return E;
706	if (Error E = readSpecialSections())
707	return E;
708	adjustCommandLineOptions();
709	discoverFileObjects();
710
711	if (opts::Instrument && !BC->IsStaticExecutable)
712	if (Error E = discoverRtFiniAddress())
713	return E;
714
715	preprocessProfileData();
716
717	// Skip disassembling if we have a translation table and we are running an
718	// aggregation job.
719	if (opts::AggregateOnly && BAT->enabledFor(InputFile)) {
720	// YAML profile in BAT mode requires CFG for .bolt.org.text functions
721	if (!opts::SaveProfile.empty() ||
722	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) {
723	selectFunctionsToProcess();
724	disassembleFunctions();
725	processMetadataPreCFG();
726	buildFunctionsCFG();
727	}
728	processProfileData();
729	return Error::success();
730	}
731
732	selectFunctionsToProcess();
733
734	readDebugInfo();
735
736	disassembleFunctions();
737
738	processMetadataPreCFG();
739
740	buildFunctionsCFG();
741
742	processProfileData();
743
744	// Save input binary metadata if BAT section needs to be emitted
745	if (opts::EnableBAT)
746	BAT->saveMetadata(BC&: *BC);
747
748	postProcessFunctions();
749
750	processMetadataPostCFG();
751
752	if (opts::DiffOnly)
753	return Error::success();
754
755	if (opts::BinaryAnalysisMode) {
756	runBinaryAnalyses();
757	return Error::success();
758	}
759
760	preregisterSections();
761
762	runOptimizationPasses();
763
764	finalizeMetadataPreEmit();
765
766	emitAndLink();
767
768	updateMetadata();
769
770	if (opts::Instrument && !BC->IsStaticExecutable)
771	updateRtFiniReloc();
772
773	if (opts::OutputFilename == "/dev/null") {
774	BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n";
775	return Error::success();
776	} else if (BC->IsLinuxKernel) {
777	BC->errs() << "BOLT-WARNING: Linux kernel support is experimental\n";
778	}
779
780	// Rewrite allocatable contents and copy non-allocatable parts with mods.
781	rewriteFile();
782	return Error::success();
783	}
784
785	void RewriteInstance::discoverFileObjects() {
786	NamedRegionTimer T("discoverFileObjects", "discover file objects",
787	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
788
789	// For local symbols we want to keep track of associated FILE symbol name for
790	// disambiguation by combined name.
791	for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
792	Expected<StringRef> NameOrError = Symbol.getName();
793	if (NameOrError && NameOrError->starts_with(Prefix: "__asan_init")) {
794	BC->errs()
795	<< "BOLT-ERROR: input file was compiled or linked with sanitizer "
796	"support. Cannot optimize.\n";
797	exit(status: 1);
798	}
799	if (NameOrError && NameOrError->starts_with(Prefix: "__llvm_coverage_mapping")) {
800	BC->errs()
801	<< "BOLT-ERROR: input file was compiled or linked with coverage "
802	"support. Cannot optimize.\n";
803	exit(status: 1);
804	}
805
806	if (cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Undefined)
807	continue;
808
809	if (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_File)
810	FileSymbols.emplace_back(args: Symbol);
811	}
812
813	// Sort symbols in the file by value. Ignore symbols from non-allocatable
814	// sections. We memoize getAddress(), as it has rather high overhead.
815	struct SymbolInfo {
816	uint64_t Address;
817	SymbolRef Symbol;
818	};
819	std::vector<SymbolInfo> SortedSymbols;
820	auto isSymbolInMemory = [this](const SymbolRef &Sym) {
821	if (cantFail(ValOrErr: Sym.getType()) == SymbolRef::ST_File)
822	return false;
823	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Absolute)
824	return true;
825	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Undefined)
826	return false;
827	BinarySection Section(*BC, *cantFail(ValOrErr: Sym.getSection()));
828	return Section.isAllocatable();
829	};
830	auto checkSymbolInSection = [this](const SymbolInfo &S) {
831	// Sometimes, we encounter symbols with addresses outside their section. If
832	// such symbols happen to fall into another section, they can interfere with
833	// disassembly. Notably, this occurs with AArch64 marker symbols ($d and $t)
834	// that belong to .eh_frame, but end up pointing into .text.
835	// As a workaround, we ignore all symbols that lie outside their sections.
836	auto Section = cantFail(ValOrErr: S.Symbol.getSection());
837
838	// Accept all absolute symbols.
839	if (Section == InputFile->section_end())
840	return true;
841
842	uint64_t SecStart = Section->getAddress();
843	uint64_t SecEnd = SecStart + Section->getSize();
844	uint64_t SymEnd = S.Address + ELFSymbolRef(S.Symbol).getSize();
845	if (S.Address >= SecStart && SymEnd <= SecEnd)
846	return true;
847
848	auto SymType = cantFail(ValOrErr: S.Symbol.getType());
849	// Skip warnings for common benign cases.
850	if (opts::Verbosity < 1 && SymType == SymbolRef::ST_Other)
851	return false; // E.g. ELF::STT_TLS.
852
853	auto SymName = S.Symbol.getName();
854	auto SecName = cantFail(ValOrErr: S.Symbol.getSection())->getName();
855	BC->errs() << "BOLT-WARNING: ignoring symbol "
856	<< (SymName ? *SymName : "[unnamed]") << " at 0x"
857	<< Twine::utohexstr(Val: S.Address) << ", which lies outside "
858	<< (SecName ? *SecName : "[unnamed]") << "\n";
859
860	return false;
861	};
862	for (const SymbolRef &Symbol : InputFile->symbols())
863	if (isSymbolInMemory(Symbol)) {
864	SymbolInfo SymInfo{.Address: cantFail(ValOrErr: Symbol.getAddress()), .Symbol: Symbol};
865	if (checkSymbolInSection(SymInfo))
866	SortedSymbols.push_back(x: SymInfo);
867	}
868
869	auto CompareSymbols = [this](const SymbolInfo &A, const SymbolInfo &B) {
870	if (A.Address != B.Address)
871	return A.Address < B.Address;
872
873	const bool AMarker = BC->isMarker(Symbol: A.Symbol);
874	const bool BMarker = BC->isMarker(Symbol: B.Symbol);
875	if (AMarker || BMarker) {
876	return AMarker && !BMarker;
877	}
878
879	const auto AType = cantFail(ValOrErr: A.Symbol.getType());
880	const auto BType = cantFail(ValOrErr: B.Symbol.getType());
881	if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function)
882	return true;
883	if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug)
884	return true;
885
886	return false;
887	};
888	llvm::stable_sort(Range&: SortedSymbols, C: CompareSymbols);
889
890	auto LastSymbol = SortedSymbols.end();
891	if (!SortedSymbols.empty())
892	--LastSymbol;
893
894	// For aarch64, the ABI defines mapping symbols so we identify data in the
895	// code section (see IHI0056B). $d identifies data contents.
896	// Compilers usually merge multiple data objects in a single $d-$x interval,
897	// but we need every data object to be marked with $d. Because of that we
898	// create a vector of MarkerSyms with all locations of data objects.
899
900	struct MarkerSym {
901	uint64_t Address;
902	MarkerSymType Type;
903	};
904
905	std::vector<MarkerSym> SortedMarkerSymbols;
906	auto addExtraDataMarkerPerSymbol = [&]() {
907	bool IsData = false;
908	uint64_t LastAddr = 0;
909	for (const auto &SymInfo : SortedSymbols) {
910	if (LastAddr == SymInfo.Address) // don't repeat markers
911	continue;
912
913	MarkerSymType MarkerType = BC->getMarkerType(Symbol: SymInfo.Symbol);
914	if (MarkerType != MarkerSymType::NONE) {
915	SortedMarkerSymbols.push_back(x: MarkerSym{.Address: SymInfo.Address, .Type: MarkerType});
916	LastAddr = SymInfo.Address;
917	IsData = MarkerType == MarkerSymType::DATA;
918	continue;
919	}
920
921	if (IsData) {
922	SortedMarkerSymbols.push_back(x: {.Address: SymInfo.Address, .Type: MarkerSymType::DATA});
923	LastAddr = SymInfo.Address;
924	}
925	}
926	};
927
928	if (BC->isAArch64() || BC->isRISCV()) {
929	addExtraDataMarkerPerSymbol();
930	LastSymbol = std::stable_partition(
931	first: SortedSymbols.begin(), last: SortedSymbols.end(),
932	pred: [this](const SymbolInfo &S) { return !BC->isMarker(Symbol: S.Symbol); });
933	if (!SortedSymbols.empty())
934	--LastSymbol;
935	}
936
937	BinaryFunction *PreviousFunction = nullptr;
938	unsigned AnonymousId = 0;
939
940	const auto SortedSymbolsEnd =
941	LastSymbol == SortedSymbols.end() ? LastSymbol : std::next(x: LastSymbol);
942	for (auto Iter = SortedSymbols.begin(); Iter != SortedSymbolsEnd; ++Iter) {
943	const SymbolRef &Symbol = Iter->Symbol;
944	const uint64_t SymbolAddress = Iter->Address;
945	const auto SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
946	const SymbolRef::Type SymbolType = cantFail(ValOrErr: Symbol.getType());
947
948	if (SymbolType == SymbolRef::ST_File)
949	continue;
950
951	StringRef SymName = cantFail(ValOrErr: Symbol.getName(), Msg: "cannot get symbol name");
952	if (SymbolAddress == 0) {
953	if (opts::Verbosity >= 1 && SymbolType == SymbolRef::ST_Function)
954	BC->errs() << "BOLT-WARNING: function with 0 address seen\n";
955	continue;
956	}
957
958	// Ignore input hot markers unless in heatmap mode
959	if ((SymName == "__hot_start" || SymName == "__hot_end") &&
960	!opts::HeatmapMode)
961	continue;
962
963	FileSymRefs.emplace(args: SymbolAddress, args: Symbol);
964
965	// Skip section symbols that will be registered by disassemblePLT().
966	if (SymbolType == SymbolRef::ST_Debug) {
967	ErrorOr<BinarySection &> BSection =
968	BC->getSectionForAddress(Address: SymbolAddress);
969	if (BSection && getPLTSectionInfo(SectionName: BSection->getName()))
970	continue;
971	}
972
973	/// It is possible we are seeing a globalized local. LLVM might treat it as
974	/// a local if it has a "private global" prefix, e.g. ".L". Thus we have to
975	/// change the prefix to enforce global scope of the symbol.
976	std::string Name =
977	SymName.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix())
978	? "PG" + std::string(SymName)
979	: std::string(SymName);
980
981	// Disambiguate all local symbols before adding to symbol table.
982	// Since we don't know if we will see a global with the same name,
983	// always modify the local name.
984	//
985	// NOTE: the naming convention for local symbols should match
986	// the one we use for profile data.
987	std::string UniqueName;
988	std::string AlternativeName;
989	if (Name.empty()) {
990	UniqueName = "ANONYMOUS." + std::to_string(val: AnonymousId++);
991	} else if (SymbolFlags & SymbolRef::SF_Global) {
992	if (const BinaryData *BD = BC->getBinaryDataByName(Name)) {
993	if (BD->getSize() == ELFSymbolRef(Symbol).getSize() &&
994	BD->getAddress() == SymbolAddress) {
995	if (opts::Verbosity > 1)
996	BC->errs() << "BOLT-WARNING: ignoring duplicate global symbol "
997	<< Name << "\n";
998	// Ignore duplicate entry - possibly a bug in the linker
999	continue;
1000	}
1001	BC->errs() << "BOLT-ERROR: bad input binary, global symbol \"" << Name
1002	<< "\" is not unique\n";
1003	exit(status: 1);
1004	}
1005	UniqueName = Name;
1006	} else {
1007	// If we have a local file name, we should create 2 variants for the
1008	// function name. The reason is that perf profile might have been
1009	// collected on a binary that did not have the local file name (e.g. as
1010	// a side effect of stripping debug info from the binary):
1011	//
1012	// primary: <function>/<id>
1013	// alternative: <function>/<file>/<id2>
1014	//
1015	// The <id> field is used for disambiguation of local symbols since there
1016	// could be identical function names coming from identical file names
1017	// (e.g. from different directories).
1018	auto SFI = llvm::upper_bound(Range&: FileSymbols, Value: ELFSymbolRef(Symbol));
1019	if (SymbolType == SymbolRef::ST_Function && SFI != FileSymbols.begin()) {
1020	StringRef FileSymbolName = cantFail(ValOrErr: SFI[-1].getName());
1021	if (!FileSymbolName.empty())
1022	AlternativeName = NR.uniquify(Name: Name + "/" + FileSymbolName.str());
1023	}
1024
1025	UniqueName = NR.uniquify(Name);
1026	}
1027
1028	uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1029	uint64_t SymbolAlignment = Symbol.getAlignment();
1030
1031	auto registerName = [&](uint64_t FinalSize) {
1032	// Register names even if it's not a function, e.g. for an entry point.
1033	BC->registerNameAtAddress(Name: UniqueName, Address: SymbolAddress, Size: FinalSize,
1034	Alignment: SymbolAlignment, Flags: SymbolFlags);
1035	if (!AlternativeName.empty())
1036	BC->registerNameAtAddress(Name: AlternativeName, Address: SymbolAddress, Size: FinalSize,
1037	Alignment: SymbolAlignment, Flags: SymbolFlags);
1038	};
1039
1040	section_iterator Section =
1041	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
1042	if (Section == InputFile->section_end()) {
1043	// Could be an absolute symbol. Used on RISC-V for __global_pointer$ so we
1044	// need to record it to handle relocations against it. For other instances
1045	// of absolute symbols, we record for pretty printing.
1046	LLVM_DEBUG(if (opts::Verbosity > 1) {
1047	dbgs() << "BOLT-INFO: absolute sym " << UniqueName << "\n";
1048	});
1049	registerName(SymbolSize);
1050	continue;
1051	}
1052
1053	if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) {
1054	registerName(SymbolSize);
1055	continue;
1056	}
1057
1058	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName
1059	<< " for function\n");
1060
1061	if (SymbolAddress == Section->getAddress() + Section->getSize()) {
1062	assert(SymbolSize == 0 &&
1063	"unexpect non-zero sized symbol at end of section");
1064	LLVM_DEBUG(
1065	dbgs()
1066	<< "BOLT-DEBUG: rejecting as symbol points to end of its section\n");
1067	registerName(SymbolSize);
1068	continue;
1069	}
1070
1071	if (!Section->isText() || Section->isVirtual()) {
1072	assert(SymbolType != SymbolRef::ST_Function &&
1073	"unexpected function inside non-code section");
1074	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code or "
1075	"is in nobits section\n");
1076	registerName(SymbolSize);
1077	continue;
1078	}
1079
1080	// Assembly functions could be ST_NONE with 0 size. Check that the
1081	// corresponding section is a code section and they are not inside any
1082	// other known function to consider them.
1083	//
1084	// Sometimes assembly functions are not marked as functions and neither are
1085	// their local labels. The only way to tell them apart is to look at
1086	// symbol scope - global vs local.
1087	if (PreviousFunction && SymbolType != SymbolRef::ST_Function) {
1088	if (PreviousFunction->containsAddress(PC: SymbolAddress)) {
1089	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1090	LLVM_DEBUG(dbgs()
1091	<< "BOLT-DEBUG: symbol is a function local symbol\n");
1092	} else if (SymbolAddress == PreviousFunction->getAddress() &&
1093	!SymbolSize) {
1094	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring symbol as a marker\n");
1095	} else if (opts::Verbosity > 1) {
1096	BC->errs() << "BOLT-WARNING: symbol " << UniqueName
1097	<< " seen in the middle of function " << *PreviousFunction
1098	<< ". Could be a new entry.\n";
1099	}
1100	registerName(SymbolSize);
1101	continue;
1102	} else if (PreviousFunction->getSize() == 0 &&
1103	PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1104	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: symbol is a function local symbol\n");
1105	registerName(SymbolSize);
1106	continue;
1107	}
1108	}
1109
1110	if (PreviousFunction && PreviousFunction->containsAddress(PC: SymbolAddress) &&
1111	PreviousFunction->getAddress() != SymbolAddress) {
1112	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1113	if (opts::Verbosity >= 1)
1114	BC->outs()
1115	<< "BOLT-INFO: skipping possibly another entry for function "
1116	<< *PreviousFunction << " : " << UniqueName << '\n';
1117	registerName(SymbolSize);
1118	} else {
1119	BC->outs() << "BOLT-INFO: using " << UniqueName
1120	<< " as another entry to "
1121	<< "function " << *PreviousFunction << '\n';
1122
1123	registerName(0);
1124
1125	PreviousFunction->addEntryPointAtOffset(Offset: SymbolAddress -
1126	PreviousFunction->getAddress());
1127
1128	// Remove the symbol from FileSymRefs so that we can skip it from
1129	// in the future.
1130	auto SI = llvm::find_if(
1131	Range: llvm::make_range(p: FileSymRefs.equal_range(x: SymbolAddress)),
1132	P: [&](auto SymIt) { return SymIt.second == Symbol; });
1133	assert(SI != FileSymRefs.end() && "symbol expected to be present");
1134	assert(SI->second == Symbol && "wrong symbol found");
1135	FileSymRefs.erase(position: SI);
1136	}
1137	continue;
1138	}
1139
1140	// Checkout for conflicts with function data from FDEs.
1141	bool IsSimple = true;
1142	auto FDEI = CFIRdWrt->getFDEs().lower_bound(x: SymbolAddress);
1143	if (FDEI != CFIRdWrt->getFDEs().end()) {
1144	const dwarf::FDE &FDE = *FDEI->second;
1145	if (FDEI->first != SymbolAddress) {
1146	// There's no matching starting address in FDE. Make sure the previous
1147	// FDE does not contain this address.
1148	if (FDEI != CFIRdWrt->getFDEs().begin()) {
1149	--FDEI;
1150	const dwarf::FDE &PrevFDE = *FDEI->second;
1151	uint64_t PrevStart = PrevFDE.getInitialLocation();
1152	uint64_t PrevLength = PrevFDE.getAddressRange();
1153	if (SymbolAddress > PrevStart &&
1154	SymbolAddress < PrevStart + PrevLength) {
1155	BC->errs() << "BOLT-ERROR: function " << UniqueName
1156	<< " is in conflict with FDE ["
1157	<< Twine::utohexstr(Val: PrevStart) << ", "
1158	<< Twine::utohexstr(Val: PrevStart + PrevLength)
1159	<< "). Skipping.\n";
1160	IsSimple = false;
1161	}
1162	}
1163	} else if (FDE.getAddressRange() != SymbolSize) {
1164	if (SymbolSize) {
1165	// Function addresses match but sizes differ.
1166	BC->errs() << "BOLT-WARNING: sizes differ for function " << UniqueName
1167	<< ". FDE : " << FDE.getAddressRange()
1168	<< "; symbol table : " << SymbolSize
1169	<< ". Using max size.\n";
1170	}
1171	SymbolSize = std::max(a: SymbolSize, b: FDE.getAddressRange());
1172	if (BC->getBinaryDataAtAddress(Address: SymbolAddress)) {
1173	BC->setBinaryDataSize(Address: SymbolAddress, Size: SymbolSize);
1174	} else {
1175	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: No BD @ 0x"
1176	<< Twine::utohexstr(SymbolAddress) << "\n");
1177	}
1178	}
1179	}
1180
1181	BinaryFunction *BF = nullptr;
1182	// Since function may not have yet obtained its real size, do a search
1183	// using the list of registered functions instead of calling
1184	// getBinaryFunctionAtAddress().
1185	auto BFI = BC->getBinaryFunctions().find(x: SymbolAddress);
1186	if (BFI != BC->getBinaryFunctions().end()) {
1187	BF = &BFI->second;
1188	// Duplicate the function name. Make sure everything matches before we add
1189	// an alternative name.
1190	if (SymbolSize != BF->getSize()) {
1191	if (opts::Verbosity >= 1) {
1192	if (SymbolSize && BF->getSize())
1193	BC->errs() << "BOLT-WARNING: size mismatch for duplicate entries "
1194	<< *BF << " and " << UniqueName << '\n';
1195	BC->outs() << "BOLT-INFO: adjusting size of function " << *BF
1196	<< " old " << BF->getSize() << " new " << SymbolSize
1197	<< "\n";
1198	}
1199	BF->setSize(std::max(a: SymbolSize, b: BF->getSize()));
1200	BC->setBinaryDataSize(Address: SymbolAddress, Size: BF->getSize());
1201	}
1202	BF->addAlternativeName(NewName: UniqueName);
1203	} else {
1204	ErrorOr<BinarySection &> Section =
1205	BC->getSectionForAddress(Address: SymbolAddress);
1206	// Skip symbols from invalid sections
1207	if (!Section) {
1208	BC->errs() << "BOLT-WARNING: " << UniqueName << " (0x"
1209	<< Twine::utohexstr(Val: SymbolAddress)
1210	<< ") does not have any section\n";
1211	continue;
1212	}
1213
1214	// Skip symbols from zero-sized sections.
1215	if (!Section->getSize())
1216	continue;
1217
1218	BF = BC->createBinaryFunction(Name: UniqueName, Section&: *Section, Address: SymbolAddress,
1219	Size: SymbolSize);
1220	if (!IsSimple)
1221	BF->setSimple(false);
1222	}
1223
1224	// Check if it's a cold function fragment.
1225	if (FunctionFragmentTemplate.match(String: SymName)) {
1226	static bool PrintedWarning = false;
1227	if (!PrintedWarning) {
1228	PrintedWarning = true;
1229	BC->errs() << "BOLT-WARNING: split function detected on input : "
1230	<< SymName;
1231	if (BC->HasRelocations)
1232	BC->errs() << ". The support is limited in relocation mode\n";
1233	else
1234	BC->errs() << '\n';
1235	}
1236	BC->HasSplitFunctions = true;
1237	BF->IsFragment = true;
1238	}
1239
1240	if (!AlternativeName.empty())
1241	BF->addAlternativeName(NewName: AlternativeName);
1242
1243	registerName(SymbolSize);
1244	PreviousFunction = BF;
1245	}
1246
1247	// Read dynamic relocation first as their presence affects the way we process
1248	// static relocations. E.g. we will ignore a static relocation at an address
1249	// that is a subject to dynamic relocation processing.
1250	processDynamicRelocations();
1251
1252	// Process PLT section.
1253	disassemblePLT();
1254
1255	// See if we missed any functions marked by FDE.
1256	for (const auto &FDEI : CFIRdWrt->getFDEs()) {
1257	const uint64_t Address = FDEI.first;
1258	const dwarf::FDE *FDE = FDEI.second;
1259	const BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address);
1260	if (BF)
1261	continue;
1262
1263	BF = BC->getBinaryFunctionContainingAddress(Address);
1264	if (BF) {
1265	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1266	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1267	<< ") conflicts with function " << *BF << '\n';
1268	continue;
1269	}
1270
1271	if (opts::Verbosity >= 1)
1272	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1273	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1274	<< ") has no corresponding symbol table entry\n";
1275
1276	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
1277	assert(Section && "cannot get section for address from FDE");
1278	std::string FunctionName =
1279	"__BOLT_FDE_FUNCat" + Twine::utohexstr(Val: Address).str();
1280	BC->createBinaryFunction(Name: FunctionName, Section&: *Section, Address,
1281	Size: FDE->getAddressRange());
1282	}
1283
1284	BC->setHasSymbolsWithFileName(FileSymbols.size());
1285
1286	// Now that all the functions were created - adjust their boundaries.
1287	adjustFunctionBoundaries();
1288
1289	// Annotate functions with code/data markers in AArch64
1290	for (auto ISym = SortedMarkerSymbols.begin();
1291	ISym != SortedMarkerSymbols.end(); ++ISym) {
1292
1293	auto *BF =
1294	BC->getBinaryFunctionContainingAddress(Address: ISym->Address, CheckPastEnd: true, UseMaxSize: true);
1295
1296	if (!BF) {
1297	// Stray marker
1298	continue;
1299	}
1300	const auto EntryOffset = ISym->Address - BF->getAddress();
1301	if (ISym->Type == MarkerSymType::CODE) {
1302	BF->markCodeAtOffset(Offset: EntryOffset);
1303	continue;
1304	}
1305	if (ISym->Type == MarkerSymType::DATA) {
1306	BF->markDataAtOffset(Offset: EntryOffset);
1307	BC->AddressToConstantIslandMap[ISym->Address] = BF;
1308	continue;
1309	}
1310	llvm_unreachable("Unknown marker");
1311	}
1312
1313	if (BC->isAArch64()) {
1314	// Check for dynamic relocations that might be contained in
1315	// constant islands.
1316	for (const BinarySection &Section : BC->allocatableSections()) {
1317	const uint64_t SectionAddress = Section.getAddress();
1318	for (const Relocation &Rel : Section.dynamicRelocations()) {
1319	const uint64_t RelAddress = SectionAddress + Rel.Offset;
1320	BinaryFunction *BF =
1321	BC->getBinaryFunctionContainingAddress(Address: RelAddress,
1322	/*CheckPastEnd*/ false,
1323	/*UseMaxSize*/ true);
1324	if (BF) {
1325	assert(Rel.isRelative() && "Expected relative relocation for island");
1326	BC->logBOLTErrorsAndQuitOnFatal(
1327	E: BF->markIslandDynamicRelocationAtAddress(Address: RelAddress));
1328	}
1329	}
1330	}
1331
1332	// The linker may omit data markers for absolute long veneers. Introduce
1333	// those markers artificially to assist the disassembler.
1334	for (BinaryFunction &BF :
1335	llvm::make_second_range(c&: BC->getBinaryFunctions())) {
1336	if (BF.getOneName().starts_with(Prefix: "__AArch64AbsLongThunk_") &&
1337	BF.getSize() == 16 && !BF.getSizeOfDataInCodeAt(Offset: 8)) {
1338	BC->errs() << "BOLT-WARNING: missing data marker detected in veneer "
1339	<< BF << '\n';
1340	BF.markDataAtOffset(Offset: 8);
1341	BC->AddressToConstantIslandMap[BF.getAddress() + 8] = &BF;
1342	}
1343	}
1344	}
1345
1346	if (!BC->IsLinuxKernel) {
1347	// Read all relocations now that we have binary functions mapped.
1348	processRelocations();
1349	}
1350
1351	registerFragments();
1352	FileSymbols.clear();
1353	FileSymRefs.clear();
1354
1355	discoverBOLTReserved();
1356	}
1357
1358	void RewriteInstance::discoverBOLTReserved() {
1359	BinaryData *StartBD = BC->getBinaryDataByName(Name: getBOLTReservedStart());
1360	BinaryData *EndBD = BC->getBinaryDataByName(Name: getBOLTReservedEnd());
1361	if (!StartBD != !EndBD) {
1362	BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: "
1363	<< getBOLTReservedStart() << ", " << getBOLTReservedEnd()
1364	<< '\n';
1365	exit(status: 1);
1366	}
1367
1368	if (!StartBD)
1369	return;
1370
1371	if (StartBD->getAddress() >= EndBD->getAddress()) {
1372	BC->errs() << "BOLT-ERROR: invalid reserved space boundaries\n";
1373	exit(status: 1);
1374	}
1375	BC->BOLTReserved = AddressRange(StartBD->getAddress(), EndBD->getAddress());
1376	BC->outs() << "BOLT-INFO: using reserved space for allocating new sections\n";
1377
1378	PHDRTableOffset = 0;
1379	PHDRTableAddress = 0;
1380	NewTextSegmentAddress = 0;
1381	NewTextSegmentOffset = 0;
1382	NextAvailableAddress = BC->BOLTReserved.start();
1383	}
1384
1385	Error RewriteInstance::discoverRtFiniAddress() {
1386	// Use DT_FINI if it's available.
1387	if (BC->FiniAddress) {
1388	BC->FiniFunctionAddress = BC->FiniAddress;
1389	return Error::success();
1390	}
1391
1392	if (!BC->FiniArrayAddress || !BC->FiniArraySize) {
1393	return createStringError(
1394	EC: std::errc::not_supported,
1395	Fmt: "Instrumentation needs either DT_FINI or DT_FINI_ARRAY");
1396	}
1397
1398	if (*BC->FiniArraySize < BC->AsmInfo->getCodePointerSize()) {
1399	return createStringError(EC: std::errc::not_supported,
1400	Fmt: "Need at least 1 DT_FINI_ARRAY slot");
1401	}
1402
1403	ErrorOr<BinarySection &> FiniArraySection =
1404	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1405	if (auto EC = FiniArraySection.getError())
1406	return errorCodeToError(EC);
1407
1408	if (const Relocation *Reloc = FiniArraySection->getDynamicRelocationAt(Offset: 0)) {
1409	BC->FiniFunctionAddress = Reloc->Addend;
1410	return Error::success();
1411	}
1412
1413	if (const Relocation *Reloc = FiniArraySection->getRelocationAt(Offset: 0)) {
1414	BC->FiniFunctionAddress = Reloc->Value;
1415	return Error::success();
1416	}
1417
1418	return createStringError(EC: std::errc::not_supported,
1419	Fmt: "No relocation for first DT_FINI_ARRAY slot");
1420	}
1421
1422	void RewriteInstance::updateRtFiniReloc() {
1423	// Updating DT_FINI is handled by patchELFDynamic.
1424	if (BC->FiniAddress)
1425	return;
1426
1427	const RuntimeLibrary *RT = BC->getRuntimeLibrary();
1428	if (!RT || !RT->getRuntimeFiniAddress())
1429	return;
1430
1431	assert(BC->FiniArrayAddress && BC->FiniArraySize &&
1432	"inconsistent .fini_array state");
1433
1434	ErrorOr<BinarySection &> FiniArraySection =
1435	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1436	assert(FiniArraySection && ".fini_array removed");
1437
1438	if (std::optional<Relocation> Reloc =
1439	FiniArraySection->takeDynamicRelocationAt(Offset: 0)) {
1440	assert(Reloc->Addend == BC->FiniFunctionAddress &&
1441	"inconsistent .fini_array dynamic relocation");
1442	Reloc->Addend = RT->getRuntimeFiniAddress();
1443	FiniArraySection->addDynamicRelocation(Reloc: *Reloc);
1444	}
1445
1446	// Update the static relocation by adding a pending relocation which will get
1447	// patched when flushPendingRelocations is called in rewriteFile. Note that
1448	// flushPendingRelocations will calculate the value to patch as
1449	// "Symbol + Addend". Since we don't have a symbol, just set the addend to the
1450	// desired value.
1451	FiniArraySection->addPendingRelocation(Rel: Relocation{
1452	/*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),
1453	/*Addend*/ RT->getRuntimeFiniAddress(), /*Value*/ 0});
1454	}
1455
1456	void RewriteInstance::registerFragments() {
1457	if (!BC->HasSplitFunctions ||
1458	opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
1459	return;
1460
1461	// Process fragments with ambiguous parents separately as they are typically a
1462	// vanishing minority of cases and require expensive symbol table lookups.
1463	std::vector<std::pair<StringRef, BinaryFunction *>> AmbiguousFragments;
1464	for (auto &BFI : BC->getBinaryFunctions()) {
1465	BinaryFunction &Function = BFI.second;
1466	if (!Function.isFragment())
1467	continue;
1468	for (StringRef Name : Function.getNames()) {
1469	StringRef BaseName = NR.restore(Name);
1470	const bool IsGlobal = BaseName == Name;
1471	SmallVector<StringRef> Matches;
1472	if (!FunctionFragmentTemplate.match(String: BaseName, Matches: &Matches))
1473	continue;
1474	StringRef ParentName = Matches[1];
1475	const BinaryData *BD = BC->getBinaryDataByName(Name: ParentName);
1476	const uint64_t NumPossibleLocalParents =
1477	NR.getUniquifiedNameCount(Name: ParentName);
1478	// The most common case: single local parent fragment.
1479	if (!BD && NumPossibleLocalParents == 1) {
1480	BD = BC->getBinaryDataByName(Name: NR.getUniqueName(Name: ParentName, ID: 1));
1481	} else if (BD && (!NumPossibleLocalParents || IsGlobal)) {
1482	// Global parent and either no local candidates (second most common), or
1483	// the fragment is global as well (uncommon).
1484	} else {
1485	// Any other case: need to disambiguate using FILE symbols.
1486	AmbiguousFragments.emplace_back(args&: ParentName, args: &Function);
1487	continue;
1488	}
1489	if (BD) {
1490	BinaryFunction *BF = BC->getFunctionForSymbol(Symbol: BD->getSymbol());
1491	if (BF) {
1492	BC->registerFragment(TargetFunction&: Function, Function&: *BF);
1493	continue;
1494	}
1495	}
1496	BC->errs() << "BOLT-ERROR: parent function not found for " << Function
1497	<< '\n';
1498	exit(status: 1);
1499	}
1500	}
1501
1502	if (AmbiguousFragments.empty())
1503	return;
1504
1505	if (!BC->hasSymbolsWithFileName()) {
1506	BC->errs() << "BOLT-ERROR: input file has split functions but does not "
1507	"have FILE symbols. If the binary was stripped, preserve "
1508	"FILE symbols with --keep-file-symbols strip option\n";
1509	exit(status: 1);
1510	}
1511
1512	// The first global symbol is identified by the symbol table sh_info value.
1513	// Used as local symbol search stopping point.
1514	auto *ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
1515	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
1516	auto *SymTab = llvm::find_if(Range: cantFail(ValOrErr: Obj.sections()), P: [](const auto &Sec) {
1517	return Sec.sh_type == ELF::SHT_SYMTAB;
1518	});
1519	assert(SymTab);
1520	// Symtab sh_info contains the value one greater than the symbol table index
1521	// of the last local symbol.
1522	ELFSymbolRef LocalSymEnd = ELF64LEFile->toSymbolRef(SymTable: SymTab, SymbolNum: SymTab->sh_info);
1523
1524	for (auto &Fragment : AmbiguousFragments) {
1525	const StringRef &ParentName = Fragment.first;
1526	BinaryFunction *BF = Fragment.second;
1527	const uint64_t Address = BF->getAddress();
1528
1529	// Get fragment's own symbol
1530	const auto SymIt = llvm::find_if(
1531	Range: llvm::make_range(p: FileSymRefs.equal_range(x: Address)), P: [&](auto SI) {
1532	StringRef Name = cantFail(SI.second.getName());
1533	return Name.contains(Other: ParentName);
1534	});
1535	if (SymIt == FileSymRefs.end()) {
1536	BC->errs()
1537	<< "BOLT-ERROR: symbol lookup failed for function at address 0x"
1538	<< Twine::utohexstr(Val: Address) << '\n';
1539	exit(status: 1);
1540	}
1541
1542	// Find containing FILE symbol
1543	ELFSymbolRef Symbol = SymIt->second;
1544	auto FSI = llvm::upper_bound(Range&: FileSymbols, Value&: Symbol);
1545	if (FSI == FileSymbols.begin()) {
1546	BC->errs() << "BOLT-ERROR: owning FILE symbol not found for symbol "
1547	<< cantFail(ValOrErr: Symbol.getName()) << '\n';
1548	exit(status: 1);
1549	}
1550
1551	ELFSymbolRef StopSymbol = LocalSymEnd;
1552	if (FSI != FileSymbols.end())
1553	StopSymbol = *FSI;
1554
1555	uint64_t ParentAddress{0};
1556
1557	// Check if containing FILE symbol is BOLT emitted synthetic symbol marking
1558	// local fragments of global parents.
1559	if (cantFail(ValOrErr: FSI[-1].getName()) == getBOLTFileSymbolName())
1560	goto registerParent;
1561
1562	// BOLT split fragment symbols are emitted just before the main function
1563	// symbol.
1564	for (ELFSymbolRef NextSymbol = Symbol; NextSymbol < StopSymbol;
1565	NextSymbol.moveNext()) {
1566	StringRef Name = cantFail(ValOrErr: NextSymbol.getName());
1567	if (Name == ParentName) {
1568	ParentAddress = cantFail(ValOrErr: NextSymbol.getValue());
1569	goto registerParent;
1570	}
1571	if (Name.starts_with(Prefix: ParentName))
1572	// With multi-way splitting, there are multiple fragments with different
1573	// suffixes. Parent follows the last fragment.
1574	continue;
1575	break;
1576	}
1577
1578	// Iterate over local file symbols and check symbol names to match parent.
1579	for (ELFSymbolRef Symbol(FSI[-1]); Symbol < StopSymbol; Symbol.moveNext()) {
1580	if (cantFail(ValOrErr: Symbol.getName()) == ParentName) {
1581	ParentAddress = cantFail(ValOrErr: Symbol.getAddress());
1582	break;
1583	}
1584	}
1585
1586	registerParent:
1587	// No local parent is found, use global parent function.
1588	if (!ParentAddress)
1589	if (BinaryData *ParentBD = BC->getBinaryDataByName(Name: ParentName))
1590	ParentAddress = ParentBD->getAddress();
1591
1592	if (BinaryFunction *ParentBF =
1593	BC->getBinaryFunctionAtAddress(Address: ParentAddress)) {
1594	BC->registerFragment(TargetFunction&: *BF, Function&: *ParentBF);
1595	continue;
1596	}
1597	BC->errs() << "BOLT-ERROR: parent function not found for " << *BF << '\n';
1598	exit(status: 1);
1599	}
1600	}
1601
1602	void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,
1603	uint64_t EntryAddress,
1604	uint64_t EntrySize) {
1605	if (!TargetAddress)
1606	return;
1607
1608	auto setPLTSymbol = [&](BinaryFunction *BF, StringRef Name) {
1609	const unsigned PtrSize = BC->AsmInfo->getCodePointerSize();
1610	MCSymbol *TargetSymbol = BC->registerNameAtAddress(
1611	Name: Name.str() + "@GOT", Address: TargetAddress, Size: PtrSize, Alignment: PtrSize);
1612	BF->setPLTSymbol(TargetSymbol);
1613	};
1614
1615	BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address: EntryAddress);
1616	if (BF && BC->isAArch64()) {
1617	// Handle IFUNC trampoline with symbol
1618	setPLTSymbol(BF, BF->getOneName());
1619	return;
1620	}
1621
1622	const Relocation *Rel = BC->getDynamicRelocationAt(Address: TargetAddress);
1623	if (!Rel)
1624	return;
1625
1626	MCSymbol *Symbol = Rel->Symbol;
1627	if (!Symbol) {
1628	if (BC->isRISCV() || !Rel->Addend || !Rel->isIRelative())
1629	return;
1630
1631	// IFUNC trampoline without symbol
1632	BinaryFunction *TargetBF = BC->getBinaryFunctionAtAddress(Address: Rel->Addend);
1633	if (!TargetBF) {
1634	BC->errs()
1635	<< "BOLT-WARNING: Expected BF to be presented as IFUNC resolver at "
1636	<< Twine::utohexstr(Val: Rel->Addend) << ", skipping\n";
1637	return;
1638	}
1639
1640	Symbol = TargetBF->getSymbol();
1641	}
1642
1643	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: EntryAddress);
1644	assert(Section && "cannot get section for address");
1645	if (!BF)
1646	BF = BC->createBinaryFunction(Name: Symbol->getName().str() + "@PLT", Section&: *Section,
1647	Address: EntryAddress, Size: 0, SymbolSize: EntrySize,
1648	Alignment: Section->getAlignment());
1649	else
1650	BF->addAlternativeName(NewName: Symbol->getName().str() + "@PLT");
1651	setPLTSymbol(BF, Symbol->getName());
1652	}
1653
1654	void RewriteInstance::disassemblePLTInstruction(const BinarySection &Section,
1655	uint64_t InstrOffset,
1656	MCInst &Instruction,
1657	uint64_t &InstrSize) {
1658	const uint64_t SectionAddress = Section.getAddress();
1659	const uint64_t SectionSize = Section.getSize();
1660	StringRef PLTContents = Section.getContents();
1661	ArrayRef<uint8_t> PLTData(
1662	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1663
1664	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1665	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1666	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1667	CStream&: nulls())) {
1668	BC->errs()
1669	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1670	<< Section.getName() << formatv(Fmt: " at offset {0:x}\n", Vals&: InstrOffset);
1671	exit(status: 1);
1672	}
1673	}
1674
1675	void RewriteInstance::disassemblePLTSectionAArch64(BinarySection &Section) {
1676	const uint64_t SectionAddress = Section.getAddress();
1677	const uint64_t SectionSize = Section.getSize();
1678
1679	uint64_t InstrOffset = 0;
1680	// Locate new plt entry
1681	while (InstrOffset < SectionSize) {
1682	InstructionListType Instructions;
1683	MCInst Instruction;
1684	uint64_t EntryOffset = InstrOffset;
1685	uint64_t EntrySize = 0;
1686	uint64_t InstrSize;
1687	// Loop through entry instructions
1688	while (InstrOffset < SectionSize) {
1689	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1690	EntrySize += InstrSize;
1691	if (!BC->MIB->isIndirectBranch(Inst: Instruction)) {
1692	Instructions.emplace_back(args&: Instruction);
1693	InstrOffset += InstrSize;
1694	continue;
1695	}
1696
1697	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1698	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1699	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1700
1701	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1702	break;
1703	}
1704
1705	// Branch instruction
1706	InstrOffset += InstrSize;
1707
1708	// Skip nops if any
1709	while (InstrOffset < SectionSize) {
1710	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1711	if (!BC->MIB->isNoop(Inst: Instruction))
1712	break;
1713
1714	InstrOffset += InstrSize;
1715	}
1716	}
1717	}
1718
1719	void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) {
1720	const uint64_t SectionAddress = Section.getAddress();
1721	const uint64_t SectionSize = Section.getSize();
1722	StringRef PLTContents = Section.getContents();
1723	ArrayRef<uint8_t> PLTData(
1724	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1725
1726	auto disassembleInstruction = [&](uint64_t InstrOffset, MCInst &Instruction,
1727	uint64_t &InstrSize) {
1728	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1729	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1730	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1731	CStream&: nulls())) {
1732	BC->errs()
1733	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1734	<< Section.getName() << " at offset 0x"
1735	<< Twine::utohexstr(Val: InstrOffset) << '\n';
1736	exit(status: 1);
1737	}
1738	};
1739
1740	// Skip the first special entry since no relocation points to it.
1741	uint64_t InstrOffset = 32;
1742
1743	while (InstrOffset < SectionSize) {
1744	InstructionListType Instructions;
1745	MCInst Instruction;
1746	const uint64_t EntryOffset = InstrOffset;
1747	const uint64_t EntrySize = 16;
1748	uint64_t InstrSize;
1749
1750	while (InstrOffset < EntryOffset + EntrySize) {
1751	disassembleInstruction(InstrOffset, Instruction, InstrSize);
1752	Instructions.emplace_back(args&: Instruction);
1753	InstrOffset += InstrSize;
1754	}
1755
1756	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1757	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1758	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1759
1760	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1761	}
1762	}
1763
1764	void RewriteInstance::disassemblePLTSectionX86(BinarySection &Section,
1765	uint64_t EntrySize) {
1766	const uint64_t SectionAddress = Section.getAddress();
1767	const uint64_t SectionSize = Section.getSize();
1768
1769	for (uint64_t EntryOffset = 0; EntryOffset + EntrySize <= SectionSize;
1770	EntryOffset += EntrySize) {
1771	MCInst Instruction;
1772	uint64_t InstrSize, InstrOffset = EntryOffset;
1773	while (InstrOffset < EntryOffset + EntrySize) {
1774	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1775	// Check if the entry size needs adjustment.
1776	if (EntryOffset == 0 && BC->MIB->isTerminateBranch(Inst: Instruction) &&
1777	EntrySize == 8)
1778	EntrySize = 16;
1779
1780	if (BC->MIB->isIndirectBranch(Inst: Instruction))
1781	break;
1782
1783	InstrOffset += InstrSize;
1784	}
1785
1786	if (InstrOffset + InstrSize > EntryOffset + EntrySize)
1787	continue;
1788
1789	uint64_t TargetAddress;
1790	if (!BC->MIB->evaluateMemOperandTarget(Inst: Instruction, Target&: TargetAddress,
1791	Address: SectionAddress + InstrOffset,
1792	Size: InstrSize)) {
1793	BC->errs() << "BOLT-ERROR: error evaluating PLT instruction at offset 0x"
1794	<< Twine::utohexstr(Val: SectionAddress + InstrOffset) << '\n';
1795	exit(status: 1);
1796	}
1797
1798	createPLTBinaryFunction(TargetAddress, EntryAddress: SectionAddress + EntryOffset,
1799	EntrySize);
1800	}
1801	}
1802
1803	void RewriteInstance::disassemblePLT() {
1804	auto analyzeOnePLTSection = [&](BinarySection &Section, uint64_t EntrySize) {
1805	if (BC->isAArch64())
1806	return disassemblePLTSectionAArch64(Section);
1807	if (BC->isRISCV())
1808	return disassemblePLTSectionRISCV(Section);
1809	if (BC->isX86())
1810	return disassemblePLTSectionX86(Section, EntrySize);
1811	llvm_unreachable("Unmplemented PLT");
1812	};
1813
1814	for (BinarySection &Section : BC->allocatableSections()) {
1815	const PLTSectionInfo *PLTSI = getPLTSectionInfo(SectionName: Section.getName());
1816	if (!PLTSI)
1817	continue;
1818
1819	analyzeOnePLTSection(Section, PLTSI->EntrySize);
1820
1821	BinaryFunction *PltBF;
1822	auto BFIter = BC->getBinaryFunctions().find(x: Section.getAddress());
1823	if (BFIter != BC->getBinaryFunctions().end()) {
1824	PltBF = &BFIter->second;
1825	} else {
1826	// If we did not register any function at the start of the section,
1827	// then it must be a general PLT entry. Add a function at the location.
1828	PltBF = BC->createBinaryFunction(
1829	Name: "__BOLT_PSEUDO_" + Section.getName().str(), Section,
1830	Address: Section.getAddress(), Size: 0, SymbolSize: PLTSI->EntrySize, Alignment: Section.getAlignment());
1831	}
1832	PltBF->setPseudo(true);
1833	}
1834	}
1835
1836	void RewriteInstance::adjustFunctionBoundaries() {
1837	for (auto BFI = BC->getBinaryFunctions().begin(),
1838	BFE = BC->getBinaryFunctions().end();
1839	BFI != BFE; ++BFI) {
1840	BinaryFunction &Function = BFI->second;
1841	const BinaryFunction *NextFunction = nullptr;
1842	if (std::next(x: BFI) != BFE)
1843	NextFunction = &std::next(x: BFI)->second;
1844
1845	// Check if there's a symbol or a function with a larger address in the
1846	// same section. If there is - it determines the maximum size for the
1847	// current function. Otherwise, it is the size of a containing section
1848	// the defines it.
1849	//
1850	// NOTE: ignore some symbols that could be tolerated inside the body
1851	// of a function.
1852	auto NextSymRefI = FileSymRefs.upper_bound(x: Function.getAddress());
1853	while (NextSymRefI != FileSymRefs.end()) {
1854	SymbolRef &Symbol = NextSymRefI->second;
1855	const uint64_t SymbolAddress = NextSymRefI->first;
1856	const uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1857
1858	if (NextFunction && SymbolAddress >= NextFunction->getAddress())
1859	break;
1860
1861	if (!Function.isSymbolValidInScope(Symbol, SymbolSize))
1862	break;
1863
1864	// Skip basic block labels. This happens on RISC-V with linker relaxation
1865	// enabled because every branch needs a relocation and corresponding
1866	// symbol. We don't want to add such symbols as entry points.
1867	const auto PrivateLabelPrefix = BC->AsmInfo->getPrivateLabelPrefix();
1868	if (!PrivateLabelPrefix.empty() &&
1869	cantFail(ValOrErr: Symbol.getName()).starts_with(Prefix: PrivateLabelPrefix)) {
1870	++NextSymRefI;
1871	continue;
1872	}
1873
1874	// This is potentially another entry point into the function.
1875	uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();
1876	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "
1877	<< Function << " at offset 0x"
1878	<< Twine::utohexstr(EntryOffset) << '\n');
1879	Function.addEntryPointAtOffset(Offset: EntryOffset);
1880
1881	++NextSymRefI;
1882	}
1883
1884	// Function runs at most till the end of the containing section.
1885	uint64_t NextObjectAddress = Function.getOriginSection()->getEndAddress();
1886	// Or till the next object marked by a symbol.
1887	if (NextSymRefI != FileSymRefs.end())
1888	NextObjectAddress = std::min(a: NextSymRefI->first, b: NextObjectAddress);
1889
1890	// Or till the next function not marked by a symbol.
1891	if (NextFunction)
1892	NextObjectAddress =
1893	std::min(a: NextFunction->getAddress(), b: NextObjectAddress);
1894
1895	const uint64_t MaxSize = NextObjectAddress - Function.getAddress();
1896	if (MaxSize < Function.getSize()) {
1897	BC->errs() << "BOLT-ERROR: symbol seen in the middle of the function "
1898	<< Function << ". Skipping.\n";
1899	Function.setSimple(false);
1900	Function.setMaxSize(Function.getSize());
1901	continue;
1902	}
1903	Function.setMaxSize(MaxSize);
1904	if (!Function.getSize() && Function.isSimple()) {
1905	// Some assembly functions have their size set to 0, use the max
1906	// size as their real size.
1907	if (opts::Verbosity >= 1)
1908	BC->outs() << "BOLT-INFO: setting size of function " << Function
1909	<< " to " << Function.getMaxSize() << " (was 0)\n";
1910	Function.setSize(Function.getMaxSize());
1911	}
1912	}
1913	}
1914
1915	void RewriteInstance::relocateEHFrameSection() {
1916	assert(EHFrameSection && "Non-empty .eh_frame section expected.");
1917
1918	BinarySection *RelocatedEHFrameSection =
1919	getSection(Name: ".relocated" + getEHFrameSectionName());
1920	assert(RelocatedEHFrameSection &&
1921	"Relocated eh_frame section should be preregistered.");
1922	DWARFDataExtractor DE(EHFrameSection->getContents(),
1923	BC->AsmInfo->isLittleEndian(),
1924	BC->AsmInfo->getCodePointerSize());
1925	auto createReloc = [&](uint64_t Value, uint64_t Offset, uint64_t DwarfType) {
1926	if (DwarfType == dwarf::DW_EH_PE_omit)
1927	return;
1928
1929	// Only fix references that are relative to other locations.
1930	if (!(DwarfType & dwarf::DW_EH_PE_pcrel) &&
1931	!(DwarfType & dwarf::DW_EH_PE_textrel) &&
1932	!(DwarfType & dwarf::DW_EH_PE_funcrel) &&
1933	!(DwarfType & dwarf::DW_EH_PE_datarel))
1934	return;
1935
1936	if (!(DwarfType & dwarf::DW_EH_PE_sdata4))
1937	return;
1938
1939	uint32_t RelType;
1940	switch (DwarfType & 0x0f) {
1941	default:
1942	llvm_unreachable("unsupported DWARF encoding type");
1943	case dwarf::DW_EH_PE_sdata4:
1944	case dwarf::DW_EH_PE_udata4:
1945	RelType = Relocation::getPC32();
1946	Offset -= 4;
1947	break;
1948	case dwarf::DW_EH_PE_sdata8:
1949	case dwarf::DW_EH_PE_udata8:
1950	RelType = Relocation::getPC64();
1951	Offset -= 8;
1952	break;
1953	}
1954
1955	// Create a relocation against an absolute value since the goal is to
1956	// preserve the contents of the section independent of the new values
1957	// of referenced symbols.
1958	RelocatedEHFrameSection->addRelocation(Offset, Symbol: nullptr, Type: RelType, Addend: Value);
1959	};
1960
1961	Error E = EHFrameParser::parse(Data: DE, EHFrameAddress: EHFrameSection->getAddress(), PatcherCallback: createReloc);
1962	check_error(E: std::move(E), Message: "failed to patch EH frame");
1963	}
1964
1965	Error RewriteInstance::readSpecialSections() {
1966	NamedRegionTimer T("readSpecialSections", "read special sections",
1967	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
1968
1969	bool HasTextRelocations = false;
1970	bool HasSymbolTable = false;
1971	bool HasDebugInfo = false;
1972
1973	// Process special sections.
1974	for (const SectionRef &Section : InputFile->sections()) {
1975	Expected<StringRef> SectionNameOrErr = Section.getName();
1976	check_error(E: SectionNameOrErr.takeError(), Message: "cannot get section name");
1977	StringRef SectionName = *SectionNameOrErr;
1978
1979	if (Error E = Section.getContents().takeError())
1980	return E;
1981	BC->registerSection(Section);
1982	LLVM_DEBUG(
1983	dbgs() << "BOLT-DEBUG: registering section " << SectionName << " @ 0x"
1984	<< Twine::utohexstr(Section.getAddress()) << ":0x"
1985	<< Twine::utohexstr(Section.getAddress() + Section.getSize())
1986	<< "\n");
1987	if (isDebugSection(SectionName))
1988	HasDebugInfo = true;
1989	}
1990
1991	// Set IsRelro section attribute based on PT_GNU_RELRO segment.
1992	markGnuRelroSections();
1993
1994	if (HasDebugInfo && !opts::UpdateDebugSections && !opts::AggregateOnly) {
1995	BC->errs() << "BOLT-WARNING: debug info will be stripped from the binary. "
1996	"Use -update-debug-sections to keep it.\n";
1997	}
1998
1999	HasTextRelocations = (bool)BC->getUniqueSectionByName(
2000	SectionName: ".rela" + std::string(BC->getMainCodeSectionName()));
2001	HasSymbolTable = (bool)BC->getUniqueSectionByName(SectionName: ".symtab");
2002	EHFrameSection = BC->getUniqueSectionByName(SectionName: ".eh_frame");
2003
2004	if (ErrorOr<BinarySection &> BATSec =
2005	BC->getUniqueSectionByName(SectionName: BoltAddressTranslation::SECTION_NAME)) {
2006	BC->HasBATSection = true;
2007	// Do not read BAT when plotting a heatmap
2008	if (opts::HeatmapMode != opts::HeatmapModeKind::HM_Exclusive) {
2009	if (std::error_code EC = BAT->parse(OS&: BC->outs(), Buf: BATSec->getContents())) {
2010	BC->errs() << "BOLT-ERROR: failed to parse BOLT address translation "
2011	"table.\n";
2012	exit(status: 1);
2013	}
2014	}
2015	}
2016
2017	if (opts::PrintSections) {
2018	BC->outs() << "BOLT-INFO: Sections from original binary:\n";
2019	BC->printSections(OS&: BC->outs());
2020	}
2021
2022	if (opts::RelocationMode == cl::BOU_TRUE && !HasTextRelocations) {
2023	BC->errs()
2024	<< "BOLT-ERROR: relocations against code are missing from the input "
2025	"file. Cannot proceed in relocations mode (-relocs).\n";
2026	exit(status: 1);
2027	}
2028
2029	BC->HasRelocations =
2030	HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE);
2031
2032	if (BC->IsLinuxKernel && BC->HasRelocations) {
2033	BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";
2034	BC->HasRelocations = false;
2035	}
2036
2037	BC->IsStripped = !HasSymbolTable;
2038
2039	if (BC->IsStripped && !opts::AllowStripped) {
2040	BC->errs()
2041	<< "BOLT-ERROR: stripped binaries are not supported. If you know "
2042	"what you're doing, use --allow-stripped to proceed";
2043	exit(status: 1);
2044	}
2045
2046	// Force non-relocation mode for heatmap generation
2047	if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
2048	BC->HasRelocations = false;
2049
2050	if (BC->HasRelocations)
2051	BC->outs() << "BOLT-INFO: enabling " << (opts::StrictMode ? "strict " : "")
2052	<< "relocation mode\n";
2053
2054	// Read EH frame for function boundaries info.
2055	Expected<const DWARFDebugFrame *> EHFrameOrError = BC->DwCtx->getEHFrame();
2056	if (!EHFrameOrError)
2057	report_error(Message: "expected valid eh_frame section", E: EHFrameOrError.takeError());
2058	CFIRdWrt.reset(p: new CFIReaderWriter(*BC, *EHFrameOrError.get()));
2059
2060	processSectionMetadata();
2061
2062	// Read .dynamic/PT_DYNAMIC.
2063	return readELFDynamic();
2064	}
2065
2066	void RewriteInstance::adjustCommandLineOptions() {
2067	if (BC->isAArch64() && !BC->HasRelocations)
2068	BC->errs() << "BOLT-WARNING: non-relocation mode for AArch64 is not fully "
2069	"supported\n";
2070
2071	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
2072	RtLibrary->adjustCommandLineOptions(BC: *BC);
2073
2074	if (BC->isX86() && BC->MAB->allowAutoPadding()) {
2075	if (!BC->HasRelocations) {
2076	BC->errs()
2077	<< "BOLT-ERROR: cannot apply mitigations for Intel JCC erratum in "
2078	"non-relocation mode\n";
2079	exit(status: 1);
2080	}
2081	BC->outs()
2082	<< "BOLT-WARNING: using mitigation for Intel JCC erratum, layout "
2083	"may take several minutes\n";
2084	}
2085
2086	if (opts::SplitEH && !BC->HasRelocations) {
2087	BC->errs() << "BOLT-WARNING: disabling -split-eh in non-relocation mode\n";
2088	opts::SplitEH = false;
2089	}
2090
2091	if (opts::StrictMode && !BC->HasRelocations) {
2092	BC->errs()
2093	<< "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "
2094	"mode\n";
2095	opts::StrictMode = false;
2096	}
2097
2098	if (BC->HasRelocations && opts::AggregateOnly &&
2099	!opts::StrictMode.getNumOccurrences()) {
2100	BC->outs() << "BOLT-INFO: enabling strict relocation mode for aggregation "
2101	"purposes\n";
2102	opts::StrictMode = true;
2103	}
2104
2105	if (!BC->HasRelocations &&
2106	opts::ReorderFunctions != ReorderFunctions::RT_NONE) {
2107	BC->errs() << "BOLT-ERROR: function reordering only works when "
2108	<< "relocations are enabled\n";
2109	exit(status: 1);
2110	}
2111
2112	if (!BC->HasRelocations &&
2113	opts::ICF == IdenticalCodeFolding::ICFLevel::Safe) {
2114	BC->errs() << "BOLT-ERROR: binary built without relocations. Safe ICF is "
2115	"not supported\n";
2116	exit(status: 1);
2117	}
2118
2119	if (opts::Instrument ||
2120	(opts::ReorderFunctions != ReorderFunctions::RT_NONE &&
2121	!opts::HotText.getNumOccurrences())) {
2122	opts::HotText = true;
2123	} else if (opts::HotText && !BC->HasRelocations) {
2124	BC->errs() << "BOLT-WARNING: hot text is disabled in non-relocation mode\n";
2125	opts::HotText = false;
2126	}
2127
2128	if (opts::Instrument && opts::UseGnuStack) {
2129	BC->errs() << "BOLT-ERROR: cannot avoid having writeable and executable "
2130	"segment in instrumented binary if program headers will be "
2131	"updated in place\n";
2132	exit(status: 1);
2133	}
2134
2135	if (opts::HotText && opts::HotTextMoveSections.getNumOccurrences() == 0) {
2136	opts::HotTextMoveSections.addValue(V: ".stub");
2137	opts::HotTextMoveSections.addValue(V: ".mover");
2138	opts::HotTextMoveSections.addValue(V: ".never_hugify");
2139	}
2140
2141	if (opts::UseOldText && !BC->OldTextSectionAddress) {
2142	BC->errs()
2143	<< "BOLT-WARNING: cannot use old .text as the section was not found"
2144	"\n";
2145	opts::UseOldText = false;
2146	}
2147	if (opts::UseOldText && !BC->HasRelocations) {
2148	BC->errs() << "BOLT-WARNING: cannot use old .text in non-relocation mode\n";
2149	opts::UseOldText = false;
2150	}
2151
2152	if (!opts::AlignText.getNumOccurrences())
2153	opts::AlignText = BC->PageAlign;
2154
2155	if (opts::AlignText < opts::AlignFunctions)
2156	opts::AlignText = (unsigned)opts::AlignFunctions;
2157
2158	if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode &&
2159	!opts::UseOldText)
2160	opts::Lite = true;
2161
2162	if (opts::Lite && opts::UseOldText) {
2163	BC->errs() << "BOLT-WARNING: cannot combine -lite with -use-old-text. "
2164	"Disabling -use-old-text.\n";
2165	opts::UseOldText = false;
2166	}
2167
2168	if (opts::Lite && opts::StrictMode) {
2169	BC->errs()
2170	<< "BOLT-ERROR: -strict and -lite cannot be used at the same time\n";
2171	exit(status: 1);
2172	}
2173
2174	if (opts::Lite)
2175	BC->outs() << "BOLT-INFO: enabling lite mode\n";
2176
2177	if (BC->IsLinuxKernel) {
2178	if (!opts::KeepNops.getNumOccurrences())
2179	opts::KeepNops = true;
2180
2181	// Linux kernel may resume execution after a trap instruction in some cases.
2182	if (!opts::TerminalTrap.getNumOccurrences())
2183	opts::TerminalTrap = false;
2184	}
2185	}
2186
2187	namespace {
2188	template <typename ELFT>
2189	int64_t getRelocationAddend(const ELFObjectFile<ELFT> *Obj,
2190	const RelocationRef &RelRef) {
2191	using ELFShdrTy = typename ELFT::Shdr;
2192	using Elf_Rela = typename ELFT::Rela;
2193	int64_t Addend = 0;
2194	const ELFFile<ELFT> &EF = Obj->getELFFile();
2195	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2196	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2197	switch (RelocationSection->sh_type) {
2198	default:
2199	llvm_unreachable("unexpected relocation section type");
2200	case ELF::SHT_REL:
2201	break;
2202	case ELF::SHT_RELA: {
2203	const Elf_Rela *RelA = Obj->getRela(Rel);
2204	Addend = RelA->r_addend;
2205	break;
2206	}
2207	}
2208
2209	return Addend;
2210	}
2211
2212	int64_t getRelocationAddend(const ELFObjectFileBase *Obj,
2213	const RelocationRef &Rel) {
2214	return getRelocationAddend(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2215	}
2216
2217	template <typename ELFT>
2218	uint32_t getRelocationSymbol(const ELFObjectFile<ELFT> *Obj,
2219	const RelocationRef &RelRef) {
2220	using ELFShdrTy = typename ELFT::Shdr;
2221	uint32_t Symbol = 0;
2222	const ELFFile<ELFT> &EF = Obj->getELFFile();
2223	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2224	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2225	switch (RelocationSection->sh_type) {
2226	default:
2227	llvm_unreachable("unexpected relocation section type");
2228	case ELF::SHT_REL:
2229	Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL());
2230	break;
2231	case ELF::SHT_RELA:
2232	Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL());
2233	break;
2234	}
2235
2236	return Symbol;
2237	}
2238
2239	uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj,
2240	const RelocationRef &Rel) {
2241	return getRelocationSymbol(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2242	}
2243	} // anonymous namespace
2244
2245	bool RewriteInstance::analyzeRelocation(
2246	const RelocationRef &Rel, uint32_t &RType, std::string &SymbolName,
2247	bool &IsSectionRelocation, uint64_t &SymbolAddress, int64_t &Addend,
2248	uint64_t &ExtractedValue, bool &Skip) const {
2249	Skip = false;
2250	if (!Relocation::isSupported(Type: RType))
2251	return false;
2252
2253	auto IsWeakReference = [](const SymbolRef &Symbol) {
2254	Expected<uint32_t> SymFlagsOrErr = Symbol.getFlags();
2255	if (!SymFlagsOrErr)
2256	return false;
2257	return (*SymFlagsOrErr & SymbolRef::SF_Undefined) &&
2258	(*SymFlagsOrErr & SymbolRef::SF_Weak);
2259	};
2260
2261	const bool IsAArch64 = BC->isAArch64();
2262
2263	const size_t RelSize = Relocation::getSizeForType(Type: RType);
2264
2265	ErrorOr<uint64_t> Value =
2266	BC->getUnsignedValueAtAddress(Address: Rel.getOffset(), Size: RelSize);
2267	assert(Value && "failed to extract relocated value");
2268
2269	ExtractedValue = Relocation::extractValue(Type: RType, Contents: *Value, PC: Rel.getOffset());
2270	Addend = getRelocationAddend(Obj: InputFile, Rel);
2271
2272	const bool IsPCRelative = Relocation::isPCRelative(Type: RType);
2273	const uint64_t PCRelOffset = IsPCRelative && !IsAArch64 ? Rel.getOffset() : 0;
2274	bool SkipVerification = false;
2275	auto SymbolIter = Rel.getSymbol();
2276	if (SymbolIter == InputFile->symbol_end()) {
2277	SymbolAddress = ExtractedValue - Addend + PCRelOffset;
2278	MCSymbol *RelSymbol =
2279	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "RELSYMat");
2280	SymbolName = std::string(RelSymbol->getName());
2281	IsSectionRelocation = false;
2282	} else {
2283	const SymbolRef &Symbol = *SymbolIter;
2284	SymbolName = std::string(cantFail(ValOrErr: Symbol.getName()));
2285	SymbolAddress = cantFail(ValOrErr: Symbol.getAddress());
2286	SkipVerification = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Other);
2287	// Section symbols are marked as ST_Debug.
2288	IsSectionRelocation = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Debug);
2289	// Check for PLT entry registered with symbol name
2290	if (!SymbolAddress && !IsWeakReference(Symbol) &&
2291	(IsAArch64 || BC->isRISCV())) {
2292	const BinaryData *BD = BC->getPLTBinaryDataByName(Name: SymbolName);
2293	SymbolAddress = BD ? BD->getAddress() : 0;
2294	}
2295	}
2296	// For PIE or dynamic libs, the linker may choose not to put the relocation
2297	// result at the address if it is a X86_64_64 one because it will emit a
2298	// dynamic relocation (X86_RELATIVE) for the dynamic linker and loader to
2299	// resolve it at run time. The static relocation result goes as the addend
2300	// of the dynamic relocation in this case. We can't verify these cases.
2301	// FIXME: perhaps we can try to find if it really emitted a corresponding
2302	// RELATIVE relocation at this offset with the correct value as the addend.
2303	if (!BC->HasFixedLoadAddress && RelSize == 8)
2304	SkipVerification = true;
2305
2306	if (IsSectionRelocation && !IsAArch64) {
2307	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2308	assert(Section && "section expected for section relocation");
2309	SymbolName = "section " + std::string(Section->getName());
2310	// Convert section symbol relocations to regular relocations inside
2311	// non-section symbols.
2312	if (Section->containsAddress(Address: ExtractedValue) && !IsPCRelative) {
2313	SymbolAddress = ExtractedValue;
2314	Addend = 0;
2315	} else {
2316	Addend = ExtractedValue - (SymbolAddress - PCRelOffset);
2317	}
2318	}
2319
2320	// GOT relocation can cause the underlying instruction to be modified by the
2321	// linker, resulting in the extracted value being different from the actual
2322	// symbol. It's also possible to have a GOT entry for a symbol defined in the
2323	// binary. In the latter case, the instruction can be using the GOT version
2324	// causing the extracted value mismatch. Similar cases can happen for TLS.
2325	// Pass the relocation information as is to the disassembler and let it decide
2326	// how to use it for the operand symbolization.
2327	if (Relocation::isGOT(Type: RType) || Relocation::isTLS(Type: RType)) {
2328	SkipVerification = true;
2329	} else if (!SymbolAddress) {
2330	assert(!IsSectionRelocation);
2331	if (ExtractedValue || Addend == 0 || IsPCRelative) {
2332	SymbolAddress =
2333	truncateToSize(Value: ExtractedValue - Addend + PCRelOffset, Bytes: RelSize);
2334	} else {
2335	// This is weird case. The extracted value is zero but the addend is
2336	// non-zero and the relocation is not pc-rel. Using the previous logic,
2337	// the SymbolAddress would end up as a huge number. Seen in
2338	// exceptions_pic.test.
2339	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocation @ 0x"
2340	<< Twine::utohexstr(Rel.getOffset())
2341	<< " value does not match addend for "
2342	<< "relocation to undefined symbol.\n");
2343	return true;
2344	}
2345	}
2346
2347	auto verifyExtractedValue = [&]() {
2348	if (SkipVerification)
2349	return true;
2350
2351	if (IsAArch64 || BC->isRISCV())
2352	return true;
2353
2354	if (SymbolName == "__hot_start" || SymbolName == "__hot_end")
2355	return true;
2356
2357	if (RType == ELF::R_X86_64_PLT32)
2358	return true;
2359
2360	return truncateToSize(Value: ExtractedValue, Bytes: RelSize) ==
2361	truncateToSize(Value: SymbolAddress + Addend - PCRelOffset, Bytes: RelSize);
2362	};
2363
2364	(void)verifyExtractedValue;
2365	assert(verifyExtractedValue() && "mismatched extracted relocation value");
2366
2367	return true;
2368	}
2369
2370	void RewriteInstance::processDynamicRelocations() {
2371	// Read .relr.dyn section containing compressed R_*_RELATIVE relocations.
2372	if (DynamicRelrSize > 0) {
2373	ErrorOr<BinarySection &> DynamicRelrSectionOrErr =
2374	BC->getSectionForAddress(Address: *DynamicRelrAddress);
2375	if (!DynamicRelrSectionOrErr)
2376	report_error(Message: "unable to find section corresponding to DT_RELR",
2377	EC: DynamicRelrSectionOrErr.getError());
2378	if (DynamicRelrSectionOrErr->getSize() != DynamicRelrSize)
2379	report_error(Message: "section size mismatch for DT_RELRSZ",
2380	EC: errc::executable_format_error);
2381	readDynamicRelrRelocations(Section&: *DynamicRelrSectionOrErr);
2382	}
2383
2384	// Read relocations for PLT - DT_JMPREL.
2385	if (PLTRelocationsSize > 0) {
2386	ErrorOr<BinarySection &> PLTRelSectionOrErr =
2387	BC->getSectionForAddress(Address: *PLTRelocationsAddress);
2388	if (!PLTRelSectionOrErr)
2389	report_error(Message: "unable to find section corresponding to DT_JMPREL",
2390	EC: PLTRelSectionOrErr.getError());
2391	if (PLTRelSectionOrErr->getSize() != PLTRelocationsSize)
2392	report_error(Message: "section size mismatch for DT_PLTRELSZ",
2393	EC: errc::executable_format_error);
2394	readDynamicRelocations(Section: PLTRelSectionOrErr->getSectionRef(),
2395	/*IsJmpRel*/ true);
2396	}
2397
2398	// The rest of dynamic relocations - DT_RELA.
2399	// The static executable might have .rela.dyn secion and not have PT_DYNAMIC
2400	if (!DynamicRelocationsSize && BC->IsStaticExecutable) {
2401	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2402	BC->getUniqueSectionByName(SectionName: getRelaDynSectionName());
2403	if (DynamicRelSectionOrErr) {
2404	DynamicRelocationsAddress = DynamicRelSectionOrErr->getAddress();
2405	DynamicRelocationsSize = DynamicRelSectionOrErr->getSize();
2406	const SectionRef &SectionRef = DynamicRelSectionOrErr->getSectionRef();
2407	DynamicRelativeRelocationsCount = std::distance(
2408	first: SectionRef.relocation_begin(), last: SectionRef.relocation_end());
2409	}
2410	}
2411
2412	if (DynamicRelocationsSize > 0) {
2413	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2414	BC->getSectionForAddress(Address: *DynamicRelocationsAddress);
2415	if (!DynamicRelSectionOrErr)
2416	report_error(Message: "unable to find section corresponding to DT_RELA",
2417	EC: DynamicRelSectionOrErr.getError());
2418	auto DynamicRelSectionSize = DynamicRelSectionOrErr->getSize();
2419	// On RISC-V DT_RELASZ seems to include both .rela.dyn and .rela.plt
2420	if (DynamicRelocationsSize == DynamicRelSectionSize + PLTRelocationsSize)
2421	DynamicRelocationsSize = DynamicRelSectionSize;
2422	if (DynamicRelSectionSize != DynamicRelocationsSize)
2423	report_error(Message: "section size mismatch for DT_RELASZ",
2424	EC: errc::executable_format_error);
2425	readDynamicRelocations(Section: DynamicRelSectionOrErr->getSectionRef(),
2426	/*IsJmpRel*/ false);
2427	}
2428	}
2429
2430	void RewriteInstance::processRelocations() {
2431	if (!BC->HasRelocations)
2432	return;
2433
2434	for (const SectionRef &Section : InputFile->sections()) {
2435	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2436	if (SecIter == InputFile->section_end())
2437	continue;
2438	if (BinarySection(*BC, Section).isAllocatable())
2439	continue;
2440
2441	readRelocations(Section);
2442	}
2443
2444	if (NumFailedRelocations)
2445	BC->errs() << "BOLT-WARNING: Failed to analyze " << NumFailedRelocations
2446	<< " relocations\n";
2447	}
2448
2449	void RewriteInstance::readDynamicRelocations(const SectionRef &Section,
2450	bool IsJmpRel) {
2451	assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected");
2452
2453	LLVM_DEBUG({
2454	StringRef SectionName = cantFail(Section.getName());
2455	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2456	<< ":\n";
2457	});
2458
2459	for (const RelocationRef &Rel : Section.relocations()) {
2460	const uint32_t RType = Relocation::getType(Rel);
2461	if (Relocation::isNone(Type: RType))
2462	continue;
2463
2464	StringRef SymbolName = "<none>";
2465	MCSymbol *Symbol = nullptr;
2466	uint64_t SymbolAddress = 0;
2467	const uint64_t Addend = getRelocationAddend(Obj: InputFile, Rel);
2468
2469	symbol_iterator SymbolIter = Rel.getSymbol();
2470	if (SymbolIter != InputFile->symbol_end()) {
2471	SymbolName = cantFail(ValOrErr: SymbolIter->getName());
2472	BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName);
2473	Symbol = BD ? BD->getSymbol()
2474	: BC->getOrCreateUndefinedGlobalSymbol(Name: SymbolName);
2475	SymbolAddress = cantFail(ValOrErr: SymbolIter->getAddress());
2476	(void)SymbolAddress;
2477	}
2478
2479	LLVM_DEBUG(
2480	SmallString<16> TypeName;
2481	Rel.getTypeName(TypeName);
2482	dbgs() << "BOLT-DEBUG: dynamic relocation at 0x"
2483	<< Twine::utohexstr(Rel.getOffset()) << " : " << TypeName
2484	<< " : " << SymbolName << " : " << Twine::utohexstr(SymbolAddress)
2485	<< " : + 0x" << Twine::utohexstr(Addend) << '\n'
2486	);
2487
2488	if (IsJmpRel)
2489	IsJmpRelocation[RType] = true;
2490
2491	if (Symbol)
2492	SymbolIndex[Symbol] = getRelocationSymbol(Obj: InputFile, Rel);
2493
2494	const uint64_t ReferencedAddress = SymbolAddress + Addend;
2495	BinaryFunction *Func =
2496	BC->getBinaryFunctionContainingAddress(Address: ReferencedAddress);
2497
2498	if (Relocation::isRelative(Type: RType) && SymbolAddress == 0) {
2499	if (Func) {
2500	if (!Func->isInConstantIsland(Address: ReferencedAddress)) {
2501	if (const uint64_t ReferenceOffset =
2502	ReferencedAddress - Func->getAddress()) {
2503	Func->addEntryPointAtOffset(Offset: ReferenceOffset);
2504	}
2505	} else {
2506	BC->errs() << "BOLT-ERROR: referenced address at 0x"
2507	<< Twine::utohexstr(Val: ReferencedAddress)
2508	<< " is in constant island of function " << *Func << "\n";
2509	exit(status: 1);
2510	}
2511	}
2512	} else if (Relocation::isRelative(Type: RType) && SymbolAddress != 0) {
2513	BC->errs() << "BOLT-ERROR: symbol address non zero for RELATIVE "
2514	"relocation type\n";
2515	exit(status: 1);
2516	}
2517
2518	BC->addDynamicRelocation(Address: Rel.getOffset(), Symbol, Type: RType, Addend);
2519	}
2520	}
2521
2522	void RewriteInstance::readDynamicRelrRelocations(BinarySection &Section) {
2523	assert(Section.isAllocatable() && "allocatable expected");
2524
2525	LLVM_DEBUG({
2526	StringRef SectionName = Section.getName();
2527	dbgs() << "BOLT-DEBUG: reading relocations in section " << SectionName
2528	<< ":\n";
2529	});
2530
2531	const uint32_t RType = Relocation::getRelative();
2532	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
2533	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
2534
2535	auto ExtractAddendValue = [&](uint64_t Address) -> uint64_t {
2536	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
2537	assert(Section && "cannot get section for data address from RELR");
2538	DataExtractor DE = DataExtractor(Section->getContents(),
2539	BC->AsmInfo->isLittleEndian(), PSize);
2540	uint64_t Offset = Address - Section->getAddress();
2541	return DE.getUnsigned(offset_ptr: &Offset, byte_size: PSize);
2542	};
2543
2544	auto AddRelocation = [&](uint64_t Address) {
2545	uint64_t Addend = ExtractAddendValue(Address);
2546	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: R_*_RELATIVE relocation at 0x"
2547	<< Twine::utohexstr(Address) << " to 0x"
2548	<< Twine::utohexstr(Addend) << '\n';);
2549	BC->addDynamicRelocation(Address, Symbol: nullptr, Type: RType, Addend);
2550	};
2551
2552	DataExtractor DE = DataExtractor(Section.getContents(),
2553	BC->AsmInfo->isLittleEndian(), PSize);
2554	uint64_t Offset = 0, Address = 0;
2555	uint64_t RelrCount = DynamicRelrSize / DynamicRelrEntrySize;
2556	while (RelrCount--) {
2557	assert(DE.isValidOffset(Offset));
2558	uint64_t Entry = DE.getUnsigned(offset_ptr: &Offset, byte_size: DynamicRelrEntrySize);
2559	if ((Entry & 1) == 0) {
2560	AddRelocation(Entry);
2561	Address = Entry + PSize;
2562	} else {
2563	const uint64_t StartAddress = Address;
2564	while (Entry >>= 1) {
2565	if (Entry & 1)
2566	AddRelocation(Address);
2567
2568	Address += PSize;
2569	}
2570
2571	Address = StartAddress + MaxDelta;
2572	}
2573	}
2574	}
2575
2576	void RewriteInstance::printRelocationInfo(const RelocationRef &Rel,
2577	StringRef SymbolName,
2578	uint64_t SymbolAddress,
2579	uint64_t Addend,
2580	uint64_t ExtractedValue) const {
2581	SmallString<16> TypeName;
2582	Rel.getTypeName(Result&: TypeName);
2583	const uint64_t Address = SymbolAddress + Addend;
2584	const uint64_t Offset = Rel.getOffset();
2585	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2586	BinaryFunction *Func =
2587	BC->getBinaryFunctionContainingAddress(Address: Offset, CheckPastEnd: false, UseMaxSize: BC->isAArch64());
2588	dbgs() << formatv(Fmt: "Relocation: offset = {0:x}; type = {1}; value = {2:x}; ",
2589	Vals: Offset, Vals&: TypeName, Vals&: ExtractedValue)
2590	<< formatv(Fmt: "symbol = {0} ({1}); symbol address = {2:x}; ", Vals&: SymbolName,
2591	Vals: Section ? Section->getName() : "", Vals&: SymbolAddress)
2592	<< formatv(Fmt: "addend = {0:x}; address = {1:x}; in = ", Vals&: Addend, Vals: Address);
2593	if (Func)
2594	dbgs() << Func->getPrintName();
2595	else
2596	dbgs() << BC->getSectionForAddress(Address: Rel.getOffset())->getName();
2597	dbgs() << '\n';
2598	}
2599
2600	void RewriteInstance::readRelocations(const SectionRef &Section) {
2601	LLVM_DEBUG({
2602	StringRef SectionName = cantFail(Section.getName());
2603	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2604	<< ":\n";
2605	});
2606	if (BinarySection(*BC, Section).isAllocatable()) {
2607	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring runtime relocations\n");
2608	return;
2609	}
2610	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2611	assert(SecIter != InputFile->section_end() && "relocated section expected");
2612	SectionRef RelocatedSection = *SecIter;
2613
2614	StringRef RelocatedSectionName = cantFail(ValOrErr: RelocatedSection.getName());
2615	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is "
2616	<< RelocatedSectionName << '\n');
2617
2618	if (!BinarySection(*BC, RelocatedSection).isAllocatable()) {
2619	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocations against "
2620	<< "non-allocatable section\n");
2621	return;
2622	}
2623	const bool SkipRelocs = StringSwitch<bool>(RelocatedSectionName)
2624	.Cases(S0: ".plt", S1: ".rela.plt", S2: ".got.plt",
2625	S3: ".eh_frame", S4: ".gcc_except_table", Value: true)
2626	.Default(Value: false);
2627	if (SkipRelocs) {
2628	LLVM_DEBUG(
2629	dbgs() << "BOLT-DEBUG: ignoring relocations against known section\n");
2630	return;
2631	}
2632
2633	for (const RelocationRef &Rel : Section.relocations())
2634	handleRelocation(RelocatedSection, Rel);
2635	}
2636
2637	void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
2638	const RelocationRef &Rel) {
2639	const bool IsAArch64 = BC->isAArch64();
2640	const bool IsX86 = BC->isX86();
2641	const bool IsFromCode = RelocatedSection.isText();
2642	const bool IsWritable = BinarySection(*BC, RelocatedSection).isWritable();
2643
2644	SmallString<16> TypeName;
2645	Rel.getTypeName(Result&: TypeName);
2646	uint32_t RType = Relocation::getType(Rel);
2647	if (Relocation::skipRelocationType(Type: RType))
2648	return;
2649
2650	// Adjust the relocation type as the linker might have skewed it.
2651	if (IsX86 && (RType & ELF::R_X86_64_converted_reloc_bit)) {
2652	if (opts::Verbosity >= 1)
2653	dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n";
2654	RType &= ~ELF::R_X86_64_converted_reloc_bit;
2655	}
2656
2657	if (Relocation::isTLS(Type: RType)) {
2658	// No special handling required for TLS relocations on X86.
2659	if (IsX86)
2660	return;
2661
2662	// The non-got related TLS relocations on AArch64 and RISC-V also could be
2663	// skipped.
2664	if (!Relocation::isGOT(Type: RType))
2665	return;
2666	}
2667
2668	if (!IsAArch64 && BC->getDynamicRelocationAt(Address: Rel.getOffset())) {
2669	LLVM_DEBUG({
2670	dbgs() << formatv("BOLT-DEBUG: address {0:x} has a ", Rel.getOffset())
2671	<< "dynamic relocation against it. Ignoring static relocation.\n";
2672	});
2673	return;
2674	}
2675
2676	std::string SymbolName;
2677	uint64_t SymbolAddress;
2678	int64_t Addend;
2679	uint64_t ExtractedValue;
2680	bool IsSectionRelocation;
2681	bool Skip;
2682	if (!analyzeRelocation(Rel, RType, SymbolName, IsSectionRelocation,
2683	SymbolAddress, Addend, ExtractedValue, Skip)) {
2684	LLVM_DEBUG({
2685	dbgs() << "BOLT-WARNING: failed to analyze relocation @ offset = "
2686	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2687	});
2688	++NumFailedRelocations;
2689	return;
2690	}
2691
2692	if (Skip) {
2693	LLVM_DEBUG({
2694	dbgs() << "BOLT-DEBUG: skipping relocation @ offset = "
2695	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2696	});
2697	return;
2698	}
2699
2700	if (!IsFromCode && !IsWritable && (IsX86 || IsAArch64) &&
2701	Relocation::isPCRelative(Type: RType)) {
2702	BinaryData *BD = BC->getBinaryDataContainingAddress(Address: Rel.getOffset());
2703	if (BD && (BD->nameStartsWith(Prefix: "_ZTV") || // vtable
2704	BD->nameStartsWith(Prefix: "_ZTCN"))) { // construction vtable
2705	BinaryFunction *BF = BC->getBinaryFunctionContainingAddress(
2706	Address: SymbolAddress, /*CheckPastEnd*/ false, /*UseMaxSize*/ true);
2707	if (BF) {
2708	if (BF->getAddress() != SymbolAddress) {
2709	BC->errs()
2710	<< "BOLT-ERROR: the virtual function table entry at offset 0x"
2711	<< Twine::utohexstr(Val: Rel.getOffset())
2712	<< " points to the middle of a function @ 0x"
2713	<< Twine::utohexstr(Val: BF->getAddress()) << "\n";
2714	exit(status: 1);
2715	}
2716	BC->addRelocation(Address: Rel.getOffset(), Symbol: BF->getSymbol(), Type: RType, Addend,
2717	Value: ExtractedValue);
2718	return;
2719	}
2720	}
2721	}
2722
2723	const uint64_t Address = SymbolAddress + Addend;
2724
2725	LLVM_DEBUG({
2726	dbgs() << "BOLT-DEBUG: ";
2727	printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend, ExtractedValue);
2728	});
2729
2730	BinaryFunction *ContainingBF = nullptr;
2731	if (IsFromCode) {
2732	ContainingBF =
2733	BC->getBinaryFunctionContainingAddress(Address: Rel.getOffset(),
2734	/*CheckPastEnd*/ false,
2735	/*UseMaxSize*/ true);
2736	assert(ContainingBF && "cannot find function for address in code");
2737	if (!IsAArch64 && !ContainingBF->containsAddress(PC: Rel.getOffset())) {
2738	if (opts::Verbosity >= 1)
2739	BC->outs() << formatv(
2740	Fmt: "BOLT-INFO: {0} has relocations in padding area\n", Vals&: *ContainingBF);
2741	ContainingBF->setSize(ContainingBF->getMaxSize());
2742	ContainingBF->setSimple(false);
2743	return;
2744	}
2745	}
2746
2747	MCSymbol *ReferencedSymbol = nullptr;
2748	if (!IsSectionRelocation) {
2749	if (BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName)) {
2750	ReferencedSymbol = BD->getSymbol();
2751	} else if (BC->isGOTSymbol(SymName: SymbolName)) {
2752	if (BinaryData *BD = BC->getGOTSymbol())
2753	ReferencedSymbol = BD->getSymbol();
2754	} else if (BinaryData *BD = BC->getBinaryDataAtAddress(Address: SymbolAddress)) {
2755	ReferencedSymbol = BD->getSymbol();
2756	}
2757	}
2758
2759	ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};
2760	symbol_iterator SymbolIter = Rel.getSymbol();
2761	if (SymbolIter != InputFile->symbol_end()) {
2762	SymbolRef Symbol = *SymbolIter;
2763	section_iterator Section =
2764	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
2765	if (Section != InputFile->section_end()) {
2766	Expected<StringRef> SectionName = Section->getName();
2767	if (SectionName && !SectionName->empty())
2768	ReferencedSection = BC->getUniqueSectionByName(SectionName: *SectionName);
2769	} else if (BC->isRISCV() && ReferencedSymbol && ContainingBF &&
2770	(cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Absolute)) {
2771	// This might be a relocation for an ABS symbols like __global_pointer$ on
2772	// RISC-V
2773	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol,
2774	RelType: Relocation::getType(Rel), Addend: 0,
2775	Value: cantFail(ValOrErr: Symbol.getValue()));
2776	return;
2777	}
2778	}
2779
2780	if (!ReferencedSection)
2781	ReferencedSection = BC->getSectionForAddress(Address: SymbolAddress);
2782
2783	const bool IsToCode = ReferencedSection && ReferencedSection->isText();
2784
2785	// Special handling of PC-relative relocations.
2786	if (IsX86 && Relocation::isPCRelative(Type: RType)) {
2787	if (!IsFromCode && IsToCode) {
2788	// PC-relative relocations from data to code are tricky since the
2789	// original information is typically lost after linking, even with
2790	// '--emit-relocs'. Such relocations are normally used by PIC-style
2791	// jump tables and they reference both the jump table and jump
2792	// targets by computing the difference between the two. If we blindly
2793	// apply the relocation, it will appear that it references an arbitrary
2794	// location in the code, possibly in a different function from the one
2795	// containing the jump table.
2796	//
2797	// For that reason, we only register the fact that there is a
2798	// PC-relative relocation at a given address against the code.
2799	// The actual referenced label/address will be determined during jump
2800	// table analysis.
2801	BC->addPCRelativeDataRelocation(Address: Rel.getOffset());
2802	} else if (ContainingBF && !IsSectionRelocation && ReferencedSymbol) {
2803	// If we know the referenced symbol, register the relocation from
2804	// the code. It's required to properly handle cases where
2805	// "symbol + addend" references an object different from "symbol".
2806	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
2807	Addend, Value: ExtractedValue);
2808	} else {
2809	LLVM_DEBUG({
2810	dbgs() << "BOLT-DEBUG: not creating PC-relative relocation at"
2811	<< formatv("{0:x} for {1}\n", Rel.getOffset(), SymbolName);
2812	});
2813	}
2814
2815	return;
2816	}
2817
2818	bool ForceRelocation = BC->forceSymbolRelocations(SymbolName);
2819	if ((BC->isAArch64() || BC->isRISCV()) && Relocation::isGOT(Type: RType))
2820	ForceRelocation = true;
2821
2822	if (!ReferencedSection && !ForceRelocation) {
2823	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: cannot determine referenced section.\n");
2824	return;
2825	}
2826
2827	// Occasionally we may see a reference past the last byte of the function
2828	// typically as a result of __builtin_unreachable(). Check it here.
2829	BinaryFunction *ReferencedBF = BC->getBinaryFunctionContainingAddress(
2830	Address, /*CheckPastEnd*/ true, /*UseMaxSize*/ IsAArch64);
2831
2832	if (!IsSectionRelocation) {
2833	if (BinaryFunction *BF =
2834	BC->getBinaryFunctionContainingAddress(Address: SymbolAddress)) {
2835	if (BF != ReferencedBF) {
2836	// It's possible we are referencing a function without referencing any
2837	// code, e.g. when taking a bitmask action on a function address.
2838	BC->errs()
2839	<< "BOLT-WARNING: non-standard function reference (e.g. bitmask)"
2840	<< formatv(Fmt: " detected against function {0} from ", Vals&: *BF);
2841	if (IsFromCode)
2842	BC->errs() << formatv(Fmt: "function {0}\n", Vals&: *ContainingBF);
2843	else
2844	BC->errs() << formatv(Fmt: "data section at {0:x}\n", Vals: Rel.getOffset());
2845	LLVM_DEBUG(printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend,
2846	ExtractedValue));
2847	ReferencedBF = BF;
2848	}
2849	}
2850	} else if (ReferencedBF) {
2851	assert(ReferencedSection && "section expected for section relocation");
2852	if (*ReferencedBF->getOriginSection() != *ReferencedSection) {
2853	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring false function reference\n");
2854	ReferencedBF = nullptr;
2855	}
2856	}
2857
2858	// Workaround for a member function pointer de-virtualization bug. We check
2859	// if a non-pc-relative relocation in the code is pointing to (fptr - 1).
2860	if (IsToCode && ContainingBF && !Relocation::isPCRelative(Type: RType) &&
2861	(!ReferencedBF || (ReferencedBF->getAddress() != Address))) {
2862	if (const BinaryFunction *RogueBF =
2863	BC->getBinaryFunctionAtAddress(Address: Address + 1)) {
2864	// Do an extra check that the function was referenced previously.
2865	// It's a linear search, but it should rarely happen.
2866	auto CheckReloc = [&](const Relocation &Rel) {
2867	return Rel.Symbol == RogueBF->getSymbol() &&
2868	!Relocation::isPCRelative(Type: Rel.Type);
2869	};
2870	bool Found = llvm::any_of(
2871	Range: llvm::make_second_range(c&: ContainingBF->Relocations), P: CheckReloc);
2872
2873	if (Found) {
2874	BC->errs()
2875	<< "BOLT-WARNING: detected possible compiler de-virtualization "
2876	"bug: -1 addend used with non-pc-relative relocation against "
2877	<< formatv(Fmt: "function {0} in function {1}\n", Vals: *RogueBF,
2878	Vals&: *ContainingBF);
2879	return;
2880	}
2881	}
2882	}
2883
2884	if (ForceRelocation && !ReferencedBF) {
2885	// Create the relocation symbol if it's not defined in the binary.
2886	if (SymbolAddress == 0)
2887	ReferencedSymbol = BC->registerNameAtAddress(Name: SymbolName, Address: 0, Size: 0, Alignment: 0);
2888
2889	LLVM_DEBUG(
2890	dbgs() << "BOLT-DEBUG: forcing relocation against symbol "
2891	<< (ReferencedSymbol ? ReferencedSymbol->getName() : "<none>")
2892	<< " with addend " << Addend << '\n');
2893	} else if (ReferencedBF) {
2894	ReferencedSymbol = ReferencedBF->getSymbol();
2895	uint64_t RefFunctionOffset = 0;
2896
2897	// Adjust the point of reference to a code location inside a function.
2898	if (ReferencedBF->containsAddress(PC: Address, /*UseMaxSize = */ true)) {
2899	RefFunctionOffset = Address - ReferencedBF->getAddress();
2900	if (Relocation::isInstructionReference(Type: RType)) {
2901	// Instruction labels are created while disassembling so we just leave
2902	// the symbol empty for now. Since the extracted value is typically
2903	// unrelated to the referenced symbol (e.g., %pcrel_lo in RISC-V
2904	// references an instruction but the patched value references the low
2905	// bits of a data address), we set the extracted value to the symbol
2906	// address in order to be able to correctly reconstruct the reference
2907	// later.
2908	ReferencedSymbol = nullptr;
2909	ExtractedValue = Address;
2910	} else if (RefFunctionOffset) {
2911	if (ContainingBF && ContainingBF != ReferencedBF) {
2912	ReferencedSymbol =
2913	ReferencedBF->addEntryPointAtOffset(Offset: RefFunctionOffset);
2914	} else {
2915	ReferencedSymbol =
2916	ReferencedBF->getOrCreateLocalLabel(Address,
2917	/*CreatePastEnd =*/true);
2918
2919	// If ContainingBF != nullptr, it equals ReferencedBF (see
2920	// if-condition above) so we're handling a relocation from a function
2921	// to itself. RISC-V uses such relocations for branches, for example.
2922	// These should not be registered as externally references offsets.
2923	if (!ContainingBF)
2924	ReferencedBF->registerReferencedOffset(Offset: RefFunctionOffset);
2925	}
2926	if (opts::Verbosity > 1 &&
2927	BinarySection(*BC, RelocatedSection).isWritable())
2928	BC->errs()
2929	<< "BOLT-WARNING: writable reference into the middle of the "
2930	<< formatv(Fmt: "function {0} detected at address {1:x}\n",
2931	Vals&: *ReferencedBF, Vals: Rel.getOffset());
2932	}
2933	SymbolAddress = Address;
2934	Addend = 0;
2935	}
2936	LLVM_DEBUG({
2937	dbgs() << " referenced function " << *ReferencedBF;
2938	if (Address != ReferencedBF->getAddress())
2939	dbgs() << formatv(" at offset {0:x}", RefFunctionOffset);
2940	dbgs() << '\n';
2941	});
2942	} else {
2943	if (IsToCode && SymbolAddress) {
2944	// This can happen e.g. with PIC-style jump tables.
2945	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: no corresponding function for "
2946	"relocation against code\n");
2947	}
2948
2949	// In AArch64 there are zero reasons to keep a reference to the
2950	// "original" symbol plus addend. The original symbol is probably just a
2951	// section symbol. If we are here, this means we are probably accessing
2952	// data, so it is imperative to keep the original address.
2953	if (IsAArch64) {
2954	SymbolName = formatv(Fmt: "SYMBOLat{0:x}", Vals: Address);
2955	SymbolAddress = Address;
2956	Addend = 0;
2957	}
2958
2959	if (BinaryData *BD = BC->getBinaryDataContainingAddress(Address: SymbolAddress)) {
2960	// Note: this assertion is trying to check sanity of BinaryData objects
2961	// but AArch64 and RISCV has inferred and incomplete object locations
2962	// coming from GOT/TLS or any other non-trivial relocation (that requires
2963	// creation of sections and whose symbol address is not really what should
2964	// be encoded in the instruction). So we essentially disabled this check
2965	// for AArch64 and live with bogus names for objects.
2966	assert((IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
2967	BD->nameStartsWith(SymbolName) ||
2968	BD->nameStartsWith("PG" + SymbolName) ||
2969	(BD->nameStartsWith("ANONYMOUS") &&
2970	(BD->getSectionName().starts_with(".plt") ||
2971	BD->getSectionName().ends_with(".plt")))) &&
2972	"BOLT symbol names of all non-section relocations must match up "
2973	"with symbol names referenced in the relocation");
2974
2975	if (IsSectionRelocation)
2976	BC->markAmbiguousRelocations(BD&: *BD, Address);
2977
2978	ReferencedSymbol = BD->getSymbol();
2979	Addend += (SymbolAddress - BD->getAddress());
2980	SymbolAddress = BD->getAddress();
2981	assert(Address == SymbolAddress + Addend);
2982	} else {
2983	// These are mostly local data symbols but undefined symbols
2984	// in relocation sections can get through here too, from .plt.
2985	assert(
2986	(IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
2987	BC->getSectionNameForAddress(SymbolAddress)->starts_with(".plt")) &&
2988	"known symbols should not resolve to anonymous locals");
2989
2990	if (IsSectionRelocation) {
2991	ReferencedSymbol =
2992	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "SYMBOLat");
2993	} else {
2994	SymbolRef Symbol = *Rel.getSymbol();
2995	const uint64_t SymbolSize =
2996	IsAArch64 ? 0 : ELFSymbolRef(Symbol).getSize();
2997	const uint64_t SymbolAlignment = IsAArch64 ? 1 : Symbol.getAlignment();
2998	const uint32_t SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
2999	std::string Name;
3000	if (SymbolFlags & SymbolRef::SF_Global) {
3001	Name = SymbolName;
3002	} else {
3003	if (StringRef(SymbolName)
3004	.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix()))
3005	Name = NR.uniquify(Name: "PG" + SymbolName);
3006	else
3007	Name = NR.uniquify(Name: SymbolName);
3008	}
3009	ReferencedSymbol = BC->registerNameAtAddress(
3010	Name, Address: SymbolAddress, Size: SymbolSize, Alignment: SymbolAlignment, Flags: SymbolFlags);
3011	}
3012
3013	if (IsSectionRelocation) {
3014	BinaryData *BD = BC->getBinaryDataByName(Name: ReferencedSymbol->getName());
3015	BC->markAmbiguousRelocations(BD&: *BD, Address);
3016	}
3017	}
3018	}
3019
3020	auto checkMaxDataRelocations = [&]() {
3021	++NumDataRelocations;
3022	LLVM_DEBUG(if (opts::MaxDataRelocations &&
3023	NumDataRelocations + 1 == opts::MaxDataRelocations) {
3024	dbgs() << "BOLT-DEBUG: processing ending on data relocation "
3025	<< NumDataRelocations << ": ";
3026	printRelocationInfo(Rel, ReferencedSymbol->getName(), SymbolAddress,
3027	Addend, ExtractedValue);
3028	});
3029
3030	return (!opts::MaxDataRelocations ||
3031	NumDataRelocations < opts::MaxDataRelocations);
3032	};
3033
3034	if ((ReferencedSection && refersToReorderedSection(Section: ReferencedSection)) ||
3035	(opts::ForceToDataRelocations && checkMaxDataRelocations()) ||
3036	// RISC-V has ADD/SUB data-to-data relocations
3037	BC->isRISCV())
3038	ForceRelocation = true;
3039
3040	if (IsFromCode)
3041	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
3042	Addend, Value: ExtractedValue);
3043	else if (IsToCode || ForceRelocation)
3044	BC->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, Type: RType, Addend,
3045	Value: ExtractedValue);
3046	else
3047	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");
3048	}
3049
3050	static BinaryFunction *getInitFunctionIfStaticBinary(BinaryContext &BC) {
3051	// Workaround for https://github.com/llvm/llvm-project/issues/100096
3052	// ("[BOLT] GOT array pointer incorrectly rewritten"). In aarch64
3053	// static glibc binaries, the .init section's _init function pointer can
3054	// alias with a data pointer for the end of an array. GOT rewriting
3055	// currently can't detect this and updates the data pointer to the
3056	// moved _init, causing a runtime crash. Skipping _init on the other
3057	// hand should be harmless.
3058	if (!BC.IsStaticExecutable)
3059	return nullptr;
3060	const BinaryData *BD = BC.getBinaryDataByName(Name: "_init");
3061	if (!BD || BD->getSectionName() != ".init")
3062	return nullptr;
3063	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: skip _init in for GOT workaround.\n");
3064	return BC.getBinaryFunctionAtAddress(Address: BD->getAddress());
3065	}
3066
3067	void RewriteInstance::selectFunctionsToProcess() {
3068	// Extend the list of functions to process or skip from a file.
3069	auto populateFunctionNames = [](cl::opt<std::string> &FunctionNamesFile,
3070	cl::list<std::string> &FunctionNames) {
3071	if (FunctionNamesFile.empty())
3072	return;
3073	std::ifstream FuncsFile(FunctionNamesFile, std::ios::in);
3074	std::string FuncName;
3075	while (std::getline(is&: FuncsFile, str&: FuncName))
3076	FunctionNames.push_back(value: FuncName);
3077	};
3078	populateFunctionNames(opts::FunctionNamesFile, opts::ForceFunctionNames);
3079	populateFunctionNames(opts::SkipFunctionNamesFile, opts::SkipFunctionNames);
3080	populateFunctionNames(opts::FunctionNamesFileNR, opts::ForceFunctionNamesNR);
3081
3082	// Make a set of functions to process to speed up lookups.
3083	std::unordered_set<std::string> ForceFunctionsNR(
3084	opts::ForceFunctionNamesNR.begin(), opts::ForceFunctionNamesNR.end());
3085
3086	if ((!opts::ForceFunctionNames.empty() ||
3087	!opts::ForceFunctionNamesNR.empty()) &&
3088	!opts::SkipFunctionNames.empty()) {
3089	BC->errs()
3090	<< "BOLT-ERROR: cannot select functions to process and skip at the "
3091	"same time. Please use only one type of selection.\n";
3092	exit(status: 1);
3093	}
3094
3095	uint64_t LiteThresholdExecCount = 0;
3096	if (opts::LiteThresholdPct) {
3097	if (opts::LiteThresholdPct > 100)
3098	opts::LiteThresholdPct = 100;
3099
3100	std::vector<const BinaryFunction *> TopFunctions;
3101	for (auto &BFI : BC->getBinaryFunctions()) {
3102	const BinaryFunction &Function = BFI.second;
3103	if (ProfileReader->mayHaveProfileData(BF: Function))
3104	TopFunctions.push_back(x: &Function);
3105	}
3106	llvm::sort(
3107	C&: TopFunctions, Comp: [](const BinaryFunction *A, const BinaryFunction *B) {
3108	return A->getKnownExecutionCount() < B->getKnownExecutionCount();
3109	});
3110
3111	size_t Index = TopFunctions.size() * opts::LiteThresholdPct / 100;
3112	if (Index)
3113	--Index;
3114	LiteThresholdExecCount = TopFunctions[Index]->getKnownExecutionCount();
3115	BC->outs() << "BOLT-INFO: limiting processing to functions with at least "
3116	<< LiteThresholdExecCount << " invocations\n";
3117	}
3118	LiteThresholdExecCount = std::max(
3119	a: LiteThresholdExecCount, b: static_cast<uint64_t>(opts::LiteThresholdCount));
3120
3121	StringSet<> ReorderFunctionsUserSet;
3122	StringSet<> ReorderFunctionsLTOCommonSet;
3123	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
3124	std::vector<std::string> FunctionNames;
3125	BC->logBOLTErrorsAndQuitOnFatal(
3126	E: ReorderFunctions::readFunctionOrderFile(FunctionNames));
3127	for (const std::string &Function : FunctionNames) {
3128	ReorderFunctionsUserSet.insert(key: Function);
3129	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name: Function))
3130	ReorderFunctionsLTOCommonSet.insert(key: *LTOCommonName);
3131	}
3132	}
3133
3134	uint64_t NumFunctionsToProcess = 0;
3135	auto mustSkip = [&](const BinaryFunction &Function) {
3136	if (opts::MaxFunctions.getNumOccurrences() &&
3137	NumFunctionsToProcess >= opts::MaxFunctions)
3138	return true;
3139	for (std::string &Name : opts::SkipFunctionNames)
3140	if (Function.hasNameRegex(NameRegex: Name))
3141	return true;
3142
3143	return false;
3144	};
3145
3146	auto shouldProcess = [&](const BinaryFunction &Function) {
3147	if (mustSkip(Function))
3148	return false;
3149
3150	// If the list is not empty, only process functions from the list.
3151	if (!opts::ForceFunctionNames.empty() || !ForceFunctionsNR.empty()) {
3152	// Regex check (-funcs and -funcs-file options).
3153	for (std::string &Name : opts::ForceFunctionNames)
3154	if (Function.hasNameRegex(NameRegex: Name))
3155	return true;
3156
3157	// Non-regex check (-funcs-no-regex and -funcs-file-no-regex).
3158	for (const StringRef Name : Function.getNames())
3159	if (ForceFunctionsNR.count(x: Name.str()))
3160	return true;
3161
3162	return false;
3163	}
3164
3165	if (opts::Lite) {
3166	// Forcibly include functions specified in the -function-order file.
3167	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
3168	for (const StringRef Name : Function.getNames())
3169	if (ReorderFunctionsUserSet.contains(key: Name))
3170	return true;
3171	for (const StringRef Name : Function.getNames())
3172	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name))
3173	if (ReorderFunctionsLTOCommonSet.contains(key: *LTOCommonName))
3174	return true;
3175	}
3176
3177	if (ProfileReader && !ProfileReader->mayHaveProfileData(BF: Function))
3178	return false;
3179
3180	if (Function.getKnownExecutionCount() < LiteThresholdExecCount)
3181	return false;
3182	}
3183
3184	return true;
3185	};
3186
3187	if (BinaryFunction *Init = getInitFunctionIfStaticBinary(BC&: *BC))
3188	Init->setIgnored();
3189
3190	for (auto &BFI : BC->getBinaryFunctions()) {
3191	BinaryFunction &Function = BFI.second;
3192
3193	// Pseudo functions are explicitly marked by us not to be processed.
3194	if (Function.isPseudo()) {
3195	Function.IsIgnored = true;
3196	Function.HasExternalRefRelocations = true;
3197	continue;
3198	}
3199
3200	// Decide what to do with fragments after parent functions are processed.
3201	if (Function.isFragment())
3202	continue;
3203
3204	if (!shouldProcess(Function)) {
3205	if (opts::Verbosity >= 1) {
3206	BC->outs() << "BOLT-INFO: skipping processing " << Function
3207	<< " per user request\n";
3208	}
3209	Function.setIgnored();
3210	} else {
3211	++NumFunctionsToProcess;
3212	if (opts::MaxFunctions.getNumOccurrences() &&
3213	NumFunctionsToProcess == opts::MaxFunctions)
3214	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3215	}
3216	}
3217
3218	if (!BC->HasSplitFunctions)
3219	return;
3220
3221	// Fragment overrides:
3222	// - If the fragment must be skipped, then the parent must be skipped as well.
3223	// Otherwise, fragment should follow the parent function:
3224	// - if the parent is skipped, skip fragment,
3225	// - if the parent is processed, process the fragment(s) as well.
3226	for (auto &BFI : BC->getBinaryFunctions()) {
3227	BinaryFunction &Function = BFI.second;
3228	if (!Function.isFragment())
3229	continue;
3230	if (mustSkip(Function)) {
3231	for (BinaryFunction *Parent : Function.ParentFragments) {
3232	if (opts::Verbosity >= 1) {
3233	BC->outs() << "BOLT-INFO: skipping processing " << *Parent
3234	<< " together with fragment function\n";
3235	}
3236	Parent->setIgnored();
3237	--NumFunctionsToProcess;
3238	}
3239	Function.setIgnored();
3240	continue;
3241	}
3242
3243	bool IgnoredParent =
3244	llvm::any_of(Range&: Function.ParentFragments, P: [&](BinaryFunction *Parent) {
3245	return Parent->isIgnored();
3246	});
3247	if (IgnoredParent) {
3248	if (opts::Verbosity >= 1) {
3249	BC->outs() << "BOLT-INFO: skipping processing " << Function
3250	<< " together with parent function\n";
3251	}
3252	Function.setIgnored();
3253	} else {
3254	++NumFunctionsToProcess;
3255	if (opts::Verbosity >= 1) {
3256	BC->outs() << "BOLT-INFO: processing " << Function
3257	<< " as a sibling of non-ignored function\n";
3258	}
3259	if (opts::MaxFunctions && NumFunctionsToProcess == opts::MaxFunctions)
3260	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3261	}
3262	}
3263	}
3264
3265	void RewriteInstance::readDebugInfo() {
3266	NamedRegionTimer T("readDebugInfo", "read debug info", TimerGroupName,
3267	TimerGroupDesc, opts::TimeRewrite);
3268	if (!opts::UpdateDebugSections)
3269	return;
3270
3271	BC->preprocessDebugInfo();
3272	}
3273
3274	void RewriteInstance::preprocessProfileData() {
3275	if (!ProfileReader)
3276	return;
3277
3278	NamedRegionTimer T("preprocessprofile", "pre-process profile data",
3279	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3280
3281	BC->outs() << "BOLT-INFO: pre-processing profile using "
3282	<< ProfileReader->getReaderName() << '\n';
3283
3284	if (BAT->enabledFor(InputFile)) {
3285	BC->outs() << "BOLT-INFO: profile collection done on a binary already "
3286	"processed by BOLT\n";
3287	ProfileReader->setBAT(&*BAT);
3288	}
3289
3290	if (Error E = ProfileReader->preprocessProfile(BC&: *BC))
3291	report_error(Message: "cannot pre-process profile", E: std::move(E));
3292
3293	if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName() &&
3294	!opts::AllowStripped) {
3295	BC->errs()
3296	<< "BOLT-ERROR: input binary does not have local file symbols "
3297	"but profile data includes function names with embedded file "
3298	"names. It appears that the input binary was stripped while a "
3299	"profiled binary was not. If you know what you are doing and "
3300	"wish to proceed, use -allow-stripped option.\n";
3301	exit(status: 1);
3302	}
3303	}
3304
3305	void RewriteInstance::initializeMetadataManager() {
3306	if (BC->IsLinuxKernel)
3307	MetadataManager.registerRewriter(Rewriter: createLinuxKernelRewriter(*BC));
3308
3309	MetadataManager.registerRewriter(Rewriter: createBuildIDRewriter(*BC));
3310
3311	MetadataManager.registerRewriter(Rewriter: createPseudoProbeRewriter(*BC));
3312
3313	MetadataManager.registerRewriter(Rewriter: createSDTRewriter(*BC));
3314	}
3315
3316	void RewriteInstance::processSectionMetadata() {
3317	NamedRegionTimer T("processmetadata-section", "process section metadata",
3318	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3319	initializeMetadataManager();
3320
3321	MetadataManager.runSectionInitializers();
3322	}
3323
3324	void RewriteInstance::processMetadataPreCFG() {
3325	NamedRegionTimer T("processmetadata-precfg", "process metadata pre-CFG",
3326	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3327	MetadataManager.runInitializersPreCFG();
3328
3329	processProfileDataPreCFG();
3330	}
3331
3332	void RewriteInstance::processMetadataPostCFG() {
3333	NamedRegionTimer T("processmetadata-postcfg", "process metadata post-CFG",
3334	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3335	MetadataManager.runInitializersPostCFG();
3336	}
3337
3338	void RewriteInstance::processProfileDataPreCFG() {
3339	if (!ProfileReader)
3340	return;
3341
3342	NamedRegionTimer T("processprofile-precfg", "process profile data pre-CFG",
3343	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3344
3345	if (Error E = ProfileReader->readProfilePreCFG(BC&: *BC))
3346	report_error(Message: "cannot read profile pre-CFG", E: std::move(E));
3347	}
3348
3349	void RewriteInstance::processProfileData() {
3350	if (!ProfileReader)
3351	return;
3352
3353	NamedRegionTimer T("processprofile", "process profile data", TimerGroupName,
3354	TimerGroupDesc, opts::TimeRewrite);
3355
3356	if (Error E = ProfileReader->readProfile(BC&: *BC))
3357	report_error(Message: "cannot read profile", E: std::move(E));
3358
3359	if (opts::PrintProfile || opts::PrintAll) {
3360	for (auto &BFI : BC->getBinaryFunctions()) {
3361	BinaryFunction &Function = BFI.second;
3362	if (Function.empty())
3363	continue;
3364
3365	Function.print(OS&: BC->outs(), Annotation: "after attaching profile");
3366	}
3367	}
3368
3369	if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) {
3370	YAMLProfileWriter PW(opts::SaveProfile);
3371	PW.writeProfile(RI: *this);
3372	}
3373	if (opts::AggregateOnly &&
3374	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML &&
3375	!BAT->enabledFor(InputFile)) {
3376	YAMLProfileWriter PW(opts::OutputFilename);
3377	PW.writeProfile(RI: *this);
3378	}
3379
3380	// Release memory used by profile reader.
3381	ProfileReader.reset();
3382
3383	if (opts::AggregateOnly) {
3384	PrintProgramStats PPS(&*BAT);
3385	BC->logBOLTErrorsAndQuitOnFatal(E: PPS.runOnFunctions(BC&: *BC));
3386	TimerGroup::printAll(OS&: outs());
3387	exit(status: 0);
3388	}
3389	}
3390
3391	void RewriteInstance::disassembleFunctions() {
3392	NamedRegionTimer T("disassembleFunctions", "disassemble functions",
3393	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3394	for (auto &BFI : BC->getBinaryFunctions()) {
3395	BinaryFunction &Function = BFI.second;
3396
3397	ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
3398	if (!FunctionData) {
3399	BC->errs() << "BOLT-ERROR: corresponding section is non-executable or "
3400	<< "empty for function " << Function << '\n';
3401	exit(status: 1);
3402	}
3403
3404	// Treat zero-sized functions as non-simple ones.
3405	if (Function.getSize() == 0) {
3406	Function.setSimple(false);
3407	continue;
3408	}
3409
3410	// Offset of the function in the file.
3411	const auto *FileBegin =
3412	reinterpret_cast<const uint8_t *>(InputFile->getData().data());
3413	Function.setFileOffset(FunctionData->begin() - FileBegin);
3414
3415	if (!shouldDisassemble(BF: Function)) {
3416	NamedRegionTimer T("scan", "scan functions", "buildfuncs",
3417	"Scan Binary Functions", opts::TimeBuild);
3418	Function.scanExternalRefs();
3419	Function.setSimple(false);
3420	continue;
3421	}
3422
3423	bool DisasmFailed{false};
3424	handleAllErrors(E: Function.disassemble(), Handlers: [&](const BOLTError &E) {
3425	DisasmFailed = true;
3426	if (E.isFatal()) {
3427	E.log(OS&: BC->errs());
3428	exit(status: 1);
3429	}
3430	if (opts::processAllFunctions()) {
3431	BC->errs() << BC->generateBugReportMessage(
3432	Message: "function cannot be properly disassembled. "
3433	"Unable to continue in relocation mode.",
3434	Function);
3435	exit(status: 1);
3436	}
3437	if (opts::Verbosity >= 1)
3438	BC->outs() << "BOLT-INFO: could not disassemble function " << Function
3439	<< ". Will ignore.\n";
3440	// Forcefully ignore the function.
3441	Function.scanExternalRefs();
3442	Function.setIgnored();
3443	});
3444
3445	if (DisasmFailed)
3446	continue;
3447
3448	if (opts::PrintAll || opts::PrintDisasm)
3449	Function.print(OS&: BC->outs(), Annotation: "after disassembly");
3450	}
3451
3452	BC->processInterproceduralReferences();
3453	BC->populateJumpTables();
3454
3455	for (auto &BFI : BC->getBinaryFunctions()) {
3456	BinaryFunction &Function = BFI.second;
3457
3458	if (!shouldDisassemble(BF: Function))
3459	continue;
3460
3461	Function.postProcessEntryPoints();
3462	Function.postProcessJumpTables();
3463	}
3464
3465	BC->clearJumpTableTempData();
3466	BC->adjustCodePadding();
3467
3468	for (auto &BFI : BC->getBinaryFunctions()) {
3469	BinaryFunction &Function = BFI.second;
3470
3471	if (!shouldDisassemble(BF: Function))
3472	continue;
3473
3474	if (!Function.isSimple()) {
3475	assert((!BC->HasRelocations || Function.getSize() == 0 ||
3476	Function.hasIndirectTargetToSplitFragment()) &&
3477	"unexpected non-simple function in relocation mode");
3478	continue;
3479	}
3480
3481	// Fill in CFI information for this function
3482	if (!Function.trapsOnEntry() && !CFIRdWrt->fillCFIInfoFor(Function)) {
3483	if (BC->HasRelocations) {
3484	BC->errs() << BC->generateBugReportMessage(Message: "unable to fill CFI.",
3485	Function);
3486	exit(status: 1);
3487	} else {
3488	BC->errs() << "BOLT-WARNING: unable to fill CFI for function "
3489	<< Function << ". Skipping.\n";
3490	Function.setSimple(false);
3491	continue;
3492	}
3493	}
3494
3495	// Parse LSDA.
3496	if (Function.getLSDAAddress() != 0 &&
3497	!BC->getFragmentsToSkip().count(x: &Function)) {
3498	ErrorOr<BinarySection &> LSDASection =
3499	BC->getSectionForAddress(Address: Function.getLSDAAddress());
3500	check_error(EC: LSDASection.getError(), Message: "failed to get LSDA section");
3501	ArrayRef<uint8_t> LSDAData = ArrayRef<uint8_t>(
3502	LSDASection->getData(), LSDASection->getContents().size());
3503	BC->logBOLTErrorsAndQuitOnFatal(
3504	E: Function.parseLSDA(LSDAData, LSDAAddress: LSDASection->getAddress()));
3505	}
3506	}
3507	}
3508
3509	void RewriteInstance::buildFunctionsCFG() {
3510	NamedRegionTimer T("buildCFG", "buildCFG", "buildfuncs",
3511	"Build Binary Functions", opts::TimeBuild);
3512
3513	// Create annotation indices to allow lock-free execution
3514	BC->MIB->getOrCreateAnnotationIndex(Name: "JTIndexReg");
3515	BC->MIB->getOrCreateAnnotationIndex(Name: "NOP");
3516
3517	ParallelUtilities::WorkFuncWithAllocTy WorkFun =
3518	[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {
3519	bool HadErrors{false};
3520	handleAllErrors(E: BF.buildCFG(AllocId), Handlers: [&](const BOLTError &E) {
3521	if (!E.getMessage().empty())
3522	E.log(OS&: BC->errs());
3523	if (E.isFatal())
3524	exit(status: 1);
3525	HadErrors = true;
3526	});
3527
3528	if (HadErrors)
3529	return;
3530
3531	if (opts::PrintAll) {
3532	auto L = BC->scopeLock();
3533	BF.print(OS&: BC->outs(), Annotation: "while building cfg");
3534	}
3535	};
3536
3537	ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
3538	return !shouldDisassemble(BF) || !BF.isSimple();
3539	};
3540
3541	ParallelUtilities::runOnEachFunctionWithUniqueAllocId(
3542	BC&: *BC, SchedPolicy: ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFunction: WorkFun,
3543	SkipPredicate, LogName: "disassembleFunctions-buildCFG",
3544	/*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll);
3545
3546	BC->postProcessSymbolTable();
3547	}
3548
3549	void RewriteInstance::postProcessFunctions() {
3550	// We mark fragments as non-simple here, not during disassembly,
3551	// So we can build their CFGs.
3552	BC->skipMarkedFragments();
3553	BC->clearFragmentsToSkip();
3554
3555	BC->TotalScore = 0;
3556	BC->SumExecutionCount = 0;
3557	for (auto &BFI : BC->getBinaryFunctions()) {
3558	BinaryFunction &Function = BFI.second;
3559
3560	// Set function as non-simple if it has dynamic relocations
3561	// in constant island, we don't want this function to be optimized
3562	// e.g. function splitting is unsupported.
3563	if (Function.hasDynamicRelocationAtIsland())
3564	Function.setSimple(false);
3565
3566	if (Function.empty())
3567	continue;
3568
3569	Function.postProcessCFG();
3570
3571	if (opts::PrintAll || opts::PrintCFG)
3572	Function.print(OS&: BC->outs(), Annotation: "after building cfg");
3573
3574	if (opts::DumpDotAll)
3575	Function.dumpGraphForPass(Annotation: "00_build-cfg");
3576
3577	if (opts::PrintLoopInfo) {
3578	Function.calculateLoopInfo();
3579	Function.printLoopInfo(OS&: BC->outs());
3580	}
3581
3582	BC->TotalScore += Function.getFunctionScore();
3583	BC->SumExecutionCount += Function.getKnownExecutionCount();
3584	}
3585
3586	if (opts::PrintGlobals) {
3587	BC->outs() << "BOLT-INFO: Global symbols:\n";
3588	BC->printGlobalSymbols(OS&: BC->outs());
3589	}
3590	}
3591
3592	void RewriteInstance::runOptimizationPasses() {
3593	NamedRegionTimer T("runOptimizationPasses", "run optimization passes",
3594	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3595	BC->logBOLTErrorsAndQuitOnFatal(E: BinaryFunctionPassManager::runAllPasses(BC&: *BC));
3596	}
3597
3598	void RewriteInstance::runBinaryAnalyses() {
3599	NamedRegionTimer T("runBinaryAnalyses", "run binary analysis passes",
3600	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3601	BinaryFunctionPassManager Manager(*BC);
3602	// FIXME: add a pass that warns about which functions do not have CFG,
3603	// and therefore, analysis is most likely to be less accurate.
3604	using GSK = opts::GadgetScannerKind;
3605	using PAuthScanner = PAuthGadgetScanner::Analysis;
3606
3607	// If no command line option was given, act as if "all" was specified.
3608	bool RunAll = !opts::GadgetScannersToRun.getBits() ||
3609	opts::GadgetScannersToRun.isSet(V: GSK::GS_ALL);
3610
3611	if (RunAll || opts::GadgetScannersToRun.isSet(V: GSK::GS_PAUTH)) {
3612	Manager.registerPass(
3613	Pass: std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/args: false));
3614	} else if (RunAll || opts::GadgetScannersToRun.isSet(V: GSK::GS_PACRET)) {
3615	Manager.registerPass(
3616	Pass: std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/args: true));
3617	}
3618
3619	BC->logBOLTErrorsAndQuitOnFatal(E: Manager.runPasses());
3620	}
3621
3622	void RewriteInstance::preregisterSections() {
3623	// Preregister sections before emission to set their order in the output.
3624	const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true,
3625	/*IsText*/ false,
3626	/*IsAllocatable*/ true);
3627	if (BinarySection *EHFrameSection = getSection(Name: getEHFrameSectionName())) {
3628	// New .eh_frame.
3629	BC->registerOrUpdateSection(Name: getNewSecPrefix() + getEHFrameSectionName(),
3630	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3631	// Fully register a relocatable copy of the original .eh_frame.
3632	BC->registerSection(SectionName: ".relocated.eh_frame", OriginalSection: *EHFrameSection);
3633	}
3634	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".gcc_except_table",
3635	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3636	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata", ELFType: ELF::SHT_PROGBITS,
3637	ELFFlags: ROFlags);
3638	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata.cold",
3639	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3640	}
3641
3642	void RewriteInstance::emitAndLink() {
3643	NamedRegionTimer T("emitAndLink", "emit and link", TimerGroupName,
3644	TimerGroupDesc, opts::TimeRewrite);
3645
3646	SmallString<0> ObjectBuffer;
3647	raw_svector_ostream OS(ObjectBuffer);
3648
3649	// Implicitly MCObjectStreamer takes ownership of MCAsmBackend (MAB)
3650	// and MCCodeEmitter (MCE). ~MCObjectStreamer() will delete these
3651	// two instances.
3652	std::unique_ptr<MCStreamer> Streamer = BC->createStreamer(OS);
3653
3654	if (EHFrameSection) {
3655	if (opts::UseOldText || opts::StrictMode) {
3656	// The section is going to be regenerated from scratch.
3657	// Empty the contents, but keep the section reference.
3658	EHFrameSection->clearContents();
3659	} else {
3660	// Make .eh_frame relocatable.
3661	relocateEHFrameSection();
3662	}
3663	}
3664
3665	emitBinaryContext(Streamer&: *Streamer, BC&: *BC, OrgSecPrefix: getOrgSecPrefix());
3666
3667	Streamer->finish();
3668	if (Streamer->getContext().hadError()) {
3669	BC->errs() << "BOLT-ERROR: Emission failed.\n";
3670	exit(status: 1);
3671	}
3672
3673	if (opts::KeepTmp) {
3674	SmallString<128> OutObjectPath;
3675	sys::fs::getPotentiallyUniqueTempFileName(Prefix: "output", Suffix: "o", ResultPath&: OutObjectPath);
3676	std::error_code EC;
3677	raw_fd_ostream FOS(OutObjectPath, EC);
3678	check_error(EC, Message: "cannot create output object file");
3679	FOS << ObjectBuffer;
3680	BC->outs()
3681	<< "BOLT-INFO: intermediary output object file saved for debugging "
3682	"purposes: "
3683	<< OutObjectPath << "\n";
3684	}
3685
3686	ErrorOr<BinarySection &> TextSection =
3687	BC->getUniqueSectionByName(SectionName: BC->getMainCodeSectionName());
3688	if (BC->HasRelocations && TextSection)
3689	BC->renameSection(Section&: *TextSection,
3690	NewName: getOrgSecPrefix() + BC->getMainCodeSectionName());
3691
3692	//////////////////////////////////////////////////////////////////////////////
3693	// Assign addresses to new sections.
3694	//////////////////////////////////////////////////////////////////////////////
3695
3696	// Get output object as ObjectFile.
3697	std::unique_ptr<MemoryBuffer> ObjectMemBuffer =
3698	MemoryBuffer::getMemBuffer(InputData: ObjectBuffer, BufferName: "in-memory object file", RequiresNullTerminator: false);
3699
3700	auto EFMM = std::make_unique<ExecutableFileMemoryManager>(args&: *BC);
3701	EFMM->setNewSecPrefix(getNewSecPrefix());
3702	EFMM->setOrgSecPrefix(getOrgSecPrefix());
3703
3704	Linker = std::make_unique<JITLinkLinker>(args&: *BC, args: std::move(EFMM));
3705	Linker->loadObject(Obj: ObjectMemBuffer->getMemBufferRef(),
3706	MapSections: [this](auto MapSection) { mapFileSections(MapSection); });
3707
3708	// Update output addresses based on the new section map and
3709	// layout. Only do this for the object created by ourselves.
3710	updateOutputValues(Linker: *Linker);
3711
3712	if (opts::UpdateDebugSections) {
3713	DebugInfoRewriter->updateLineTableOffsets(
3714	Asm: static_cast<MCObjectStreamer &>(*Streamer).getAssembler());
3715	}
3716
3717	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {
3718	StartLinkingRuntimeLib = true;
3719	RtLibrary->link(BC&: *BC, ToolPath, Linker&: *Linker, MapSections: [this](auto MapSection) {
3720	// Map newly registered sections.
3721	this->mapAllocatableSections(MapSection);
3722	});
3723	}
3724
3725	// Once the code is emitted, we can rename function sections to actual
3726	// output sections and de-register sections used for emission.
3727	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
3728	ErrorOr<BinarySection &> Section = Function->getCodeSection();
3729	if (Section &&
3730	(Function->getImageAddress() == 0 || Function->getImageSize() == 0))
3731	continue;
3732
3733	// Restore origin section for functions that were emitted or supposed to
3734	// be emitted to patch sections.
3735	if (Section)
3736	BC->deregisterSection(Section&: *Section);
3737	assert(Function->getOriginSectionName() && "expected origin section");
3738	Function->CodeSectionName = Function->getOriginSectionName()->str();
3739	for (const FunctionFragment &FF :
3740	Function->getLayout().getSplitFragments()) {
3741	if (ErrorOr<BinarySection &> ColdSection =
3742	Function->getCodeSection(Fragment: FF.getFragmentNum()))
3743	BC->deregisterSection(Section&: *ColdSection);
3744	}
3745	if (Function->getLayout().isSplit())
3746	Function->setColdCodeSectionName(getBOLTTextSectionName());
3747	}
3748
3749	if (opts::PrintCacheMetrics) {
3750	BC->outs() << "BOLT-INFO: cache metrics after emitting functions:\n";
3751	CacheMetrics::printAll(OS&: BC->outs(), BinaryFunctions: BC->getSortedFunctions());
3752	}
3753	}
3754
3755	void RewriteInstance::finalizeMetadataPreEmit() {
3756	NamedRegionTimer T("finalizemetadata-preemit", "finalize metadata pre-emit",
3757	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3758	MetadataManager.runFinalizersPreEmit();
3759	}
3760
3761	void RewriteInstance::updateMetadata() {
3762	NamedRegionTimer T("updatemetadata-postemit", "update metadata post-emit",
3763	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3764	MetadataManager.runFinalizersAfterEmit();
3765
3766	if (opts::UpdateDebugSections) {
3767	NamedRegionTimer T("updateDebugInfo", "update debug info", TimerGroupName,
3768	TimerGroupDesc, opts::TimeRewrite);
3769	DebugInfoRewriter->updateDebugInfo();
3770	}
3771
3772	if (opts::WriteBoltInfoSection)
3773	addBoltInfoSection();
3774	}
3775
3776	void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
3777	BC->deregisterUnusedSections();
3778
3779	// If no new .eh_frame was written, remove relocated original .eh_frame.
3780	BinarySection *RelocatedEHFrameSection =
3781	getSection(Name: ".relocated" + getEHFrameSectionName());
3782	if (RelocatedEHFrameSection && RelocatedEHFrameSection->hasValidSectionID()) {
3783	BinarySection *NewEHFrameSection =
3784	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
3785	if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) {
3786	// JITLink will still have to process relocations for the section, hence
3787	// we need to assign it the address that wouldn't result in relocation
3788	// processing failure.
3789	MapSection(*RelocatedEHFrameSection, NextAvailableAddress);
3790	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
3791	}
3792	}
3793
3794	mapCodeSections(MapSection);
3795
3796	// Map the rest of the sections.
3797	mapAllocatableSections(MapSection);
3798
3799	if (!BC->BOLTReserved.empty()) {
3800	const uint64_t AllocatedSize =
3801	NextAvailableAddress - BC->BOLTReserved.start();
3802	if (BC->BOLTReserved.size() < AllocatedSize) {
3803	BC->errs() << "BOLT-ERROR: reserved space (" << BC->BOLTReserved.size()
3804	<< " byte" << (BC->BOLTReserved.size() == 1 ? "" : "s")
3805	<< ") is smaller than required for new allocations ("
3806	<< AllocatedSize << " bytes)\n";
3807	exit(status: 1);
3808	}
3809	}
3810	}
3811
3812	std::vector<BinarySection *> RewriteInstance::getCodeSections() {
3813	std::vector<BinarySection *> CodeSections;
3814	for (BinarySection &Section : BC->textSections())
3815	if (Section.hasValidSectionID())
3816	CodeSections.emplace_back(args: &Section);
3817
3818	auto compareSections = [&](const BinarySection *A, const BinarySection *B) {
3819	// If both A and B have names starting with ".text.cold", then
3820	// - if opts::HotFunctionsAtEnd is true, we want order
3821	// ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
3822	// - if opts::HotFunctionsAtEnd is false, we want order
3823	// ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
3824	if (A->getName().starts_with(Prefix: BC->getColdCodeSectionName()) &&
3825	B->getName().starts_with(Prefix: BC->getColdCodeSectionName())) {
3826	if (A->getName().size() != B->getName().size())
3827	return (opts::HotFunctionsAtEnd)
3828	? (A->getName().size() > B->getName().size())
3829	: (A->getName().size() < B->getName().size());
3830	return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName())
3831	: (A->getName() < B->getName());
3832	}
3833
3834	// Place movers before anything else.
3835	if (A->getName() == BC->getHotTextMoverSectionName())
3836	return true;
3837	if (B->getName() == BC->getHotTextMoverSectionName())
3838	return false;
3839
3840	// Depending on opts::HotFunctionsAtEnd, place main and warm sections in
3841	// order.
3842	if (opts::HotFunctionsAtEnd) {
3843	if (B->getName() == BC->getMainCodeSectionName())
3844	return true;
3845	if (A->getName() == BC->getMainCodeSectionName())
3846	return false;
3847	return (B->getName() == BC->getWarmCodeSectionName());
3848	} else {
3849	if (A->getName() == BC->getMainCodeSectionName())
3850	return true;
3851	if (B->getName() == BC->getMainCodeSectionName())
3852	return false;
3853	return (A->getName() == BC->getWarmCodeSectionName());
3854	}
3855	};
3856
3857	// Determine the order of sections.
3858	llvm::stable_sort(Range&: CodeSections, C: compareSections);
3859
3860	return CodeSections;
3861	}
3862
3863	void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
3864	if (!BC->HasRelocations) {
3865	mapCodeSectionsInPlace(MapSection);
3866	return;
3867	}
3868
3869	// Map sections for functions with pre-assigned addresses.
3870	for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
3871	const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
3872	if (!OutputAddress)
3873	continue;
3874
3875	ErrorOr<BinarySection &> FunctionSection =
3876	InjectedFunction->getCodeSection();
3877	assert(FunctionSection && "function should have section");
3878	FunctionSection->setOutputAddress(OutputAddress);
3879	MapSection(*FunctionSection, OutputAddress);
3880	InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
3881	InjectedFunction->setImageSize(FunctionSection->getOutputSize());
3882	}
3883
3884	// Populate the list of sections to be allocated.
3885	std::vector<BinarySection *> CodeSections = getCodeSections();
3886
3887	// Remove sections that were pre-allocated (patch sections).
3888	llvm::erase_if(C&: CodeSections, P: [](BinarySection *Section) {
3889	return Section->getOutputAddress();
3890	});
3891	LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
3892	for (const BinarySection *Section : CodeSections) dbgs()
3893	<< Section->getName() << '\n';);
3894
3895	uint64_t PaddingSize = 0; // size of padding required at the end
3896
3897	// Allocate sections starting at a given Address.
3898	auto allocateAt = [&](uint64_t Address) {
3899	const char *LastNonColdSectionName = BC->HasWarmSection
3900	? BC->getWarmCodeSectionName()
3901	: BC->getMainCodeSectionName();
3902	for (BinarySection *Section : CodeSections) {
3903	Address = alignTo(Value: Address, Align: Section->getAlignment());
3904	Section->setOutputAddress(Address);
3905	Address += Section->getOutputSize();
3906
3907	// Hugify: Additional huge page from right side due to
3908	// weird ASLR mapping addresses (4KB aligned)
3909	if (opts::Hugify && !BC->HasFixedLoadAddress &&
3910	Section->getName() == LastNonColdSectionName)
3911	Address = alignTo(Value: Address, Align: Section->getAlignment());
3912	}
3913
3914	// Make sure we allocate enough space for huge pages.
3915	ErrorOr<BinarySection &> TextSection =
3916	BC->getUniqueSectionByName(SectionName: LastNonColdSectionName);
3917	if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
3918	uint64_t HotTextEnd =
3919	TextSection->getOutputAddress() + TextSection->getOutputSize();
3920	HotTextEnd = alignTo(Value: HotTextEnd, Align: BC->PageAlign);
3921	if (HotTextEnd > Address) {
3922	PaddingSize = HotTextEnd - Address;
3923	Address = HotTextEnd;
3924	}
3925	}
3926	return Address;
3927	};
3928
3929	// Try to allocate sections before the \p Address and return an address for
3930	// the allocation of the first section, or 0 if [0, Address) range is not
3931	// big enough to fit all sections.
3932	auto allocateBefore = [&](uint64_t Address) -> uint64_t {
3933	for (BinarySection *Section : llvm::reverse(C&: CodeSections)) {
3934	if (Section->getOutputSize() > Address)
3935	return 0;
3936	Address -= Section->getOutputSize();
3937	Address = alignDown(Value: Address, Align: Section->getAlignment());
3938	Section->setOutputAddress(Address);
3939	}
3940	return Address;
3941	};
3942
3943	// Check if we can fit code in the original .text
3944	bool AllocationDone = false;
3945	if (opts::UseOldText) {
3946	uint64_t StartAddress;
3947	uint64_t EndAddress;
3948	if (opts::HotFunctionsAtEnd) {
3949	EndAddress = BC->OldTextSectionAddress + BC->OldTextSectionSize;
3950	StartAddress = allocateBefore(EndAddress);
3951	} else {
3952	StartAddress = BC->OldTextSectionAddress;
3953	EndAddress = allocateAt(BC->OldTextSectionAddress);
3954	}
3955
3956	const uint64_t CodeSize = EndAddress - StartAddress;
3957	if (CodeSize <= BC->OldTextSectionSize) {
3958	BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
3959	<< Twine::utohexstr(Val: opts::AlignText) << " alignment";
3960	if (StartAddress != BC->OldTextSectionAddress)
3961	BC->outs() << " at 0x" << Twine::utohexstr(Val: StartAddress);
3962	BC->outs() << '\n';
3963	AllocationDone = true;
3964	} else {
3965	BC->errs() << "BOLT-WARNING: original .text too small to fit the new code"
3966	<< " using 0x" << Twine::utohexstr(Val: opts::AlignText)
3967	<< " alignment. " << CodeSize << " bytes needed, have "
3968	<< BC->OldTextSectionSize << " bytes available.\n";
3969	opts::UseOldText = false;
3970	}
3971	}
3972
3973	if (!AllocationDone)
3974	NextAvailableAddress = allocateAt(NextAvailableAddress);
3975
3976	// Do the mapping for ORC layer based on the allocation.
3977	for (BinarySection *Section : CodeSections) {
3978	LLVM_DEBUG(dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
3979	<< Twine::utohexstr(Section->getAllocAddress())
3980	<< " to 0x"
3981	<< Twine::utohexstr(Section->getOutputAddress()) << '\n');
3982	MapSection(*Section, Section->getOutputAddress());
3983	Section->setOutputFileOffset(
3984	getFileOffsetForAddress(Address: Section->getOutputAddress()));
3985	}
3986
3987	// Check if we need to insert a padding section for hot text.
3988	if (PaddingSize && !opts::UseOldText)
3989	BC->outs() << "BOLT-INFO: padding code to 0x"
3990	<< Twine::utohexstr(Val: NextAvailableAddress)
3991	<< " to accommodate hot text\n";
3992	}
3993
3994	void RewriteInstance::mapCodeSectionsInPlace(
3995	BOLTLinker::SectionMapper MapSection) {
3996	// Processing in non-relocation mode.
3997	uint64_t NewTextSectionStartAddress = NextAvailableAddress;
3998
3999	for (auto &BFI : BC->getBinaryFunctions()) {
4000	BinaryFunction &Function = BFI.second;
4001	if (!Function.isEmitted())
4002	continue;
4003
4004	ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();
4005	assert(FuncSection && "cannot find section for function");
4006	FuncSection->setOutputAddress(Function.getAddress());
4007	LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
4008	<< Twine::utohexstr(FuncSection->getAllocAddress())
4009	<< " to 0x" << Twine::utohexstr(Function.getAddress())
4010	<< '\n');
4011	MapSection(*FuncSection, Function.getAddress());
4012	Function.setImageAddress(FuncSection->getAllocAddress());
4013	Function.setImageSize(FuncSection->getOutputSize());
4014	assert(Function.getImageSize() <= Function.getMaxSize() &&
4015	"Unexpected large function");
4016
4017	if (!Function.isSplit())
4018	continue;
4019
4020	assert(Function.getLayout().isHotColdSplit() &&
4021	"Cannot allocate more than two fragments per function in "
4022	"non-relocation mode.");
4023
4024	FunctionFragment &FF =
4025	Function.getLayout().getFragment(Num: FragmentNum::cold());
4026	ErrorOr<BinarySection &> ColdSection =
4027	Function.getCodeSection(Fragment: FF.getFragmentNum());
4028	assert(ColdSection && "cannot find section for cold part");
4029	// Cold fragments are aligned at 16 bytes.
4030	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 16);
4031	FF.setAddress(NextAvailableAddress);
4032	FF.setImageAddress(ColdSection->getAllocAddress());
4033	FF.setImageSize(ColdSection->getOutputSize());
4034	FF.setFileOffset(getFileOffsetForAddress(Address: NextAvailableAddress));
4035	ColdSection->setOutputAddress(FF.getAddress());
4036
4037	LLVM_DEBUG(
4038	dbgs() << formatv(
4039	"BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n",
4040	FF.getImageAddress(), FF.getAddress(), FF.getImageSize()));
4041	MapSection(*ColdSection, FF.getAddress());
4042
4043	NextAvailableAddress += FF.getImageSize();
4044	}
4045
4046	// Add the new text section aggregating all existing code sections.
4047	// This is pseudo-section that serves a purpose of creating a corresponding
4048	// entry in section header table.
4049	const uint64_t NewTextSectionSize =
4050	NextAvailableAddress - NewTextSectionStartAddress;
4051	if (NewTextSectionSize) {
4052	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
4053	/*IsText=*/true,
4054	/*IsAllocatable=*/true);
4055	BinarySection &Section =
4056	BC->registerOrUpdateSection(Name: getBOLTTextSectionName(),
4057	ELFType: ELF::SHT_PROGBITS,
4058	ELFFlags: Flags,
4059	/*Data=*/nullptr,
4060	Size: NewTextSectionSize,
4061	Alignment: 16);
4062	Section.setOutputAddress(NewTextSectionStartAddress);
4063	Section.setOutputFileOffset(
4064	getFileOffsetForAddress(Address: NewTextSectionStartAddress));
4065	}
4066	}
4067
4068	void RewriteInstance::mapAllocatableSections(
4069	BOLTLinker::SectionMapper MapSection) {
4070
4071	if (opts::UseOldText || opts::StrictMode) {
4072	auto tryRewriteSection = [&](BinarySection &OldSection,
4073	BinarySection &NewSection) {
4074	if (OldSection.getSize() < NewSection.getOutputSize())
4075	return;
4076
4077	BC->outs() << "BOLT-INFO: rewriting " << OldSection.getName()
4078	<< " in-place\n";
4079
4080	NewSection.setOutputAddress(OldSection.getAddress());
4081	NewSection.setOutputFileOffset(OldSection.getInputFileOffset());
4082	MapSection(NewSection, OldSection.getAddress());
4083
4084	// Pad contents with zeros.
4085	NewSection.addPadding(PaddingSize: OldSection.getSize() - NewSection.getOutputSize());
4086
4087	// Prevent the original section name from appearing in the section header
4088	// table.
4089	OldSection.setAnonymous(true);
4090	};
4091
4092	if (EHFrameSection) {
4093	BinarySection *NewEHFrameSection =
4094	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
4095	assert(NewEHFrameSection && "New contents expected for .eh_frame");
4096	tryRewriteSection(*EHFrameSection, *NewEHFrameSection);
4097	}
4098	BinarySection *EHSection = getSection(Name: ".gcc_except_table");
4099	BinarySection *NewEHSection =
4100	getSection(Name: getNewSecPrefix() + ".gcc_except_table");
4101	if (EHSection) {
4102	assert(NewEHSection && "New contents expected for .gcc_except_table");
4103	tryRewriteSection(*EHSection, *NewEHSection);
4104	}
4105	}
4106
4107	// Allocate read-only sections first, then writable sections.
4108	enum : uint8_t { ST_READONLY, ST_READWRITE };
4109	for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) {
4110	const uint64_t LastNextAvailableAddress = NextAvailableAddress;
4111	if (SType == ST_READWRITE) {
4112	// Align R+W segment to regular page size
4113	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->RegularPageSize);
4114	NewWritableSegmentAddress = NextAvailableAddress;
4115	}
4116
4117	for (BinarySection &Section : BC->allocatableSections()) {
4118	if (Section.isLinkOnly())
4119	continue;
4120
4121	if (!Section.hasValidSectionID())
4122	continue;
4123
4124	if (Section.isWritable() == (SType == ST_READONLY))
4125	continue;
4126
4127	if (Section.getOutputAddress()) {
4128	LLVM_DEBUG({
4129	dbgs() << "BOLT-DEBUG: section " << Section.getName()
4130	<< " is already mapped at 0x"
4131	<< Twine::utohexstr(Section.getOutputAddress()) << '\n';
4132	});
4133	continue;
4134	}
4135
4136	if (Section.hasSectionRef()) {
4137	LLVM_DEBUG({
4138	dbgs() << "BOLT-DEBUG: mapping original section " << Section.getName()
4139	<< " to 0x" << Twine::utohexstr(Section.getAddress()) << '\n';
4140	});
4141	Section.setOutputAddress(Section.getAddress());
4142	Section.setOutputFileOffset(Section.getInputFileOffset());
4143	MapSection(Section, Section.getAddress());
4144	} else {
4145	uint64_t Alignment = Section.getAlignment();
4146	if (opts::Instrument && StartLinkingRuntimeLib) {
4147	Alignment = BC->RegularPageSize;
4148	StartLinkingRuntimeLib = false;
4149	}
4150	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: Alignment);
4151
4152	LLVM_DEBUG({
4153	dbgs() << "BOLT-DEBUG: mapping section " << Section.getName()
4154	<< " (0x" << Twine::utohexstr(Section.getAllocAddress())
4155	<< ") to 0x" << Twine::utohexstr(NextAvailableAddress) << ":0x"
4156	<< Twine::utohexstr(NextAvailableAddress +
4157	Section.getOutputSize())
4158	<< '\n';
4159	});
4160
4161	MapSection(Section, NextAvailableAddress);
4162	Section.setOutputAddress(NextAvailableAddress);
4163	Section.setOutputFileOffset(
4164	getFileOffsetForAddress(Address: NextAvailableAddress));
4165
4166	NextAvailableAddress += Section.getOutputSize();
4167	}
4168	}
4169
4170	if (SType == ST_READONLY) {
4171	if (NewTextSegmentAddress)
4172	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
4173	} else if (SType == ST_READWRITE) {
4174	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
4175	// Restore NextAvailableAddress if no new writable sections
4176	if (!NewWritableSegmentSize)
4177	NextAvailableAddress = LastNextAvailableAddress;
4178	}
4179	}
4180	}
4181
4182	void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
4183	if (std::optional<AddressMap> Map = AddressMap::parse(BC&: *BC))
4184	BC->setIOAddressMap(std::move(*Map));
4185
4186	for (BinaryFunction *Function : BC->getAllBinaryFunctions())
4187	Function->updateOutputValues(Linker);
4188	}
4189
4190	void RewriteInstance::updateSegmentInfo() {
4191	// NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
4192	// last segments size based on the NextAvailableAddress variable.
4193	if (!NewWritableSegmentSize) {
4194	if (NewTextSegmentAddress)
4195	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
4196	} else {
4197	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
4198	}
4199
4200	if (NewTextSegmentSize) {
4201	SegmentInfo TextSegment = {.Address: NewTextSegmentAddress,
4202	.Size: NewTextSegmentSize,
4203	.FileOffset: NewTextSegmentOffset,
4204	.FileSize: NewTextSegmentSize,
4205	.Alignment: BC->PageAlign,
4206	.IsExecutable: true,
4207	.IsWritable: false};
4208	if (!opts::Instrument) {
4209	BC->NewSegments.push_back(x: TextSegment);
4210	} else {
4211	ErrorOr<BinarySection &> Sec =
4212	BC->getUniqueSectionByName(SectionName: ".bolt.instr.counters");
4213	assert(Sec && "expected one and only one `.bolt.instr.counters` section");
4214	const uint64_t Addr = Sec->getOutputAddress();
4215	const uint64_t Offset = Sec->getOutputFileOffset();
4216	const uint64_t Size = Sec->getOutputSize();
4217	assert(Addr > TextSegment.Address &&
4218	Addr + Size < TextSegment.Address + TextSegment.Size &&
4219	"`.bolt.instr.counters` section is expected to be included in the "
4220	"new text segment");
4221
4222	// Set correct size for the previous header since we are breaking the
4223	// new text segment into three segments.
4224	uint64_t Delta = Addr - TextSegment.Address;
4225	TextSegment.Size = Delta;
4226	TextSegment.FileSize = Delta;
4227	BC->NewSegments.push_back(x: TextSegment);
4228
4229	// Create RW segment that includes the `.bolt.instr.counters` section.
4230	SegmentInfo RWSegment = {.Address: Addr, .Size: Size, .FileOffset: Offset, .FileSize: Size, .Alignment: BC->RegularPageSize,
4231	.IsExecutable: false, .IsWritable: true};
4232	BC->NewSegments.push_back(x: RWSegment);
4233
4234	// Create RX segment that includes all RX sections from runtime library.
4235	const uint64_t AddrRX = alignTo(Value: Addr + Size, Align: BC->RegularPageSize);
4236	const uint64_t OffsetRX = alignTo(Value: Offset + Size, Align: BC->RegularPageSize);
4237	const uint64_t SizeRX =
4238	NewTextSegmentSize - (AddrRX - TextSegment.Address);
4239	SegmentInfo RXSegment = {
4240	.Address: AddrRX, .Size: SizeRX, .FileOffset: OffsetRX, .FileSize: SizeRX, .Alignment: BC->RegularPageSize, .IsExecutable: true, .IsWritable: false};
4241	BC->NewSegments.push_back(x: RXSegment);
4242	}
4243	}
4244
4245	if (NewWritableSegmentSize) {
4246	SegmentInfo DataSegmentInfo = {
4247	.Address: NewWritableSegmentAddress,
4248	.Size: NewWritableSegmentSize,
4249	.FileOffset: getFileOffsetForAddress(Address: NewWritableSegmentAddress),
4250	.FileSize: NewWritableSegmentSize,
4251	.Alignment: BC->RegularPageSize,
4252	.IsExecutable: false,
4253	.IsWritable: true};
4254	BC->NewSegments.push_back(x: DataSegmentInfo);
4255	}
4256	}
4257
4258	void RewriteInstance::patchELFPHDRTable() {
4259	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
4260	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
4261	raw_fd_ostream &OS = Out->os();
4262
4263	// Write/re-write program headers.
4264	Phnum = Obj.getHeader().e_phnum;
4265	if (PHDRTableOffset) {
4266	// Writing new pheader table and adding one new entry for R+X segment.
4267	Phnum += 1;
4268	if (NewWritableSegmentSize) {
4269	// Adding one more entry for R+W segment.
4270	Phnum += 1;
4271	}
4272	} else {
4273	assert(!PHDRTableAddress && "unexpected address for program header table");
4274	PHDRTableOffset = Obj.getHeader().e_phoff;
4275	if (NewWritableSegmentSize) {
4276	BC->errs() << "BOLT-ERROR: unable to add writable segment\n";
4277	exit(status: 1);
4278	}
4279	}
4280
4281	if (opts::Instrument)
4282	Phnum += 2;
4283
4284	if (BC->NewSegments.empty()) {
4285	BC->outs() << "BOLT-INFO: not adding new segments\n";
4286	return;
4287	}
4288
4289	const uint64_t SavedPos = OS.tell();
4290	OS.seek(off: PHDRTableOffset);
4291
4292	auto createPhdr = [](const SegmentInfo &SI) {
4293	ELF64LEPhdrTy Phdr;
4294	Phdr.p_type = ELF::PT_LOAD;
4295	Phdr.p_offset = SI.FileOffset;
4296	Phdr.p_vaddr = SI.Address;
4297	Phdr.p_paddr = SI.Address;
4298	Phdr.p_filesz = SI.FileSize;
4299	Phdr.p_memsz = SI.Size;
4300	Phdr.p_flags = ELF::PF_R;
4301	if (SI.IsExecutable)
4302	Phdr.p_flags |= ELF::PF_X;
4303	if (SI.IsWritable)
4304	Phdr.p_flags |= ELF::PF_W;
4305	Phdr.p_align = SI.Alignment;
4306
4307	return Phdr;
4308	};
4309
4310	auto writeNewSegmentPhdrs = [&]() {
4311	for (const SegmentInfo &SI : BC->NewSegments) {
4312	ELF64LEPhdrTy Phdr = createPhdr(SI);
4313	OS.write(Ptr: reinterpret_cast<const char *>(&Phdr), Size: sizeof(Phdr));
4314	}
4315	};
4316
4317	bool ModdedGnuStack = false;
4318	bool AddedSegment = false;
4319
4320	// Copy existing program headers with modifications.
4321	for (const ELF64LE::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers())) {
4322	ELF64LE::Phdr NewPhdr = Phdr;
4323	switch (Phdr.p_type) {
4324	case ELF::PT_PHDR:
4325	if (PHDRTableAddress) {
4326	NewPhdr.p_offset = PHDRTableOffset;
4327	NewPhdr.p_vaddr = PHDRTableAddress;
4328	NewPhdr.p_paddr = PHDRTableAddress;
4329	NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum;
4330	NewPhdr.p_memsz = sizeof(NewPhdr) * Phnum;
4331	}
4332	break;
4333	case ELF::PT_GNU_EH_FRAME: {
4334	ErrorOr<BinarySection &> EHFrameHdrSec = BC->getUniqueSectionByName(
4335	SectionName: getNewSecPrefix() + getEHFrameHdrSectionName());
4336	if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() &&
4337	EHFrameHdrSec->isFinalized()) {
4338	NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset();
4339	NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress();
4340	NewPhdr.p_paddr = EHFrameHdrSec->getOutputAddress();
4341	NewPhdr.p_filesz = EHFrameHdrSec->getOutputSize();
4342	NewPhdr.p_memsz = EHFrameHdrSec->getOutputSize();
4343	}
4344	break;
4345	}
4346	case ELF::PT_GNU_STACK:
4347	if (opts::UseGnuStack) {
4348	// Overwrite the header with the new segment header.
4349	assert(BC->NewSegments.size() == 1 &&
4350	"Expected exactly one new segment");
4351	NewPhdr = createPhdr(BC->NewSegments.front());
4352	ModdedGnuStack = true;
4353	}
4354	break;
4355	case ELF::PT_DYNAMIC:
4356	if (!opts::UseGnuStack) {
4357	// Insert new headers before DYNAMIC.
4358	writeNewSegmentPhdrs();
4359	AddedSegment = true;
4360	}
4361	break;
4362	}
4363	OS.write(Ptr: reinterpret_cast<const char *>(&NewPhdr), Size: sizeof(NewPhdr));
4364	}
4365
4366	if (!opts::UseGnuStack && !AddedSegment) {
4367	// Append new headers to the end of the table.
4368	writeNewSegmentPhdrs();
4369	}
4370
4371	if (opts::UseGnuStack && !ModdedGnuStack) {
4372	BC->errs()
4373	<< "BOLT-ERROR: could not find PT_GNU_STACK program header to modify\n";
4374	exit(status: 1);
4375	}
4376
4377	OS.seek(off: SavedPos);
4378	}
4379
4380	namespace {
4381
4382	/// Write padding to \p OS such that its current \p Offset becomes aligned
4383	/// at \p Alignment. Return new (aligned) offset.
4384	uint64_t appendPadding(raw_pwrite_stream &OS, uint64_t Offset,
4385	uint64_t Alignment) {
4386	if (!Alignment)
4387	return Offset;
4388
4389	const uint64_t PaddingSize =
4390	offsetToAlignment(Value: Offset, Alignment: llvm::Align(Alignment));
4391	for (unsigned I = 0; I < PaddingSize; ++I)
4392	OS.write(C: (unsigned char)0);
4393	return Offset + PaddingSize;
4394	}
4395
4396	}
4397
4398	void RewriteInstance::rewriteNoteSections() {
4399	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
4400	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
4401	raw_fd_ostream &OS = Out->os();
4402
4403	uint64_t NextAvailableOffset = std::max(
4404	a: getFileOffsetForAddress(Address: NextAvailableAddress), b: FirstNonAllocatableOffset);
4405	OS.seek(off: NextAvailableOffset);
4406
4407	// Copy over non-allocatable section contents and update file offsets.
4408	for (const ELF64LE::Shdr &Section : cantFail(ValOrErr: Obj.sections())) {
4409	if (Section.sh_type == ELF::SHT_NULL)
4410	continue;
4411	if (Section.sh_flags & ELF::SHF_ALLOC)
4412	continue;
4413
4414	SectionRef SecRef = ELF64LEFile->toSectionRef(Sec: &Section);
4415	BinarySection *BSec = BC->getSectionForSectionRef(Section: SecRef);
4416	assert(BSec && !BSec->isAllocatable() &&
4417	"Matching non-allocatable BinarySection should exist.");
4418
4419	StringRef SectionName =
4420	cantFail(ValOrErr: Obj.getSectionName(Section), Msg: "cannot get section name");
4421	if (shouldStrip(Section, SectionName))
4422	continue;
4423
4424	// Insert padding as needed.
4425	NextAvailableOffset =
4426	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.sh_addralign);
4427
4428	// New section size.
4429	uint64_t Size = 0;
4430	bool DataWritten = false;
4431	// Copy over section contents unless it's one of the sections we overwrite.
4432	if (!willOverwriteSection(SectionName)) {
4433	Size = Section.sh_size;
4434	StringRef Dataref = InputFile->getData().substr(Start: Section.sh_offset, N: Size);
4435	std::string Data;
4436	if (BSec->getPatcher()) {
4437	Data = BSec->getPatcher()->patchBinary(BinaryContents: Dataref);
4438	Dataref = StringRef(Data);
4439	}
4440
4441	// Section was expanded, so need to treat it as overwrite.
4442	if (Size != Dataref.size()) {
4443	BSec = &BC->registerOrUpdateNoteSection(
4444	Name: SectionName, Data: copyByteArray(Buffer: Dataref), Size: Dataref.size());
4445	Size = 0;
4446	} else {
4447	OS << Dataref;
4448	DataWritten = true;
4449
4450	// Add padding as the section extension might rely on the alignment.
4451	Size = appendPadding(OS, Offset: Size, Alignment: Section.sh_addralign);
4452	}
4453	}
4454
4455	// Perform section post-processing.
4456	assert(BSec->getAlignment() <= Section.sh_addralign &&
4457	"alignment exceeds value in file");
4458
4459	if (BSec->getAllocAddress()) {
4460	assert(!DataWritten && "Writing section twice.");
4461	(void)DataWritten;
4462	Size += BSec->write(OS);
4463	}
4464
4465	BSec->setOutputFileOffset(NextAvailableOffset);
4466	BSec->flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
4467	return getNewValueForSymbol(Name: S->getName());
4468	});
4469
4470	// Section contents are no longer needed, but we need to update the size so
4471	// that it will be reflected in the section header table.
4472	BSec->updateContents(NewData: nullptr, NewSize: Size);
4473
4474	NextAvailableOffset += Size;
4475	}
4476
4477	// Write new note sections.
4478	for (BinarySection &Section : BC->nonAllocatableSections()) {
4479	if (Section.getOutputFileOffset() || !Section.getAllocAddress())
4480	continue;
4481
4482	assert(!Section.hasPendingRelocations() && "cannot have pending relocs");
4483
4484	NextAvailableOffset =
4485	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.getAlignment());
4486	Section.setOutputFileOffset(NextAvailableOffset);
4487
4488	LLVM_DEBUG(
4489	dbgs() << "BOLT-DEBUG: writing out new section " << Section.getName()
4490	<< " of size " << Section.getOutputSize() << " at offset 0x"
4491	<< Twine::utohexstr(Section.getOutputFileOffset()) << '\n');
4492
4493	NextAvailableOffset += Section.write(OS);
4494	}
4495	}
4496
4497	template <typename ELFT>
4498	void RewriteInstance::finalizeSectionStringTable(ELFObjectFile<ELFT> *File) {
4499	// Pre-populate section header string table.
4500	for (const BinarySection &Section : BC->sections())
4501	if (!Section.isAnonymous())
4502	SHStrTab.add(S: Section.getOutputName());
4503	SHStrTab.finalize();
4504
4505	const size_t SHStrTabSize = SHStrTab.getSize();
4506	uint8_t *DataCopy = new uint8_t[SHStrTabSize];
4507	memset(s: DataCopy, c: 0, n: SHStrTabSize);
4508	SHStrTab.write(Buf: DataCopy);
4509	BC->registerOrUpdateNoteSection(Name: ".shstrtab",
4510	Data: DataCopy,
4511	Size: SHStrTabSize,
4512	/*Alignment=*/1,
4513	/*IsReadOnly=*/true,
4514	ELFType: ELF::SHT_STRTAB);
4515	}
4516
4517	void RewriteInstance::addBoltInfoSection() {
4518	std::string DescStr;
4519	raw_string_ostream DescOS(DescStr);
4520
4521	DescOS << "BOLT revision: " << BoltRevision << ", "
4522	<< "command line:";
4523	for (int I = 0; I < Argc; ++I)
4524	DescOS << " " << Argv[I];
4525
4526	// Encode as GNU GOLD VERSION so it is easily printable by 'readelf -n'
4527	const std::string BoltInfo =
4528	BinarySection::encodeELFNote(NameStr: "GNU", DescStr, Type: 4 /*NT_GNU_GOLD_VERSION*/);
4529	BC->registerOrUpdateNoteSection(Name: ".note.bolt_info", Data: copyByteArray(Buffer: BoltInfo),
4530	Size: BoltInfo.size(),
4531	/*Alignment=*/1,
4532	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4533	}
4534
4535	void RewriteInstance::addBATSection() {
4536	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME, Data: nullptr,
4537	Size: 0,
4538	/*Alignment=*/1,
4539	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4540	}
4541
4542	void RewriteInstance::encodeBATSection() {
4543	std::string DescStr;
4544	raw_string_ostream DescOS(DescStr);
4545
4546	BAT->write(BC: *BC, OS&: DescOS);
4547
4548	const std::string BoltInfo =
4549	BinarySection::encodeELFNote(NameStr: "BOLT", DescStr, Type: BinarySection::NT_BOLT_BAT);
4550	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME,
4551	Data: copyByteArray(Buffer: BoltInfo), Size: BoltInfo.size(),
4552	/*Alignment=*/1,
4553	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4554	BC->outs() << "BOLT-INFO: BAT section size (bytes): " << BoltInfo.size()
4555	<< '\n';
4556	}
4557
4558	template <typename ELFShdrTy>
4559	bool RewriteInstance::shouldStrip(const ELFShdrTy &Section,
4560	StringRef SectionName) {
4561	// Strip non-allocatable relocation sections.
4562	if (!(Section.sh_flags & ELF::SHF_ALLOC) && Section.sh_type == ELF::SHT_RELA)
4563	return true;
4564
4565	// Strip debug sections if not updating them.
4566	if (isDebugSection(SectionName) && !opts::UpdateDebugSections)
4567	return true;
4568
4569	// Strip symtab section if needed
4570	if (opts::RemoveSymtab && Section.sh_type == ELF::SHT_SYMTAB)
4571	return true;
4572
4573	return false;
4574	}
4575
4576	template <typename ELFT>
4577	std::vector<typename object::ELFObjectFile<ELFT>::Elf_Shdr>
4578	RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
4579	std::vector<uint32_t> &NewSectionIndex) {
4580	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4581	const ELFFile<ELFT> &Obj = File->getELFFile();
4582	typename ELFT::ShdrRange Sections = cantFail(Obj.sections());
4583
4584	// Keep track of section header entries attached to the corresponding section.
4585	std::vector<std::pair<BinarySection *, ELFShdrTy>> OutputSections;
4586	auto addSection = [&](const ELFShdrTy &Section, BinarySection &BinSec) {
4587	ELFShdrTy NewSection = Section;
4588	NewSection.sh_name = SHStrTab.getOffset(S: BinSec.getOutputName());
4589	OutputSections.emplace_back(&BinSec, std::move(NewSection));
4590	};
4591
4592	// Copy over entries for original allocatable sections using modified name.
4593	for (const ELFShdrTy &Section : Sections) {
4594	// Always ignore this section.
4595	if (Section.sh_type == ELF::SHT_NULL) {
4596	OutputSections.emplace_back(nullptr, Section);
4597	continue;
4598	}
4599
4600	if (!(Section.sh_flags & ELF::SHF_ALLOC))
4601	continue;
4602
4603	SectionRef SecRef = File->toSectionRef(&Section);
4604	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4605	assert(BinSec && "Matching BinarySection should exist.");
4606
4607	// Exclude anonymous sections.
4608	if (BinSec->isAnonymous())
4609	continue;
4610
4611	addSection(Section, *BinSec);
4612	}
4613
4614	for (BinarySection &Section : BC->allocatableSections()) {
4615	if (!Section.isFinalized())
4616	continue;
4617
4618	if (Section.hasSectionRef() || Section.isAnonymous()) {
4619	if (opts::Verbosity)
4620	BC->outs() << "BOLT-INFO: not writing section header for section "
4621	<< Section.getOutputName() << '\n';
4622	continue;
4623	}
4624
4625	if (opts::Verbosity >= 1)
4626	BC->outs() << "BOLT-INFO: writing section header for "
4627	<< Section.getOutputName() << '\n';
4628	ELFShdrTy NewSection;
4629	NewSection.sh_type = ELF::SHT_PROGBITS;
4630	NewSection.sh_addr = Section.getOutputAddress();
4631	NewSection.sh_offset = Section.getOutputFileOffset();
4632	NewSection.sh_size = Section.getOutputSize();
4633	NewSection.sh_entsize = 0;
4634	NewSection.sh_flags = Section.getELFFlags();
4635	NewSection.sh_link = 0;
4636	NewSection.sh_info = 0;
4637	NewSection.sh_addralign = Section.getAlignment();
4638	addSection(NewSection, Section);
4639	}
4640
4641	// Sort all allocatable sections by their offset.
4642	llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) {
4643	return A.second.sh_offset < B.second.sh_offset;
4644	});
4645
4646	// Fix section sizes to prevent overlapping.
4647	ELFShdrTy *PrevSection = nullptr;
4648	BinarySection *PrevBinSec = nullptr;
4649	for (auto &SectionKV : OutputSections) {
4650	ELFShdrTy &Section = SectionKV.second;
4651
4652	// Ignore NOBITS sections as they don't take any space in the file.
4653	if (Section.sh_type == ELF::SHT_NOBITS)
4654	continue;
4655
4656	// Note that address continuity is not guaranteed as sections could be
4657	// placed in different loadable segments.
4658	if (PrevSection &&
4659	PrevSection->sh_offset + PrevSection->sh_size > Section.sh_offset) {
4660	if (opts::Verbosity > 1)
4661	BC->outs() << "BOLT-INFO: adjusting size for section "
4662	<< PrevBinSec->getOutputName() << '\n';
4663	PrevSection->sh_size = Section.sh_offset - PrevSection->sh_offset;
4664	}
4665
4666	PrevSection = &Section;
4667	PrevBinSec = SectionKV.first;
4668	}
4669
4670	uint64_t LastFileOffset = 0;
4671
4672	// Copy over entries for non-allocatable sections performing necessary
4673	// adjustments.
4674	for (const ELFShdrTy &Section : Sections) {
4675	if (Section.sh_type == ELF::SHT_NULL)
4676	continue;
4677	if (Section.sh_flags & ELF::SHF_ALLOC)
4678	continue;
4679
4680	StringRef SectionName =
4681	cantFail(Obj.getSectionName(Section), "cannot get section name");
4682
4683	if (shouldStrip(Section, SectionName))
4684	continue;
4685
4686	SectionRef SecRef = File->toSectionRef(&Section);
4687	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4688	assert(BinSec && "Matching BinarySection should exist.");
4689
4690	ELFShdrTy NewSection = Section;
4691	NewSection.sh_offset = BinSec->getOutputFileOffset();
4692	NewSection.sh_size = BinSec->getOutputSize();
4693
4694	if (NewSection.sh_type == ELF::SHT_SYMTAB)
4695	NewSection.sh_info = NumLocalSymbols;
4696
4697	addSection(NewSection, *BinSec);
4698
4699	LastFileOffset = BinSec->getOutputFileOffset();
4700	}
4701
4702	// Create entries for new non-allocatable sections.
4703	for (BinarySection &Section : BC->nonAllocatableSections()) {
4704	if (Section.getOutputFileOffset() <= LastFileOffset)
4705	continue;
4706
4707	if (opts::Verbosity >= 1)
4708	BC->outs() << "BOLT-INFO: writing section header for "
4709	<< Section.getOutputName() << '\n';
4710
4711	ELFShdrTy NewSection;
4712	NewSection.sh_type = Section.getELFType();
4713	NewSection.sh_addr = 0;
4714	NewSection.sh_offset = Section.getOutputFileOffset();
4715	NewSection.sh_size = Section.getOutputSize();
4716	NewSection.sh_entsize = 0;
4717	NewSection.sh_flags = Section.getELFFlags();
4718	NewSection.sh_link = 0;
4719	NewSection.sh_info = 0;
4720	NewSection.sh_addralign = Section.getAlignment();
4721
4722	addSection(NewSection, Section);
4723	}
4724
4725	// Assign indices to sections.
4726	for (uint32_t Index = 1; Index < OutputSections.size(); ++Index)
4727	OutputSections[Index].first->setIndex(Index);
4728
4729	// Update section index mapping
4730	NewSectionIndex.clear();
4731	NewSectionIndex.resize(Sections.size(), 0);
4732	for (const ELFShdrTy &Section : Sections) {
4733	if (Section.sh_type == ELF::SHT_NULL)
4734	continue;
4735
4736	size_t OrgIndex = std::distance(Sections.begin(), &Section);
4737
4738	SectionRef SecRef = File->toSectionRef(&Section);
4739	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4740	assert(BinSec && "BinarySection should exist for an input section.");
4741
4742	// Some sections are stripped
4743	if (!BinSec->hasValidIndex())
4744	continue;
4745
4746	NewSectionIndex[OrgIndex] = BinSec->getIndex();
4747	}
4748
4749	std::vector<ELFShdrTy> SectionsOnly(OutputSections.size());
4750	llvm::copy(llvm::make_second_range(OutputSections), SectionsOnly.begin());
4751
4752	return SectionsOnly;
4753	}
4754
4755	// Rewrite section header table inserting new entries as needed. The sections
4756	// header table size itself may affect the offsets of other sections,
4757	// so we are placing it at the end of the binary.
4758	//
4759	// As we rewrite entries we need to track how many sections were inserted
4760	// as it changes the sh_link value. We map old indices to new ones for
4761	// existing sections.
4762	template <typename ELFT>
4763	void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {
4764	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4765	using ELFEhdrTy = typename ELFObjectFile<ELFT>::Elf_Ehdr;
4766	raw_fd_ostream &OS = Out->os();
4767	const ELFFile<ELFT> &Obj = File->getELFFile();
4768
4769	// Mapping from old section indices to new ones
4770	std::vector<uint32_t> NewSectionIndex;
4771	std::vector<ELFShdrTy> OutputSections =
4772	getOutputSections(File, NewSectionIndex);
4773	LLVM_DEBUG(
4774	dbgs() << "BOLT-DEBUG: old to new section index mapping:\n";
4775	for (uint64_t I = 0; I < NewSectionIndex.size(); ++I)
4776	dbgs() << " " << I << " -> " << NewSectionIndex[I] << '\n';
4777	);
4778
4779	// Align starting address for section header table. There's no architecutal
4780	// need to align this, it is just for pleasant human readability.
4781	uint64_t SHTOffset = OS.tell();
4782	SHTOffset = appendPadding(OS, Offset: SHTOffset, Alignment: 16);
4783
4784	// Write all section header entries while patching section references.
4785	for (ELFShdrTy &Section : OutputSections) {
4786	Section.sh_link = NewSectionIndex[Section.sh_link];
4787	if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA)
4788	Section.sh_info = NewSectionIndex[Section.sh_info];
4789	OS.write(Ptr: reinterpret_cast<const char *>(&Section), Size: sizeof(Section));
4790	}
4791
4792	// Fix ELF header.
4793	ELFEhdrTy NewEhdr = Obj.getHeader();
4794
4795	if (BC->HasRelocations) {
4796	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
4797	NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();
4798	else
4799	NewEhdr.e_entry = getNewFunctionAddress(OldAddress: NewEhdr.e_entry);
4800	assert((NewEhdr.e_entry || !Obj.getHeader().e_entry) &&
4801	"cannot find new address for entry point");
4802	}
4803	if (PHDRTableOffset) {
4804	NewEhdr.e_phoff = PHDRTableOffset;
4805	NewEhdr.e_phnum = Phnum;
4806	}
4807	NewEhdr.e_shoff = SHTOffset;
4808	NewEhdr.e_shnum = OutputSections.size();
4809	NewEhdr.e_shstrndx = NewSectionIndex[NewEhdr.e_shstrndx];
4810	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewEhdr), Size: sizeof(NewEhdr), Offset: 0);
4811	}
4812
4813	template <typename ELFT, typename WriteFuncTy, typename StrTabFuncTy>
4814	void RewriteInstance::updateELFSymbolTable(
4815	ELFObjectFile<ELFT> *File, bool IsDynSym,
4816	const typename object::ELFObjectFile<ELFT>::Elf_Shdr &SymTabSection,
4817	const std::vector<uint32_t> &NewSectionIndex, WriteFuncTy Write,
4818	StrTabFuncTy AddToStrTab) {
4819	const ELFFile<ELFT> &Obj = File->getELFFile();
4820	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
4821
4822	StringRef StringSection =
4823	cantFail(Obj.getStringTableForSymtab(SymTabSection));
4824
4825	unsigned NumHotTextSymsUpdated = 0;
4826	unsigned NumHotDataSymsUpdated = 0;
4827
4828	std::map<const BinaryFunction *, uint64_t> IslandSizes;
4829	auto getConstantIslandSize = [&IslandSizes](const BinaryFunction &BF) {
4830	auto Itr = IslandSizes.find(x: &BF);
4831	if (Itr != IslandSizes.end())
4832	return Itr->second;
4833	return IslandSizes[&BF] = BF.estimateConstantIslandSize();
4834	};
4835
4836	// Symbols for the new symbol table.
4837	std::vector<ELFSymTy> Symbols;
4838
4839	bool EmittedColdFileSymbol = false;
4840
4841	auto getNewSectionIndex = [&](uint32_t OldIndex) {
4842	// For dynamic symbol table, the section index could be wrong on the input,
4843	// and its value is ignored by the runtime if it's different from
4844	// SHN_UNDEF and SHN_ABS.
4845	// However, we still need to update dynamic symbol table, so return a
4846	// section index, even though the index is broken.
4847	if (IsDynSym && OldIndex >= NewSectionIndex.size())
4848	return OldIndex;
4849
4850	assert(OldIndex < NewSectionIndex.size() && "section index out of bounds");
4851	const uint32_t NewIndex = NewSectionIndex[OldIndex];
4852
4853	// We may have stripped the section that dynsym was referencing due to
4854	// the linker bug. In that case return the old index avoiding marking
4855	// the symbol as undefined.
4856	if (IsDynSym && NewIndex != OldIndex && NewIndex == ELF::SHN_UNDEF)
4857	return OldIndex;
4858	return NewIndex;
4859	};
4860
4861	// Get the extra symbol name of a split fragment; used in addExtraSymbols.
4862	auto getSplitSymbolName = [&](const FunctionFragment &FF,
4863	const ELFSymTy &FunctionSymbol) {
4864	SmallString<256> SymbolName;
4865	if (BC->HasWarmSection)
4866	SymbolName =
4867	formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
4868	FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
4869	else
4870	SymbolName = formatv("{0}.cold.{1}",
4871	cantFail(FunctionSymbol.getName(StringSection)),
4872	FF.getFragmentNum().get() - 1);
4873	return SymbolName;
4874	};
4875
4876	// Add extra symbols for the function.
4877	//
4878	// Note that addExtraSymbols() could be called multiple times for the same
4879	// function with different FunctionSymbol matching the main function entry
4880	// point.
4881	auto addExtraSymbols = [&](const BinaryFunction &Function,
4882	const ELFSymTy &FunctionSymbol) {
4883	if (Function.isFolded()) {
4884	BinaryFunction *ICFParent = Function.getFoldedIntoFunction();
4885	while (ICFParent->isFolded())
4886	ICFParent = ICFParent->getFoldedIntoFunction();
4887	ELFSymTy ICFSymbol = FunctionSymbol;
4888	SmallVector<char, 256> Buf;
4889	ICFSymbol.st_name =
4890	AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection)))
4891	.concat(Suffix: ".icf.0")
4892	.toStringRef(Out&: Buf));
4893	ICFSymbol.st_value = ICFParent->getOutputAddress();
4894	ICFSymbol.st_size = ICFParent->getOutputSize();
4895	ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex();
4896	Symbols.emplace_back(ICFSymbol);
4897	}
4898	if (Function.isSplit()) {
4899	// Prepend synthetic FILE symbol to prevent local cold fragments from
4900	// colliding with existing symbols with the same name.
4901	if (!EmittedColdFileSymbol &&
4902	FunctionSymbol.getBinding() == ELF::STB_GLOBAL) {
4903	ELFSymTy FileSymbol;
4904	FileSymbol.st_shndx = ELF::SHN_ABS;
4905	FileSymbol.st_name = AddToStrTab(getBOLTFileSymbolName());
4906	FileSymbol.st_value = 0;
4907	FileSymbol.st_size = 0;
4908	FileSymbol.st_other = 0;
4909	FileSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FILE);
4910	Symbols.emplace_back(FileSymbol);
4911	EmittedColdFileSymbol = true;
4912	}
4913	for (const FunctionFragment &FF :
4914	Function.getLayout().getSplitFragments()) {
4915	if (FF.getAddress()) {
4916	ELFSymTy NewColdSym = FunctionSymbol;
4917	const SmallString<256> SymbolName =
4918	getSplitSymbolName(FF, FunctionSymbol);
4919	NewColdSym.st_name = AddToStrTab(SymbolName);
4920	NewColdSym.st_shndx =
4921	Function.getCodeSection(Fragment: FF.getFragmentNum())->getIndex();
4922	NewColdSym.st_value = FF.getAddress();
4923	NewColdSym.st_size = FF.getImageSize();
4924	NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
4925	Symbols.emplace_back(NewColdSym);
4926	}
4927	}
4928	}
4929	if (Function.hasConstantIsland()) {
4930	uint64_t DataMark = Function.getOutputDataAddress();
4931	uint64_t CISize = getConstantIslandSize(Function);
4932	uint64_t CodeMark = DataMark + CISize;
4933	ELFSymTy DataMarkSym = FunctionSymbol;
4934	DataMarkSym.st_name = AddToStrTab("$d");
4935	DataMarkSym.st_value = DataMark;
4936	DataMarkSym.st_size = 0;
4937	DataMarkSym.setType(ELF::STT_NOTYPE);
4938	DataMarkSym.setBinding(ELF::STB_LOCAL);
4939	ELFSymTy CodeMarkSym = DataMarkSym;
4940	CodeMarkSym.st_name = AddToStrTab("$x");
4941	CodeMarkSym.st_value = CodeMark;
4942	Symbols.emplace_back(DataMarkSym);
4943	Symbols.emplace_back(CodeMarkSym);
4944	}
4945	if (Function.hasConstantIsland() && Function.isSplit()) {
4946	uint64_t DataMark = Function.getOutputColdDataAddress();
4947	uint64_t CISize = getConstantIslandSize(Function);
4948	uint64_t CodeMark = DataMark + CISize;
4949	ELFSymTy DataMarkSym = FunctionSymbol;
4950	DataMarkSym.st_name = AddToStrTab("$d");
4951	DataMarkSym.st_value = DataMark;
4952	DataMarkSym.st_size = 0;
4953	DataMarkSym.setType(ELF::STT_NOTYPE);
4954	DataMarkSym.setBinding(ELF::STB_LOCAL);
4955	ELFSymTy CodeMarkSym = DataMarkSym;
4956	CodeMarkSym.st_name = AddToStrTab("$x");
4957	CodeMarkSym.st_value = CodeMark;
4958	Symbols.emplace_back(DataMarkSym);
4959	Symbols.emplace_back(CodeMarkSym);
4960	}
4961	};
4962
4963	// For regular (non-dynamic) symbol table, exclude symbols referring
4964	// to non-allocatable sections.
4965	auto shouldStrip = [&](const ELFSymTy &Symbol) {
4966	if (Symbol.isAbsolute() || !Symbol.isDefined())
4967	return false;
4968
4969	// If we cannot link the symbol to a section, leave it as is.
4970	Expected<const typename ELFT::Shdr *> Section =
4971	Obj.getSection(Symbol.st_shndx);
4972	if (!Section)
4973	return false;
4974
4975	// Remove the section symbol iif the corresponding section was stripped.
4976	if (Symbol.getType() == ELF::STT_SECTION) {
4977	if (!getNewSectionIndex(Symbol.st_shndx))
4978	return true;
4979	return false;
4980	}
4981
4982	// Symbols in non-allocatable sections are typically remnants of relocations
4983	// emitted under "-emit-relocs" linker option. Delete those as we delete
4984	// relocations against non-allocatable sections.
4985	if (!((*Section)->sh_flags & ELF::SHF_ALLOC))
4986	return true;
4987
4988	return false;
4989	};
4990
4991	for (const ELFSymTy &Symbol : cantFail(Obj.symbols(&SymTabSection))) {
4992	// For regular (non-dynamic) symbol table strip unneeded symbols.
4993	if (!IsDynSym && shouldStrip(Symbol))
4994	continue;
4995
4996	const BinaryFunction *Function =
4997	BC->getBinaryFunctionAtAddress(Symbol.st_value);
4998	// Ignore false function references, e.g. when the section address matches
4999	// the address of the function.
5000	if (Function && Symbol.getType() == ELF::STT_SECTION)
5001	Function = nullptr;
5002
5003	// For non-dynamic symtab, make sure the symbol section matches that of
5004	// the function. It can mismatch e.g. if the symbol is a section marker
5005	// in which case we treat the symbol separately from the function.
5006	// For dynamic symbol table, the section index could be wrong on the input,
5007	// and its value is ignored by the runtime if it's different from
5008	// SHN_UNDEF and SHN_ABS.
5009	if (!IsDynSym && Function &&
5010	Symbol.st_shndx !=
5011	Function->getOriginSection()->getSectionRef().getIndex())
5012	Function = nullptr;
5013
5014	// Create a new symbol based on the existing symbol.
5015	ELFSymTy NewSymbol = Symbol;
5016
5017	// Handle special symbols based on their name.
5018	Expected<StringRef> SymbolName = Symbol.getName(StringSection);
5019	assert(SymbolName && "cannot get symbol name");
5020
5021	auto updateSymbolValue = [&](const StringRef Name,
5022	std::optional<uint64_t> Value = std::nullopt) {
5023	NewSymbol.st_value = Value ? *Value : getNewValueForSymbol(Name);
5024	NewSymbol.st_shndx = ELF::SHN_ABS;
5025	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
5026	<< Twine::utohexstr(Val: NewSymbol.st_value) << '\n';
5027	};
5028
5029	if (*SymbolName == "__hot_start" || *SymbolName == "__hot_end") {
5030	if (opts::HotText) {
5031	updateSymbolValue(*SymbolName);
5032	++NumHotTextSymsUpdated;
5033	}
5034	goto registerSymbol;
5035	}
5036
5037	if (*SymbolName == "__hot_data_start" || *SymbolName == "__hot_data_end") {
5038	if (opts::HotData) {
5039	updateSymbolValue(*SymbolName);
5040	++NumHotDataSymsUpdated;
5041	}
5042	goto registerSymbol;
5043	}
5044
5045	if (*SymbolName == "_end") {
5046	if (NextAvailableAddress > Symbol.st_value)
5047	updateSymbolValue(*SymbolName, NextAvailableAddress);
5048	goto registerSymbol;
5049	}
5050
5051	if (Function) {
5052	// If the symbol matched a function that was not emitted, update the
5053	// corresponding section index but otherwise leave it unchanged.
5054	if (Function->isEmitted()) {
5055	NewSymbol.st_value = Function->getOutputAddress();
5056	NewSymbol.st_size = Function->getOutputSize();
5057	NewSymbol.st_shndx = Function->getCodeSection()->getIndex();
5058	} else if (Symbol.st_shndx < ELF::SHN_LORESERVE) {
5059	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
5060	}
5061
5062	// Add new symbols to the symbol table if necessary.
5063	if (!IsDynSym)
5064	addExtraSymbols(*Function, NewSymbol);
5065	} else {
5066	// Check if the function symbol matches address inside a function, i.e.
5067	// it marks a secondary entry point.
5068	Function =
5069	(Symbol.getType() == ELF::STT_FUNC)
5070	? BC->getBinaryFunctionContainingAddress(Symbol.st_value,
5071	/*CheckPastEnd=*/false,
5072	/*UseMaxSize=*/true)
5073	: nullptr;
5074
5075	if (Function && Function->isEmitted()) {
5076	assert(Function->getLayout().isHotColdSplit() &&
5077	"Adding symbols based on cold fragment when there are more than "
5078	"2 fragments");
5079	const uint64_t OutputAddress =
5080	Function->translateInputToOutputAddress(Address: Symbol.st_value);
5081
5082	NewSymbol.st_value = OutputAddress;
5083	// Force secondary entry points to have zero size.
5084	NewSymbol.st_size = 0;
5085
5086	// Find fragment containing entrypoint
5087	FunctionLayout::fragment_const_iterator FF = llvm::find_if(
5088	Function->getLayout().fragments(), [&](const FunctionFragment &FF) {
5089	uint64_t Lo = FF.getAddress();
5090	uint64_t Hi = Lo + FF.getImageSize();
5091	return Lo <= OutputAddress && OutputAddress < Hi;
5092	});
5093
5094	if (FF == Function->getLayout().fragment_end()) {
5095	assert(
5096	OutputAddress >= Function->getCodeSection()->getOutputAddress() &&
5097	OutputAddress < (Function->getCodeSection()->getOutputAddress() +
5098	Function->getCodeSection()->getOutputSize()) &&
5099	"Cannot locate fragment containing secondary entrypoint");
5100	FF = Function->getLayout().fragment_begin();
5101	}
5102
5103	NewSymbol.st_shndx =
5104	Function->getCodeSection(Fragment: FF->getFragmentNum())->getIndex();
5105	} else {
5106	// Check if the symbol belongs to moved data object and update it.
5107	BinaryData *BD = opts::ReorderData.empty()
5108	? nullptr
5109	: BC->getBinaryDataAtAddress(Symbol.st_value);
5110	if (BD && BD->isMoved() && !BD->isJumpTable()) {
5111	assert((!BD->getSize() || !Symbol.st_size ||
5112	Symbol.st_size == BD->getSize()) &&
5113	"sizes must match");
5114
5115	BinarySection &OutputSection = BD->getOutputSection();
5116	assert(OutputSection.getIndex());
5117	LLVM_DEBUG(dbgs()
5118	<< "BOLT-DEBUG: moving " << BD->getName() << " from "
5119	<< *BC->getSectionNameForAddress(Symbol.st_value) << " ("
5120	<< Symbol.st_shndx << ") to " << OutputSection.getName()
5121	<< " (" << OutputSection.getIndex() << ")\n");
5122	NewSymbol.st_shndx = OutputSection.getIndex();
5123	NewSymbol.st_value = BD->getOutputAddress();
5124	} else {
5125	// Otherwise just update the section for the symbol.
5126	if (Symbol.st_shndx < ELF::SHN_LORESERVE)
5127	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
5128	}
5129
5130	// Detect local syms in the text section that we didn't update
5131	// and that were preserved by the linker to support relocations against
5132	// .text. Remove them from the symtab.
5133	if (Symbol.getType() == ELF::STT_NOTYPE &&
5134	Symbol.getBinding() == ELF::STB_LOCAL && Symbol.st_size == 0) {
5135	if (BC->getBinaryFunctionContainingAddress(Symbol.st_value,
5136	/*CheckPastEnd=*/false,
5137	/*UseMaxSize=*/true)) {
5138	// Can only delete the symbol if not patching. Such symbols should
5139	// not exist in the dynamic symbol table.
5140	assert(!IsDynSym && "cannot delete symbol");
5141	continue;
5142	}
5143	}
5144	}
5145	}
5146
5147	registerSymbol:
5148	if (IsDynSym)
5149	Write((&Symbol - cantFail(Obj.symbols(&SymTabSection)).begin()) *
5150	sizeof(ELFSymTy),
5151	NewSymbol);
5152	else
5153	Symbols.emplace_back(NewSymbol);
5154	}
5155
5156	if (IsDynSym) {
5157	assert(Symbols.empty());
5158	return;
5159	}
5160
5161	// Add symbols of injected functions
5162	for (BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {
5163	if (Function->isAnonymous())
5164	continue;
5165	ELFSymTy NewSymbol;
5166	BinarySection *OriginSection = Function->getOriginSection();
5167	NewSymbol.st_shndx =
5168	OriginSection
5169	? getNewSectionIndex(OriginSection->getSectionRef().getIndex())
5170	: Function->getCodeSection()->getIndex();
5171	NewSymbol.st_value = Function->getOutputAddress();
5172	NewSymbol.st_name = AddToStrTab(Function->getOneName());
5173	NewSymbol.st_size = Function->getOutputSize();
5174	NewSymbol.st_other = 0;
5175	NewSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
5176	Symbols.emplace_back(NewSymbol);
5177
5178	if (Function->isSplit()) {
5179	assert(Function->getLayout().isHotColdSplit() &&
5180	"Adding symbols based on cold fragment when there are more than "
5181	"2 fragments");
5182	ELFSymTy NewColdSym = NewSymbol;
5183	NewColdSym.setType(ELF::STT_NOTYPE);
5184	SmallVector<char, 256> Buf;
5185	NewColdSym.st_name = AddToStrTab(
5186	Twine(Function->getPrintName()).concat(Suffix: ".cold.0").toStringRef(Out&: Buf));
5187	const FunctionFragment &ColdFF =
5188	Function->getLayout().getFragment(Num: FragmentNum::cold());
5189	NewColdSym.st_value = ColdFF.getAddress();
5190	NewColdSym.st_size = ColdFF.getImageSize();
5191	Symbols.emplace_back(NewColdSym);
5192	}
5193	}
5194
5195	auto AddSymbol = [&](const StringRef &Name, uint64_t Address) {
5196	if (!Address)
5197	return;
5198
5199	ELFSymTy Symbol;
5200	Symbol.st_value = Address;
5201	Symbol.st_shndx = ELF::SHN_ABS;
5202	Symbol.st_name = AddToStrTab(Name);
5203	Symbol.st_size = 0;
5204	Symbol.st_other = 0;
5205	Symbol.setBindingAndType(ELF::STB_WEAK, ELF::STT_NOTYPE);
5206
5207	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
5208	<< Twine::utohexstr(Val: Symbol.st_value) << '\n';
5209
5210	Symbols.emplace_back(Symbol);
5211	};
5212
5213	// Add runtime library start and fini address symbols
5214	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {
5215	AddSymbol("__bolt_runtime_start", RtLibrary->getRuntimeStartAddress());
5216	AddSymbol("__bolt_runtime_fini", RtLibrary->getRuntimeFiniAddress());
5217	}
5218
5219	assert((!NumHotTextSymsUpdated || NumHotTextSymsUpdated == 2) &&
5220	"either none or both __hot_start/__hot_end symbols were expected");
5221	assert((!NumHotDataSymsUpdated || NumHotDataSymsUpdated == 2) &&
5222	"either none or both __hot_data_start/__hot_data_end symbols were "
5223	"expected");
5224
5225	auto AddEmittedSymbol = [&](const StringRef &Name) {
5226	AddSymbol(Name, getNewValueForSymbol(Name));
5227	};
5228
5229	if (opts::HotText && !NumHotTextSymsUpdated) {
5230	AddEmittedSymbol("__hot_start");
5231	AddEmittedSymbol("__hot_end");
5232	}
5233
5234	if (opts::HotData && !NumHotDataSymsUpdated) {
5235	AddEmittedSymbol("__hot_data_start");
5236	AddEmittedSymbol("__hot_data_end");
5237	}
5238
5239	// Put local symbols at the beginning.
5240	llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) {
5241	if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL)
5242	return true;
5243	return false;
5244	});
5245
5246	for (const ELFSymTy &Symbol : Symbols)
5247	Write(0, Symbol);
5248	}
5249
5250	template <typename ELFT>
5251	void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
5252	const ELFFile<ELFT> &Obj = File->getELFFile();
5253	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
5254	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
5255
5256	// Compute a preview of how section indices will change after rewriting, so
5257	// we can properly update the symbol table based on new section indices.
5258	std::vector<uint32_t> NewSectionIndex;
5259	getOutputSections(File, NewSectionIndex);
5260
5261	// Update dynamic symbol table.
5262	const ELFShdrTy *DynSymSection = nullptr;
5263	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
5264	if (Section.sh_type == ELF::SHT_DYNSYM) {
5265	DynSymSection = &Section;
5266	break;
5267	}
5268	}
5269	assert((DynSymSection || BC->IsStaticExecutable) &&
5270	"dynamic symbol table expected");
5271	if (DynSymSection) {
5272	updateELFSymbolTable(
5273	File,
5274	/*IsDynSym=*/true,
5275	*DynSymSection,
5276	NewSectionIndex,
5277	[&](size_t Offset, const ELFSymTy &Sym) {
5278	Out->os().pwrite(Ptr: reinterpret_cast<const char *>(&Sym),
5279	Size: sizeof(ELFSymTy),
5280	Offset: DynSymSection->sh_offset + Offset);
5281	},
5282	[](StringRef) -> size_t { return 0; });
5283	}
5284
5285	if (opts::RemoveSymtab)
5286	return;
5287
5288	// (re)create regular symbol table.
5289	const ELFShdrTy *SymTabSection = nullptr;
5290	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
5291	if (Section.sh_type == ELF::SHT_SYMTAB) {
5292	SymTabSection = &Section;
5293	break;
5294	}
5295	}
5296	if (!SymTabSection) {
5297	BC->errs() << "BOLT-WARNING: no symbol table found\n";
5298	return;
5299	}
5300
5301	const ELFShdrTy *StrTabSection =
5302	cantFail(Obj.getSection(SymTabSection->sh_link));
5303	std::string NewContents;
5304	std::string NewStrTab = std::string(
5305	File->getData().substr(StrTabSection->sh_offset, StrTabSection->sh_size));
5306	StringRef SecName = cantFail(Obj.getSectionName(*SymTabSection));
5307	StringRef StrSecName = cantFail(Obj.getSectionName(*StrTabSection));
5308
5309	NumLocalSymbols = 0;
5310	updateELFSymbolTable(
5311	File,
5312	/*IsDynSym=*/false,
5313	*SymTabSection,
5314	NewSectionIndex,
5315	[&](size_t Offset, const ELFSymTy &Sym) {
5316	if (Sym.getBinding() == ELF::STB_LOCAL)
5317	++NumLocalSymbols;
5318	NewContents.append(s: reinterpret_cast<const char *>(&Sym),
5319	n: sizeof(ELFSymTy));
5320	},
5321	[&](StringRef Str) {
5322	size_t Idx = NewStrTab.size();
5323	NewStrTab.append(str: NameResolver::restore(Name: Str).str());
5324	NewStrTab.append(n: 1, c: '\0');
5325	return Idx;
5326	});
5327
5328	BC->registerOrUpdateNoteSection(Name: SecName,
5329	Data: copyByteArray(Buffer: NewContents),
5330	Size: NewContents.size(),
5331	/*Alignment=*/1,
5332	/*IsReadOnly=*/true,
5333	ELFType: ELF::SHT_SYMTAB);
5334
5335	BC->registerOrUpdateNoteSection(Name: StrSecName,
5336	Data: copyByteArray(Buffer: NewStrTab),
5337	Size: NewStrTab.size(),
5338	/*Alignment=*/1,
5339	/*IsReadOnly=*/true,
5340	ELFType: ELF::SHT_STRTAB);
5341	}
5342
5343	template <typename ELFT>
5344	void RewriteInstance::patchELFAllocatableRelrSection(
5345	ELFObjectFile<ELFT> *File) {
5346	if (!DynamicRelrAddress)
5347	return;
5348
5349	raw_fd_ostream &OS = Out->os();
5350	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
5351	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
5352
5353	auto FixAddend = [&](const BinarySection &Section, const Relocation &Rel,
5354	uint64_t FileOffset) {
5355	// Fix relocation symbol value in place if no static relocation found
5356	// on the same address. We won't check the BF relocations here since it
5357	// is rare case and no optimization is required.
5358	if (Section.getRelocationAt(Offset: Rel.Offset))
5359	return;
5360
5361	// No fixup needed if symbol address was not changed
5362	const uint64_t Addend = getNewFunctionOrDataAddress(OldAddress: Rel.Addend);
5363	if (!Addend)
5364	return;
5365
5366	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Addend), Size: PSize, Offset: FileOffset);
5367	};
5368
5369	// Fill new relative relocation offsets set
5370	std::set<uint64_t> RelOffsets;
5371	for (const BinarySection &Section : BC->allocatableSections()) {
5372	const uint64_t SectionInputAddress = Section.getAddress();
5373	uint64_t SectionAddress = Section.getOutputAddress();
5374	if (!SectionAddress)
5375	SectionAddress = SectionInputAddress;
5376
5377	for (const Relocation &Rel : Section.dynamicRelocations()) {
5378	if (!Rel.isRelative())
5379	continue;
5380
5381	uint64_t RelOffset =
5382	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5383
5384	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5385	assert((RelOffset & 1) == 0 && "Wrong relocation offset");
5386	RelOffsets.emplace(args&: RelOffset);
5387	FixAddend(Section, Rel, RelOffset);
5388	}
5389	}
5390
5391	ErrorOr<BinarySection &> Section =
5392	BC->getSectionForAddress(Address: *DynamicRelrAddress);
5393	assert(Section && "cannot get .relr.dyn section");
5394	assert(Section->isRelr() && "Expected section to be SHT_RELR type");
5395	uint64_t RelrDynOffset = Section->getInputFileOffset();
5396	const uint64_t RelrDynEndOffset = RelrDynOffset + Section->getSize();
5397
5398	auto WriteRelr = [&](uint64_t Value) {
5399	if (RelrDynOffset + DynamicRelrEntrySize > RelrDynEndOffset) {
5400	BC->errs() << "BOLT-ERROR: Offset overflow for relr.dyn section\n";
5401	exit(status: 1);
5402	}
5403
5404	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Value), Size: DynamicRelrEntrySize,
5405	Offset: RelrDynOffset);
5406	RelrDynOffset += DynamicRelrEntrySize;
5407	};
5408
5409	for (auto RelIt = RelOffsets.begin(); RelIt != RelOffsets.end();) {
5410	WriteRelr(*RelIt);
5411	uint64_t Base = *RelIt++ + PSize;
5412	while (1) {
5413	uint64_t Bitmap = 0;
5414	for (; RelIt != RelOffsets.end(); ++RelIt) {
5415	const uint64_t Delta = *RelIt - Base;
5416	if (Delta >= MaxDelta || Delta % PSize)
5417	break;
5418
5419	Bitmap |= (1ULL << (Delta / PSize));
5420	}
5421
5422	if (!Bitmap)
5423	break;
5424
5425	WriteRelr((Bitmap << 1) | 1);
5426	Base += MaxDelta;
5427	}
5428	}
5429
5430	// Fill the rest of the section with empty bitmap value
5431	while (RelrDynOffset != RelrDynEndOffset)
5432	WriteRelr(1);
5433	}
5434
5435	template <typename ELFT>
5436	void
5437	RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile<ELFT> *File) {
5438	using Elf_Rela = typename ELFT::Rela;
5439	raw_fd_ostream &OS = Out->os();
5440	const ELFFile<ELFT> &EF = File->getELFFile();
5441
5442	uint64_t RelDynOffset = 0, RelDynEndOffset = 0;
5443	uint64_t RelPltOffset = 0, RelPltEndOffset = 0;
5444
5445	auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start,
5446	uint64_t &End) {
5447	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
5448	assert(Section && "cannot get relocation section");
5449	Start = Section->getInputFileOffset();
5450	End = Start + Section->getSize();
5451	};
5452
5453	if (!DynamicRelocationsAddress && !PLTRelocationsAddress)
5454	return;
5455
5456	if (DynamicRelocationsAddress)
5457	setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset,
5458	RelDynEndOffset);
5459
5460	if (PLTRelocationsAddress)
5461	setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset,
5462	RelPltEndOffset);
5463
5464	DynamicRelativeRelocationsCount = 0;
5465
5466	auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) {
5467	OS.pwrite(Ptr: reinterpret_cast<const char *>(RelA), Size: sizeof(*RelA), Offset);
5468	Offset += sizeof(*RelA);
5469	};
5470
5471	auto writeRelocations = [&](bool PatchRelative) {
5472	for (BinarySection &Section : BC->allocatableSections()) {
5473	const uint64_t SectionInputAddress = Section.getAddress();
5474	uint64_t SectionAddress = Section.getOutputAddress();
5475	if (!SectionAddress)
5476	SectionAddress = SectionInputAddress;
5477
5478	for (const Relocation &Rel : Section.dynamicRelocations()) {
5479	const bool IsRelative = Rel.isRelative();
5480	if (PatchRelative != IsRelative)
5481	continue;
5482
5483	if (IsRelative)
5484	++DynamicRelativeRelocationsCount;
5485
5486	Elf_Rela NewRelA;
5487	MCSymbol *Symbol = Rel.Symbol;
5488	uint32_t SymbolIdx = 0;
5489	uint64_t Addend = Rel.Addend;
5490	uint64_t RelOffset =
5491	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5492
5493	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5494	if (Rel.Symbol) {
5495	SymbolIdx = getOutputDynamicSymbolIndex(Symbol);
5496	} else {
5497	// Usually this case is used for R_*_(I)RELATIVE relocations
5498	const uint64_t Address = getNewFunctionOrDataAddress(OldAddress: Addend);
5499	if (Address)
5500	Addend = Address;
5501	}
5502
5503	NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL());
5504	NewRelA.r_offset = RelOffset;
5505	NewRelA.r_addend = Addend;
5506
5507	const bool IsJmpRel = IsJmpRelocation.contains(Val: Rel.Type);
5508	uint64_t &Offset = IsJmpRel ? RelPltOffset : RelDynOffset;
5509	const uint64_t &EndOffset =
5510	IsJmpRel ? RelPltEndOffset : RelDynEndOffset;
5511	if (!Offset || !EndOffset) {
5512	BC->errs() << "BOLT-ERROR: Invalid offsets for dynamic relocation\n";
5513	exit(status: 1);
5514	}
5515
5516	if (Offset + sizeof(NewRelA) > EndOffset) {
5517	BC->errs() << "BOLT-ERROR: Offset overflow for dynamic relocation\n";
5518	exit(status: 1);
5519	}
5520
5521	writeRela(&NewRelA, Offset);
5522	}
5523	}
5524	};
5525
5526	// Place R_*_RELATIVE relocations in RELA section if RELR is not presented.
5527	// The dynamic linker expects all R_*_RELATIVE relocations in RELA
5528	// to be emitted first.
5529	if (!DynamicRelrAddress)
5530	writeRelocations(/* PatchRelative */ true);
5531	writeRelocations(/* PatchRelative */ false);
5532
5533	auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) {
5534	if (!Offset)
5535	return;
5536
5537	typename ELFObjectFile<ELFT>::Elf_Rela RelA;
5538	RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL());
5539	RelA.r_offset = 0;
5540	RelA.r_addend = 0;
5541	while (Offset < EndOffset)
5542	writeRela(&RelA, Offset);
5543
5544	assert(Offset == EndOffset && "Unexpected section overflow");
5545	};
5546
5547	// Fill the rest of the sections with R_*_NONE relocations
5548	fillNone(RelDynOffset, RelDynEndOffset);
5549	fillNone(RelPltOffset, RelPltEndOffset);
5550	}
5551
5552	template <typename ELFT>
5553	void RewriteInstance::patchELFGOT(ELFObjectFile<ELFT> *File) {
5554	raw_fd_ostream &OS = Out->os();
5555
5556	SectionRef GOTSection;
5557	for (const SectionRef &Section : File->sections()) {
5558	StringRef SectionName = cantFail(ValOrErr: Section.getName());
5559	if (SectionName == ".got") {
5560	GOTSection = Section;
5561	break;
5562	}
5563	}
5564	if (!GOTSection.getObject()) {
5565	if (!BC->IsStaticExecutable)
5566	BC->errs() << "BOLT-INFO: no .got section found\n";
5567	return;
5568	}
5569
5570	StringRef GOTContents = cantFail(ValOrErr: GOTSection.getContents());
5571	for (const uint64_t *GOTEntry =
5572	reinterpret_cast<const uint64_t *>(GOTContents.data());
5573	GOTEntry < reinterpret_cast<const uint64_t *>(GOTContents.data() +
5574	GOTContents.size());
5575	++GOTEntry) {
5576	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: *GOTEntry)) {
5577	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x"
5578	<< Twine::utohexstr(*GOTEntry) << " with 0x"
5579	<< Twine::utohexstr(NewAddress) << '\n');
5580	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewAddress), Size: sizeof(NewAddress),
5581	Offset: reinterpret_cast<const char *>(GOTEntry) -
5582	File->getData().data());
5583	}
5584	}
5585	}
5586
5587	template <typename ELFT>
5588	void RewriteInstance::patchELFDynamic(ELFObjectFile<ELFT> *File) {
5589	if (BC->IsStaticExecutable)
5590	return;
5591
5592	const ELFFile<ELFT> &Obj = File->getELFFile();
5593	raw_fd_ostream &OS = Out->os();
5594
5595	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5596	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5597
5598	// Locate DYNAMIC by looking through program headers.
5599	uint64_t DynamicOffset = 0;
5600	const Elf_Phdr *DynamicPhdr = nullptr;
5601	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5602	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5603	DynamicOffset = Phdr.p_offset;
5604	DynamicPhdr = &Phdr;
5605	assert(Phdr.p_memsz == Phdr.p_filesz && "dynamic sizes should match");
5606	break;
5607	}
5608	}
5609	assert(DynamicPhdr && "missing dynamic in ELF binary");
5610
5611	bool ZNowSet = false;
5612
5613	// Go through all dynamic entries and patch functions addresses with
5614	// new ones.
5615	typename ELFT::DynRange DynamicEntries =
5616	cantFail(Obj.dynamicEntries(), "error accessing dynamic table");
5617	auto DTB = DynamicEntries.begin();
5618	for (const Elf_Dyn &Dyn : DynamicEntries) {
5619	Elf_Dyn NewDE = Dyn;
5620	bool ShouldPatch = true;
5621	switch (Dyn.d_tag) {
5622	default:
5623	ShouldPatch = false;
5624	break;
5625	case ELF::DT_RELACOUNT:
5626	NewDE.d_un.d_val = DynamicRelativeRelocationsCount;
5627	break;
5628	case ELF::DT_INIT:
5629	case ELF::DT_FINI: {
5630	if (BC->HasRelocations) {
5631	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: Dyn.getPtr())) {
5632	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching dynamic entry of type "
5633	<< Dyn.getTag() << '\n');
5634	NewDE.d_un.d_ptr = NewAddress;
5635	}
5636	}
5637	RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary();
5638	if (RtLibrary && Dyn.getTag() == ELF::DT_FINI) {
5639	if (uint64_t Addr = RtLibrary->getRuntimeFiniAddress())
5640	NewDE.d_un.d_ptr = Addr;
5641	}
5642	if (RtLibrary && Dyn.getTag() == ELF::DT_INIT && !BC->HasInterpHeader) {
5643	if (auto Addr = RtLibrary->getRuntimeStartAddress()) {
5644	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set DT_INIT to 0x"
5645	<< Twine::utohexstr(Addr) << '\n');
5646	NewDE.d_un.d_ptr = Addr;
5647	}
5648	}
5649	break;
5650	}
5651	case ELF::DT_FLAGS:
5652	if (BC->RequiresZNow) {
5653	NewDE.d_un.d_val |= ELF::DF_BIND_NOW;
5654	ZNowSet = true;
5655	}
5656	break;
5657	case ELF::DT_FLAGS_1:
5658	if (BC->RequiresZNow) {
5659	NewDE.d_un.d_val |= ELF::DF_1_NOW;
5660	ZNowSet = true;
5661	}
5662	break;
5663	}
5664	if (ShouldPatch)
5665	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewDE), Size: sizeof(NewDE),
5666	Offset: DynamicOffset + (&Dyn - DTB) * sizeof(Dyn));
5667	}
5668
5669	if (BC->RequiresZNow && !ZNowSet) {
5670	BC->errs()
5671	<< "BOLT-ERROR: output binary requires immediate relocation "
5672	"processing which depends on DT_FLAGS or DT_FLAGS_1 presence in "
5673	".dynamic. Please re-link the binary with -znow.\n";
5674	exit(status: 1);
5675	}
5676	}
5677
5678	template <typename ELFT>
5679	Error RewriteInstance::readELFDynamic(ELFObjectFile<ELFT> *File) {
5680	const ELFFile<ELFT> &Obj = File->getELFFile();
5681
5682	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5683	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5684
5685	// Locate DYNAMIC by looking through program headers.
5686	const Elf_Phdr *DynamicPhdr = nullptr;
5687	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5688	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5689	DynamicPhdr = &Phdr;
5690	break;
5691	}
5692	}
5693
5694	if (!DynamicPhdr) {
5695	BC->outs() << "BOLT-INFO: static input executable detected\n";
5696	// TODO: static PIE executable might have dynamic header
5697	BC->IsStaticExecutable = true;
5698	return Error::success();
5699	}
5700
5701	if (DynamicPhdr->p_memsz != DynamicPhdr->p_filesz)
5702	return createStringError(EC: errc::executable_format_error,
5703	S: "dynamic section sizes should match");
5704
5705	// Go through all dynamic entries to locate entries of interest.
5706	auto DynamicEntriesOrErr = Obj.dynamicEntries();
5707	if (!DynamicEntriesOrErr)
5708	return DynamicEntriesOrErr.takeError();
5709	typename ELFT::DynRange DynamicEntries = DynamicEntriesOrErr.get();
5710
5711	for (const Elf_Dyn &Dyn : DynamicEntries) {
5712	switch (Dyn.d_tag) {
5713	case ELF::DT_INIT:
5714	if (!BC->HasInterpHeader) {
5715	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set start function address\n");
5716	BC->StartFunctionAddress = Dyn.getPtr();
5717	}
5718	break;
5719	case ELF::DT_FINI:
5720	BC->FiniAddress = Dyn.getPtr();
5721	break;
5722	case ELF::DT_FINI_ARRAY:
5723	BC->FiniArrayAddress = Dyn.getPtr();
5724	break;
5725	case ELF::DT_FINI_ARRAYSZ:
5726	BC->FiniArraySize = Dyn.getPtr();
5727	break;
5728	case ELF::DT_RELA:
5729	DynamicRelocationsAddress = Dyn.getPtr();
5730	break;
5731	case ELF::DT_RELASZ:
5732	DynamicRelocationsSize = Dyn.getVal();
5733	break;
5734	case ELF::DT_JMPREL:
5735	PLTRelocationsAddress = Dyn.getPtr();
5736	break;
5737	case ELF::DT_PLTRELSZ:
5738	PLTRelocationsSize = Dyn.getVal();
5739	break;
5740	case ELF::DT_RELACOUNT:
5741	DynamicRelativeRelocationsCount = Dyn.getVal();
5742	break;
5743	case ELF::DT_RELR:
5744	DynamicRelrAddress = Dyn.getPtr();
5745	break;
5746	case ELF::DT_RELRSZ:
5747	DynamicRelrSize = Dyn.getVal();
5748	break;
5749	case ELF::DT_RELRENT:
5750	DynamicRelrEntrySize = Dyn.getVal();
5751	break;
5752	}
5753	}
5754
5755	if (!DynamicRelocationsAddress || !DynamicRelocationsSize) {
5756	DynamicRelocationsAddress.reset();
5757	DynamicRelocationsSize = 0;
5758	}
5759
5760	if (!PLTRelocationsAddress || !PLTRelocationsSize) {
5761	PLTRelocationsAddress.reset();
5762	PLTRelocationsSize = 0;
5763	}
5764
5765	if (!DynamicRelrAddress || !DynamicRelrSize) {
5766	DynamicRelrAddress.reset();
5767	DynamicRelrSize = 0;
5768	} else if (!DynamicRelrEntrySize) {
5769	BC->errs() << "BOLT-ERROR: expected DT_RELRENT to be presented "
5770	<< "in DYNAMIC section\n";
5771	exit(status: 1);
5772	} else if (DynamicRelrSize % DynamicRelrEntrySize) {
5773	BC->errs() << "BOLT-ERROR: expected RELR table size to be divisible "
5774	<< "by RELR entry size\n";
5775	exit(status: 1);
5776	}
5777
5778	return Error::success();
5779	}
5780
5781	uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {
5782	const BinaryFunction *Function = BC->getBinaryFunctionAtAddress(Address: OldAddress);
5783	if (!Function)
5784	return 0;
5785
5786	return Function->getOutputAddress();
5787	}
5788
5789	uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) {
5790	if (uint64_t Function = getNewFunctionAddress(OldAddress))
5791	return Function;
5792
5793	const BinaryData *BD = BC->getBinaryDataAtAddress(Address: OldAddress);
5794	if (BD && BD->isMoved())
5795	return BD->getOutputAddress();
5796
5797	if (const BinaryFunction *BF =
5798	BC->getBinaryFunctionContainingAddress(Address: OldAddress)) {
5799	if (BF->isEmitted()) {
5800	// If OldAddress is the another entry point of
5801	// the function, then BOLT could get the new address.
5802	if (BF->isMultiEntry()) {
5803	for (const BinaryBasicBlock &BB : *BF)
5804	if (BB.isEntryPoint() &&
5805	(BF->getAddress() + BB.getOffset()) == OldAddress)
5806	return BB.getOutputStartAddress();
5807	}
5808	BC->errs() << "BOLT-ERROR: unable to get new address corresponding to "
5809	"input address 0x"
5810	<< Twine::utohexstr(Val: OldAddress) << " in function " << *BF
5811	<< ". Consider adding this function to --skip-funcs=...\n";
5812	exit(status: 1);
5813	}
5814	}
5815
5816	return 0;
5817	}
5818
5819	void RewriteInstance::rewriteFile() {
5820	std::error_code EC;
5821	Out = std::make_unique<ToolOutputFile>(args&: opts::OutputFilename, args&: EC,
5822	args: sys::fs::OF_None);
5823	check_error(EC, Message: "cannot create output executable file");
5824
5825	raw_fd_ostream &OS = Out->os();
5826
5827	// Copy allocatable part of the input.
5828	OS << InputFile->getData().substr(Start: 0, N: FirstNonAllocatableOffset);
5829
5830	auto Streamer = BC->createStreamer(OS);
5831	// Make sure output stream has enough reserved space, otherwise
5832	// pwrite() will fail.
5833	uint64_t Offset = std::max(a: getFileOffsetForAddress(Address: NextAvailableAddress),
5834	b: FirstNonAllocatableOffset);
5835	Offset = OS.seek(off: Offset);
5836	assert((Offset != (uint64_t)-1) && "Error resizing output file");
5837
5838	// Overwrite functions with fixed output address. This is mostly used by
5839	// non-relocation mode, with one exception: injected functions are covered
5840	// here in both modes.
5841	uint64_t CountOverwrittenFunctions = 0;
5842	uint64_t OverwrittenScore = 0;
5843	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
5844	if (Function->getImageAddress() == 0 || Function->getImageSize() == 0)
5845	continue;
5846
5847	assert(Function->getImageSize() <= Function->getMaxSize() &&
5848	"Unexpected large function");
5849
5850	const auto HasAddress = [](const FunctionFragment &FF) {
5851	return FF.empty() ||
5852	(FF.getImageAddress() != 0 && FF.getImageSize() != 0);
5853	};
5854	const bool SplitFragmentsHaveAddress =
5855	llvm::all_of(Range: Function->getLayout().getSplitFragments(), P: HasAddress);
5856	if (Function->isSplit() && !SplitFragmentsHaveAddress) {
5857	const auto HasNoAddress = [](const FunctionFragment &FF) {
5858	return FF.getImageAddress() == 0 && FF.getImageSize() == 0;
5859	};
5860	assert(llvm::all_of(Function->getLayout().getSplitFragments(),
5861	HasNoAddress) &&
5862	"Some split fragments have an address while others do not");
5863	(void)HasNoAddress;
5864	continue;
5865	}
5866
5867	OverwrittenScore += Function->getFunctionScore();
5868	++CountOverwrittenFunctions;
5869
5870	// Overwrite function in the output file.
5871	if (opts::Verbosity >= 2)
5872	BC->outs() << "BOLT: rewriting function \"" << *Function << "\"\n";
5873
5874	OS.pwrite(Ptr: reinterpret_cast<char *>(Function->getImageAddress()),
5875	Size: Function->getImageSize(), Offset: Function->getFileOffset());
5876
5877	// Write nops at the end of the function.
5878	if (Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {
5879	uint64_t Pos = OS.tell();
5880	OS.seek(off: Function->getFileOffset() + Function->getImageSize());
5881	BC->MAB->writeNopData(
5882	OS, Count: Function->getMaxSize() - Function->getImageSize(), STI: &*BC->STI);
5883
5884	OS.seek(off: Pos);
5885	}
5886
5887	if (!Function->isSplit())
5888	continue;
5889
5890	// Write cold part
5891	if (opts::Verbosity >= 2) {
5892	BC->outs() << formatv(Fmt: "BOLT: rewriting function \"{0}\" (split parts)\n",
5893	Vals&: *Function);
5894	}
5895
5896	for (const FunctionFragment &FF :
5897	Function->getLayout().getSplitFragments()) {
5898	OS.pwrite(Ptr: reinterpret_cast<char *>(FF.getImageAddress()),
5899	Size: FF.getImageSize(), Offset: FF.getFileOffset());
5900	}
5901	}
5902
5903	// Print function statistics for non-relocation mode.
5904	if (!BC->HasRelocations) {
5905	BC->outs() << "BOLT: " << CountOverwrittenFunctions << " out of "
5906	<< BC->getBinaryFunctions().size()
5907	<< " functions were overwritten.\n";
5908	if (BC->TotalScore != 0) {
5909	double Coverage = OverwrittenScore / (double)BC->TotalScore * 100.0;
5910	BC->outs() << format(Fmt: "BOLT-INFO: rewritten functions cover %.2lf",
5911	Vals: Coverage)
5912	<< "% of the execution count of simple functions of "
5913	"this binary\n";
5914	}
5915	}
5916
5917	if (BC->HasRelocations && opts::TrapOldCode) {
5918	uint64_t SavedPos = OS.tell();
5919	// Overwrite function body to make sure we never execute these instructions.
5920	for (auto &BFI : BC->getBinaryFunctions()) {
5921	BinaryFunction &BF = BFI.second;
5922	if (!BF.getFileOffset() || !BF.isEmitted())
5923	continue;
5924	OS.seek(off: BF.getFileOffset());
5925	StringRef TrapInstr = BC->MIB->getTrapFillValue();
5926	unsigned NInstr = BF.getMaxSize() / TrapInstr.size();
5927	for (unsigned I = 0; I < NInstr; ++I)
5928	OS.write(Ptr: TrapInstr.data(), Size: TrapInstr.size());
5929	}
5930	OS.seek(off: SavedPos);
5931	}
5932
5933	// Write all allocatable sections - reloc-mode text is written here as well
5934	for (BinarySection &Section : BC->allocatableSections()) {
5935	if (!Section.isFinalized() || !Section.getOutputData()) {
5936	LLVM_DEBUG(if (opts::Verbosity > 1) {
5937	dbgs() << "BOLT-INFO: new section is finalized or !getOutputData, skip "
5938	<< Section.getName() << '\n';
5939	});
5940	continue;
5941	}
5942	if (Section.isLinkOnly()) {
5943	LLVM_DEBUG(if (opts::Verbosity > 1) {
5944	dbgs() << "BOLT-INFO: new section is link only, skip "
5945	<< Section.getName() << '\n';
5946	});
5947	continue;
5948	}
5949
5950	if (opts::Verbosity >= 1)
5951	BC->outs() << "BOLT: writing new section " << Section.getName()
5952	<< "\n data at 0x"
5953	<< Twine::utohexstr(Val: Section.getAllocAddress()) << "\n of size "
5954	<< Section.getOutputSize() << "\n at offset "
5955	<< Section.getOutputFileOffset() << " with content size "
5956	<< Section.getOutputContents().size() << '\n';
5957	OS.seek(off: Section.getOutputFileOffset());
5958	Section.write(OS);
5959	}
5960
5961	for (BinarySection &Section : BC->allocatableSections())
5962	Section.flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
5963	return getNewValueForSymbol(Name: S->getName());
5964	});
5965
5966	// If .eh_frame is present create .eh_frame_hdr.
5967	if (EHFrameSection)
5968	writeEHFrameHeader();
5969
5970	// Add BOLT Addresses Translation maps to allow profile collection to
5971	// happen in the output binary
5972	if (opts::EnableBAT)
5973	addBATSection();
5974
5975	// Patch program header table.
5976	if (!BC->IsLinuxKernel) {
5977	updateSegmentInfo();
5978	patchELFPHDRTable();
5979	}
5980
5981	// Finalize memory image of section string table.
5982	finalizeSectionStringTable();
5983
5984	// Update symbol tables.
5985	patchELFSymTabs();
5986
5987	if (opts::EnableBAT)
5988	encodeBATSection();
5989
5990	// Copy non-allocatable sections once allocatable part is finished.
5991	rewriteNoteSections();
5992
5993	if (BC->HasRelocations) {
5994	patchELFAllocatableRelaSections();
5995	patchELFAllocatableRelrSection();
5996	patchELFGOT();
5997	}
5998
5999	// Patch dynamic section/segment.
6000	patchELFDynamic();
6001
6002	// Update ELF book-keeping info.
6003	patchELFSectionHeaderTable();
6004
6005	if (opts::PrintSections) {
6006	BC->outs() << "BOLT-INFO: Sections after processing:\n";
6007	BC->printSections(OS&: BC->outs());
6008	}
6009
6010	Out->keep();
6011	EC = sys::fs::setPermissions(
6012	Path: opts::OutputFilename,
6013	Permissions: static_cast<sys::fs::perms>(sys::fs::perms::all_all &
6014	~sys::fs::getUmask()));
6015	check_error(EC, Message: "cannot set permissions of output file");
6016	}
6017
6018	void RewriteInstance::writeEHFrameHeader() {
6019	BinarySection *NewEHFrameSection =
6020	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
6021
6022	// No need to update the header if no new .eh_frame was created.
6023	if (!NewEHFrameSection)
6024	return;
6025
6026	DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true,
6027	NewEHFrameSection->getOutputAddress());
6028	Error E = NewEHFrame.parse(Data: DWARFDataExtractor(
6029	NewEHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(),
6030	BC->AsmInfo->getCodePointerSize()));
6031	check_error(E: std::move(E), Message: "failed to parse EH frame");
6032
6033	uint64_t RelocatedEHFrameAddress = 0;
6034	StringRef RelocatedEHFrameContents;
6035	BinarySection *RelocatedEHFrameSection =
6036	getSection(Name: ".relocated" + getEHFrameSectionName());
6037	if (RelocatedEHFrameSection) {
6038	RelocatedEHFrameAddress = RelocatedEHFrameSection->getOutputAddress();
6039	RelocatedEHFrameContents = RelocatedEHFrameSection->getOutputContents();
6040	}
6041	DWARFDebugFrame RelocatedEHFrame(BC->TheTriple->getArch(), true,
6042	RelocatedEHFrameAddress);
6043	Error Er = RelocatedEHFrame.parse(Data: DWARFDataExtractor(
6044	RelocatedEHFrameContents, BC->AsmInfo->isLittleEndian(),
6045	BC->AsmInfo->getCodePointerSize()));
6046	check_error(E: std::move(Er), Message: "failed to parse EH frame");
6047
6048	LLVM_DEBUG(dbgs() << "BOLT: writing a new " << getEHFrameHdrSectionName()
6049	<< '\n');
6050
6051	// Try to overwrite the original .eh_frame_hdr if the size permits.
6052	uint64_t EHFrameHdrOutputAddress = 0;
6053	uint64_t EHFrameHdrFileOffset = 0;
6054	std::vector<char> NewEHFrameHdr;
6055	BinarySection *OldEHFrameHdrSection = getSection(Name: getEHFrameHdrSectionName());
6056	if (OldEHFrameHdrSection) {
6057	NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
6058	OldEHFrame: RelocatedEHFrame, NewEHFrame, EHFrameHeaderAddress: OldEHFrameHdrSection->getAddress());
6059	if (NewEHFrameHdr.size() <= OldEHFrameHdrSection->getSize()) {
6060	BC->outs() << "BOLT-INFO: rewriting " << getEHFrameHdrSectionName()
6061	<< " in-place\n";
6062	EHFrameHdrOutputAddress = OldEHFrameHdrSection->getAddress();
6063	EHFrameHdrFileOffset = OldEHFrameHdrSection->getInputFileOffset();
6064	} else {
6065	OldEHFrameHdrSection->setOutputName(getOrgSecPrefix() +
6066	getEHFrameHdrSectionName());
6067	OldEHFrameHdrSection = nullptr;
6068	}
6069	}
6070
6071	// If there was not enough space, allocate more memory for .eh_frame_hdr.
6072	if (!OldEHFrameHdrSection) {
6073	NextAvailableAddress =
6074	appendPadding(OS&: Out->os(), Offset: NextAvailableAddress, Alignment: EHFrameHdrAlign);
6075
6076	EHFrameHdrOutputAddress = NextAvailableAddress;
6077	EHFrameHdrFileOffset = getFileOffsetForAddress(Address: NextAvailableAddress);
6078
6079	NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
6080	OldEHFrame: RelocatedEHFrame, NewEHFrame, EHFrameHeaderAddress: EHFrameHdrOutputAddress);
6081
6082	NextAvailableAddress += NewEHFrameHdr.size();
6083	if (!BC->BOLTReserved.empty() &&
6084	(NextAvailableAddress > BC->BOLTReserved.end())) {
6085	BC->errs() << "BOLT-ERROR: unable to fit " << getEHFrameHdrSectionName()
6086	<< " into reserved space\n";
6087	exit(status: 1);
6088	}
6089
6090	// Create a new entry in the section header table.
6091	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
6092	/*IsText=*/false,
6093	/*IsAllocatable=*/true);
6094	BinarySection &EHFrameHdrSec = BC->registerOrUpdateSection(
6095	Name: getNewSecPrefix() + getEHFrameHdrSectionName(), ELFType: ELF::SHT_PROGBITS,
6096	ELFFlags: Flags, Data: nullptr, Size: NewEHFrameHdr.size(), /*Alignment=*/1);
6097	EHFrameHdrSec.setOutputFileOffset(EHFrameHdrFileOffset);
6098	EHFrameHdrSec.setOutputAddress(EHFrameHdrOutputAddress);
6099	EHFrameHdrSec.setOutputName(getEHFrameHdrSectionName());
6100	}
6101
6102	Out->os().seek(off: EHFrameHdrFileOffset);
6103	Out->os().write(Ptr: NewEHFrameHdr.data(), Size: NewEHFrameHdr.size());
6104
6105	// Pad the contents if overwriting in-place.
6106	if (OldEHFrameHdrSection)
6107	Out->os().write_zeros(NumZeros: OldEHFrameHdrSection->getSize() -
6108	NewEHFrameHdr.size());
6109
6110	// Merge new .eh_frame with the relocated original so that gdb can locate all
6111	// FDEs.
6112	if (RelocatedEHFrameSection) {
6113	const uint64_t NewEHFrameSectionSize =
6114	RelocatedEHFrameSection->getOutputAddress() +
6115	RelocatedEHFrameSection->getOutputSize() -
6116	NewEHFrameSection->getOutputAddress();
6117	NewEHFrameSection->updateContents(NewData: NewEHFrameSection->getOutputData(),
6118	NewSize: NewEHFrameSectionSize);
6119	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
6120	}
6121
6122	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of .eh_frame after merge is "
6123	<< NewEHFrameSection->getOutputSize() << '\n');
6124	}
6125
6126	uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) {
6127	auto Value = Linker->lookupSymbolInfo(Name);
6128	if (Value)
6129	return Value->Address;
6130
6131	// Return the original value if we haven't emitted the symbol.
6132	BinaryData *BD = BC->getBinaryDataByName(Name);
6133	if (!BD)
6134	return 0;
6135
6136	return BD->getAddress();
6137	}
6138
6139	uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const {
6140	// Check if it's possibly part of the new segment.
6141	if (NewTextSegmentAddress && Address >= NewTextSegmentAddress)
6142	return Address - NewTextSegmentAddress + NewTextSegmentOffset;
6143
6144	// Find an existing segment that matches the address.
6145	const auto SegmentInfoI = BC->SegmentMapInfo.upper_bound(x: Address);
6146	if (SegmentInfoI == BC->SegmentMapInfo.begin())
6147	return 0;
6148
6149	const SegmentInfo &SegmentInfo = std::prev(x: SegmentInfoI)->second;
6150	if (Address < SegmentInfo.Address ||
6151	Address >= SegmentInfo.Address + SegmentInfo.FileSize)
6152	return 0;
6153
6154	return SegmentInfo.FileOffset + Address - SegmentInfo.Address;
6155	}
6156
6157	bool RewriteInstance::willOverwriteSection(StringRef SectionName) {
6158	if (llvm::is_contained(Range: SectionsToOverwrite, Element: SectionName))
6159	return true;
6160	if (llvm::is_contained(Range&: DebugSectionsToOverwrite, Element: SectionName))
6161	return true;
6162
6163	ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
6164	return Section && Section->isAllocatable() && Section->isFinalized();
6165	}
6166
6167	bool RewriteInstance::isDebugSection(StringRef SectionName) {
6168	if (SectionName.starts_with(Prefix: ".debug_") ||
6169	SectionName.starts_with(Prefix: ".zdebug_") || SectionName == ".gdb_index" ||
6170	SectionName == ".stab" || SectionName == ".stabstr")
6171	return true;
6172
6173	return false;
6174	}
6175