1	//===- bolt/Rewrite/RewriteInstance.cpp - ELF rewriter --------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "bolt/Rewrite/RewriteInstance.h"
10	#include "bolt/Core/AddressMap.h"
11	#include "bolt/Core/BinaryContext.h"
12	#include "bolt/Core/BinaryEmitter.h"
13	#include "bolt/Core/BinaryFunction.h"
14	#include "bolt/Core/DebugData.h"
15	#include "bolt/Core/Exceptions.h"
16	#include "bolt/Core/FunctionLayout.h"
17	#include "bolt/Core/MCPlusBuilder.h"
18	#include "bolt/Core/ParallelUtilities.h"
19	#include "bolt/Core/Relocation.h"
20	#include "bolt/Passes/BinaryPasses.h"
21	#include "bolt/Passes/CacheMetrics.h"
22	#include "bolt/Passes/IdenticalCodeFolding.h"
23	#include "bolt/Passes/PAuthGadgetScanner.h"
24	#include "bolt/Passes/ReorderFunctions.h"
25	#include "bolt/Profile/BoltAddressTranslation.h"
26	#include "bolt/Profile/DataAggregator.h"
27	#include "bolt/Profile/DataReader.h"
28	#include "bolt/Profile/YAMLProfileReader.h"
29	#include "bolt/Profile/YAMLProfileWriter.h"
30	#include "bolt/Rewrite/BinaryPassManager.h"
31	#include "bolt/Rewrite/DWARFRewriter.h"
32	#include "bolt/Rewrite/ExecutableFileMemoryManager.h"
33	#include "bolt/Rewrite/JITLinkLinker.h"
34	#include "bolt/Rewrite/MetadataRewriters.h"
35	#include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h"
36	#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
37	#include "bolt/Utils/CommandLineOpts.h"
38	#include "bolt/Utils/Utils.h"
39	#include "llvm/ADT/AddressRanges.h"
40	#include "llvm/ADT/STLExtras.h"
41	#include "llvm/DebugInfo/DWARF/DWARFContext.h"
42	#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
43	#include "llvm/MC/MCAsmBackend.h"
44	#include "llvm/MC/MCAsmInfo.h"
45	#include "llvm/MC/MCDisassembler/MCDisassembler.h"
46	#include "llvm/MC/MCObjectStreamer.h"
47	#include "llvm/MC/MCStreamer.h"
48	#include "llvm/MC/MCSymbol.h"
49	#include "llvm/MC/TargetRegistry.h"
50	#include "llvm/Object/ObjectFile.h"
51	#include "llvm/Support/Alignment.h"
52	#include "llvm/Support/Casting.h"
53	#include "llvm/Support/CommandLine.h"
54	#include "llvm/Support/DataExtractor.h"
55	#include "llvm/Support/Errc.h"
56	#include "llvm/Support/Error.h"
57	#include "llvm/Support/FileSystem.h"
58	#include "llvm/Support/ManagedStatic.h"
59	#include "llvm/Support/Timer.h"
60	#include "llvm/Support/ToolOutputFile.h"
61	#include "llvm/Support/raw_ostream.h"
62	#include <algorithm>
63	#include <fstream>
64	#include <memory>
65	#include <optional>
66	#include <system_error>
67
68	#undef DEBUG_TYPE
69	#define DEBUG_TYPE "bolt"
70
71	using namespace llvm;
72	using namespace object;
73	using namespace bolt;
74
75	extern cl::opt<uint32_t> X86AlignBranchBoundary;
76	extern cl::opt<bool> X86AlignBranchWithin32BBoundaries;
77
78	namespace opts {
79
80	extern cl::list<std::string> HotTextMoveSections;
81	extern cl::opt<bool> Hugify;
82	extern cl::opt<bool> Instrument;
83	extern cl::opt<bool> KeepNops;
84	extern cl::opt<bool> Lite;
85	extern cl::list<std::string> ReorderData;
86	extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
87	extern cl::opt<bool> TerminalTrap;
88	extern cl::opt<bool> TimeBuild;
89	extern cl::opt<bool> TimeRewrite;
90	extern cl::opt<bolt::IdenticalCodeFolding::ICFLevel, false,
91	llvm::bolt::DeprecatedICFNumericOptionParser>
92	ICF;
93
94	static cl::opt<bool>
95	AllowStripped("allow-stripped",
96	cl::desc("allow processing of stripped binaries"), cl::Hidden,
97	cl::cat(BoltCategory));
98
99	static cl::opt<bool> ForceToDataRelocations(
100	"force-data-relocations",
101	cl::desc("force relocations to data sections to always be processed"),
102
103	cl::Hidden, cl::cat(BoltCategory));
104
105	static cl::opt<std::string>
106	BoltID("bolt-id",
107	cl::desc("add any string to tag this execution in the "
108	"output binary via bolt info section"),
109	cl::cat(BoltCategory));
110
111	cl::opt<bool> DumpDotAll(
112	"dump-dot-all",
113	cl::desc("dump function CFGs to graphviz format after each stage;"
114	"enable '-print-loops' for color-coded blocks"),
115	cl::Hidden, cl::cat(BoltCategory));
116
117	static cl::list<std::string>
118	ForceFunctionNames("funcs",
119	cl::CommaSeparated,
120	cl::desc("limit optimizations to functions from the list"),
121	cl::value_desc("func1,func2,func3,..."),
122	cl::Hidden,
123	cl::cat(BoltCategory));
124
125	static cl::opt<std::string>
126	FunctionNamesFile("funcs-file",
127	cl::desc("file with list of functions to optimize"),
128	cl::Hidden,
129	cl::cat(BoltCategory));
130
131	static cl::list<std::string> ForceFunctionNamesNR(
132	"funcs-no-regex", cl::CommaSeparated,
133	cl::desc("limit optimizations to functions from the list (non-regex)"),
134	cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));
135
136	static cl::opt<std::string> FunctionNamesFileNR(
137	"funcs-file-no-regex",
138	cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,
139	cl::cat(BoltCategory));
140
141	cl::opt<bool>
142	KeepTmp("keep-tmp",
143	cl::desc("preserve intermediate .o file"),
144	cl::Hidden,
145	cl::cat(BoltCategory));
146
147	static cl::opt<unsigned>
148	LiteThresholdPct("lite-threshold-pct",
149	cl::desc("threshold (in percent) for selecting functions to process in lite "
150	"mode. Higher threshold means fewer functions to process. E.g "
151	"threshold of 90 means only top 10 percent of functions with "
152	"profile will be processed."),
153	cl::init(Val: 0),
154	cl::ZeroOrMore,
155	cl::Hidden,
156	cl::cat(BoltOptCategory));
157
158	static cl::opt<unsigned> LiteThresholdCount(
159	"lite-threshold-count",
160	cl::desc("similar to '-lite-threshold-pct' but specify threshold using "
161	"absolute function call count. I.e. limit processing to functions "
162	"executed at least the specified number of times."),
163	cl::init(Val: 0), cl::Hidden, cl::cat(BoltOptCategory));
164
165	static cl::opt<unsigned>
166	MaxFunctions("max-funcs",
167	cl::desc("maximum number of functions to process"), cl::Hidden,
168	cl::cat(BoltCategory));
169
170	static cl::opt<unsigned> MaxDataRelocations(
171	"max-data-relocations",
172	cl::desc("maximum number of data relocations to process"), cl::Hidden,
173	cl::cat(BoltCategory));
174
175	cl::opt<bool> PrintAll("print-all",
176	cl::desc("print functions after each stage"), cl::Hidden,
177	cl::cat(BoltCategory));
178
179	static cl::opt<bool>
180	PrintProfile("print-profile",
181	cl::desc("print functions after attaching profile"),
182	cl::Hidden, cl::cat(BoltCategory));
183
184	cl::opt<bool> PrintCFG("print-cfg",
185	cl::desc("print functions after CFG construction"),
186	cl::Hidden, cl::cat(BoltCategory));
187
188	cl::opt<bool> PrintDisasm("print-disasm",
189	cl::desc("print function after disassembly"),
190	cl::Hidden, cl::cat(BoltCategory));
191
192	static cl::opt<bool>
193	PrintGlobals("print-globals",
194	cl::desc("print global symbols after disassembly"), cl::Hidden,
195	cl::cat(BoltCategory));
196
197	extern cl::opt<bool> PrintSections;
198
199	static cl::opt<bool> PrintLoopInfo("print-loops",
200	cl::desc("print loop related information"),
201	cl::Hidden, cl::cat(BoltCategory));
202
203	static cl::opt<cl::boolOrDefault> RelocationMode(
204	"relocs", cl::desc("use relocations in the binary (default=autodetect)"),
205	cl::cat(BoltCategory));
206
207	extern cl::opt<std::string> SaveProfile;
208
209	static cl::list<std::string>
210	SkipFunctionNames("skip-funcs",
211	cl::CommaSeparated,
212	cl::desc("list of functions to skip"),
213	cl::value_desc("func1,func2,func3,..."),
214	cl::Hidden,
215	cl::cat(BoltCategory));
216
217	static cl::opt<std::string>
218	SkipFunctionNamesFile("skip-funcs-file",
219	cl::desc("file with list of functions to skip"),
220	cl::Hidden,
221	cl::cat(BoltCategory));
222
223	static cl::opt<bool> TrapOldCode(
224	"trap-old-code",
225	cl::desc("insert traps in old function bodies (relocation mode)"),
226	cl::Hidden, cl::cat(BoltCategory));
227
228	static cl::opt<std::string> DWPPathName("dwp",
229	cl::desc("Path and name to DWP file."),
230	cl::Hidden, cl::init(Val: ""),
231	cl::cat(BoltCategory));
232
233	static cl::opt<bool>
234	UseGnuStack("use-gnu-stack",
235	cl::desc("use GNU_STACK program header for new segment (workaround for "
236	"issues with strip/objcopy)"),
237	cl::ZeroOrMore,
238	cl::cat(BoltCategory));
239
240	static cl::opt<uint64_t> CustomAllocationVMA(
241	"custom-allocation-vma",
242	cl::desc("use a custom address at which new code will be put, "
243	"bypassing BOLT's logic to detect where to put code"),
244	cl::Hidden, cl::cat(BoltCategory));
245
246	static cl::opt<bool>
247	SequentialDisassembly("sequential-disassembly",
248	cl::desc("performs disassembly sequentially"),
249	cl::init(Val: false),
250	cl::cat(BoltOptCategory));
251
252	static cl::opt<bool> WriteBoltInfoSection(
253	"bolt-info", cl::desc("write bolt info section in the output binary"),
254	cl::init(Val: true), cl::Hidden, cl::cat(BoltOutputCategory));
255
256	cl::bits<GadgetScannerKind> GadgetScannersToRun(
257	"scanners", cl::desc("which gadget scanners to run"),
258	cl::values(
259	clEnumValN(GS_PACRET, "pacret",
260	"pac-ret: return address protection (subset of \"pauth\")"),
261	clEnumValN(GS_PAUTH, "pauth", "All Pointer Authentication scanners"),
262	clEnumValN(GS_ALL, "all", "All implemented scanners")),
263	cl::ZeroOrMore, cl::CommaSeparated, cl::cat(BinaryAnalysisCategory));
264
265	} // namespace opts
266
267	// FIXME: implement a better way to mark sections for replacement.
268	constexpr const char *RewriteInstance::SectionsToOverwrite[];
269	std::vector<std::string> RewriteInstance::DebugSectionsToOverwrite = {
270	".debug_abbrev", ".debug_aranges", ".debug_line", ".debug_line_str",
271	".debug_loc", ".debug_loclists", ".debug_ranges", ".debug_rnglists",
272	".gdb_index", ".debug_addr", ".debug_abbrev", ".debug_info",
273	".debug_types", ".pseudo_probe"};
274
275	const char RewriteInstance::TimerGroupName[] = "rewrite";
276	const char RewriteInstance::TimerGroupDesc[] = "Rewrite passes";
277
278	namespace llvm {
279	namespace bolt {
280
281	extern const char *BoltRevision;
282
283	// Weird location for createMCPlusBuilder, but this is here to avoid a
284	// cyclic dependency of libCore (its natural place) and libTarget. libRewrite
285	// can depend on libTarget, but not libCore. Since libRewrite is the only
286	// user of this function, we define it here.
287	MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
288	const MCInstrAnalysis *Analysis,
289	const MCInstrInfo *Info,
290	const MCRegisterInfo *RegInfo,
291	const MCSubtargetInfo *STI) {
292	#ifdef X86_AVAILABLE
293	if (Arch == Triple::x86_64)
294	return createX86MCPlusBuilder(Analysis, Info, RegInfo, STI);
295	#endif
296
297	#ifdef AARCH64_AVAILABLE
298	if (Arch == Triple::aarch64)
299	return createAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
300	#endif
301
302	#ifdef RISCV_AVAILABLE
303	if (Arch == Triple::riscv64)
304	return createRISCVMCPlusBuilder(Analysis, Info, RegInfo, STI);
305	#endif
306
307	llvm_unreachable("architecture unsupported by MCPlusBuilder");
308	}
309
310	} // namespace bolt
311	} // namespace llvm
312
313	using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;
314
315	namespace {
316
317	bool refersToReorderedSection(ErrorOr<BinarySection &> Section) {
318	return llvm::any_of(Range&: opts::ReorderData, P: [&](const std::string &SectionName) {
319	return Section && Section->getName() == SectionName;
320	});
321	}
322
323	} // anonymous namespace
324
325	Expected<std::unique_ptr<RewriteInstance>>
326	RewriteInstance::create(ELFObjectFileBase *File, const int Argc,
327	const char *const *Argv, StringRef ToolPath,
328	raw_ostream &Stdout, raw_ostream &Stderr) {
329	Error Err = Error::success();
330	auto RI = std::make_unique<RewriteInstance>(args&: File, args: Argc, args&: Argv, args&: ToolPath,
331	args&: Stdout, args&: Stderr, args&: Err);
332	if (Err)
333	return std::move(Err);
334	return std::move(RI);
335	}
336
337	RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,
338	const char *const *Argv, StringRef ToolPath,
339	raw_ostream &Stdout, raw_ostream &Stderr,
340	Error &Err)
341	: InputFile(File), Argc(Argc), Argv(Argv), ToolPath(ToolPath),
342	SHStrTab(StringTableBuilder::ELF) {
343	ErrorAsOutParameter EAO(&Err);
344	auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(Val: InputFile);
345	if (!ELF64LEFile) {
346	Err = createStringError(EC: errc::not_supported,
347	S: "Only 64-bit LE ELF binaries are supported");
348	return;
349	}
350
351	bool IsPIC = false;
352	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
353	if (Obj.getHeader().e_type != ELF::ET_EXEC) {
354	Stdout << "BOLT-INFO: shared object or position-independent executable "
355	"detected\n";
356	IsPIC = true;
357	}
358
359	// Make sure we don't miss any output on core dumps.
360	Stdout.SetUnbuffered();
361	Stderr.SetUnbuffered();
362	LLVM_DEBUG(dbgs().SetUnbuffered());
363
364	// Read RISCV subtarget features from input file
365	std::unique_ptr<SubtargetFeatures> Features;
366	Triple TheTriple = File->makeTriple();
367	if (TheTriple.getArch() == llvm::Triple::riscv64) {
368	Expected<SubtargetFeatures> FeaturesOrErr = File->getFeatures();
369	if (auto E = FeaturesOrErr.takeError()) {
370	Err = std::move(E);
371	return;
372	} else {
373	Features.reset(p: new SubtargetFeatures(*FeaturesOrErr));
374	}
375	}
376
377	Relocation::Arch = TheTriple.getArch();
378	auto BCOrErr = BinaryContext::createBinaryContext(
379	TheTriple, SSP: std::make_shared<orc::SymbolStringPool>(), InputFileName: File->getFileName(),
380	Features: Features.get(), IsPIC,
381	DwCtx: DWARFContext::create(Obj: *File, RelocAction: DWARFContext::ProcessDebugRelocations::Ignore,
382	L: nullptr, DWPName: opts::DWPPathName,
383	RecoverableErrorHandler: WithColor::defaultErrorHandler,
384	WarningHandler: WithColor::defaultWarningHandler),
385	Logger: JournalingStreams{.Out: Stdout, .Err: Stderr});
386	if (Error E = BCOrErr.takeError()) {
387	Err = std::move(E);
388	return;
389	}
390	BC = std::move(BCOrErr.get());
391	BC->initializeTarget(TargetBuilder: std::unique_ptr<MCPlusBuilder>(
392	createMCPlusBuilder(Arch: BC->TheTriple->getArch(), Analysis: BC->MIA.get(),
393	Info: BC->MII.get(), RegInfo: BC->MRI.get(), STI: BC->STI.get())));
394
395	BAT = std::make_unique<BoltAddressTranslation>();
396
397	if (opts::UpdateDebugSections)
398	DebugInfoRewriter = std::make_unique<DWARFRewriter>(args&: *BC);
399
400	if (opts::Instrument)
401	BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
402	else if (opts::Hugify)
403	BC->setRuntimeLibrary(std::make_unique<HugifyRuntimeLibrary>());
404	}
405
406	RewriteInstance::~RewriteInstance() {}
407
408	Error RewriteInstance::setProfile(StringRef Filename) {
409	if (!sys::fs::exists(Path: Filename))
410	return errorCodeToError(EC: make_error_code(E: errc::no_such_file_or_directory));
411
412	if (ProfileReader) {
413	// Already exists
414	return make_error<StringError>(Args: Twine("multiple profiles specified: ") +
415	ProfileReader->getFilename() + " and " +
416	Filename,
417	Args: inconvertibleErrorCode());
418	}
419
420	// Spawn a profile reader based on file contents.
421	if (DataAggregator::checkPerfDataMagic(FileName: Filename))
422	ProfileReader = std::make_unique<DataAggregator>(args&: Filename);
423	else if (YAMLProfileReader::isYAML(Filename))
424	ProfileReader = std::make_unique<YAMLProfileReader>(args&: Filename);
425	else
426	ProfileReader = std::make_unique<DataReader>(args&: Filename);
427
428	return Error::success();
429	}
430
431	/// Return true if the function \p BF should be disassembled.
432	static bool shouldDisassemble(const BinaryFunction &BF) {
433	if (BF.isPseudo())
434	return false;
435
436	if (opts::processAllFunctions())
437	return true;
438
439	return !BF.isIgnored();
440	}
441
442	// Return if a section stored in the image falls into a segment address space.
443	// If not, Set \p Overlap to true if there's a partial overlap.
444	template <class ELFT>
445	static bool checkOffsets(const typename ELFT::Phdr &Phdr,
446	const typename ELFT::Shdr &Sec, bool &Overlap) {
447	// SHT_NOBITS sections don't need to have an offset inside the segment.
448	if (Sec.sh_type == ELF::SHT_NOBITS)
449	return true;
450
451	// Only non-empty sections can be at the end of a segment.
452	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
453	AddressRange SectionAddressRange((uint64_t)Sec.sh_offset,
454	Sec.sh_offset + SectionSize);
455	AddressRange SegmentAddressRange(Phdr.p_offset,
456	Phdr.p_offset + Phdr.p_filesz);
457	if (SegmentAddressRange.contains(R: SectionAddressRange))
458	return true;
459
460	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
461	return false;
462	}
463
464	// Check that an allocatable section belongs to a virtual address
465	// space of a segment.
466	template <class ELFT>
467	static bool checkVMA(const typename ELFT::Phdr &Phdr,
468	const typename ELFT::Shdr &Sec, bool &Overlap) {
469	// Only non-empty sections can be at the end of a segment.
470	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
471	AddressRange SectionAddressRange((uint64_t)Sec.sh_addr,
472	Sec.sh_addr + SectionSize);
473	AddressRange SegmentAddressRange(Phdr.p_vaddr, Phdr.p_vaddr + Phdr.p_memsz);
474
475	if (SegmentAddressRange.contains(R: SectionAddressRange))
476	return true;
477	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
478	return false;
479	}
480
481	void RewriteInstance::markGnuRelroSections() {
482	using ELFT = ELF64LE;
483	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
484	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
485	const ELFFile<ELFT> &Obj = ELF64LEFile->getELFFile();
486
487	auto handleSection = [&](const ELFT::Phdr &Phdr, SectionRef SecRef) {
488	BinarySection *BinarySection = BC->getSectionForSectionRef(Section: SecRef);
489	// If the section is non-allocatable, ignore it for GNU_RELRO purposes:
490	// it can't be made read-only after runtime relocations processing.
491	if (!BinarySection || !BinarySection->isAllocatable())
492	return;
493	const ELFShdrTy *Sec = cantFail(ValOrErr: Obj.getSection(Index: SecRef.getIndex()));
494	bool ImageOverlap{false}, VMAOverlap{false};
495	bool ImageContains = checkOffsets<ELFT>(Phdr, Sec: *Sec, Overlap&: ImageOverlap);
496	bool VMAContains = checkVMA<ELFT>(Phdr, Sec: *Sec, Overlap&: VMAOverlap);
497	if (ImageOverlap) {
498	if (opts::Verbosity >= 1)
499	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial file offset "
500	<< "overlap with section " << BinarySection->getName()
501	<< '\n';
502	return;
503	}
504	if (VMAOverlap) {
505	if (opts::Verbosity >= 1)
506	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial VMA overlap "
507	<< "with section " << BinarySection->getName() << '\n';
508	return;
509	}
510	if (!ImageContains || !VMAContains)
511	return;
512	BinarySection->setRelro();
513	if (opts::Verbosity >= 1)
514	BC->outs() << "BOLT-INFO: marking " << BinarySection->getName()
515	<< " as GNU_RELRO\n";
516	};
517
518	for (const ELFT::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers()))
519	if (Phdr.p_type == ELF::PT_GNU_RELRO)
520	for (SectionRef SecRef : InputFile->sections())
521	handleSection(Phdr, SecRef);
522	}
523
524	Error RewriteInstance::discoverStorage() {
525	NamedRegionTimer T("discoverStorage", "discover storage", TimerGroupName,
526	TimerGroupDesc, opts::TimeRewrite);
527
528	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
529	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
530
531	BC->StartFunctionAddress = Obj.getHeader().e_entry;
532
533	NextAvailableAddress = 0;
534	uint64_t NextAvailableOffset = 0;
535	Expected<ELF64LE::PhdrRange> PHsOrErr = Obj.program_headers();
536	if (Error E = PHsOrErr.takeError())
537	return E;
538
539	ELF64LE::PhdrRange PHs = PHsOrErr.get();
540	for (const ELF64LE::Phdr &Phdr : PHs) {
541	switch (Phdr.p_type) {
542	case ELF::PT_LOAD:
543	BC->FirstAllocAddress = std::min(a: BC->FirstAllocAddress,
544	b: static_cast<uint64_t>(Phdr.p_vaddr));
545	NextAvailableAddress = std::max(a: NextAvailableAddress,
546	b: Phdr.p_vaddr + Phdr.p_memsz);
547	NextAvailableOffset = std::max(a: NextAvailableOffset,
548	b: Phdr.p_offset + Phdr.p_filesz);
549
550	BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
551	.Address: Phdr.p_vaddr, .Size: Phdr.p_memsz, .FileOffset: Phdr.p_offset,
552	.FileSize: Phdr.p_filesz, .Alignment: Phdr.p_align, .IsExecutable: ((Phdr.p_flags & ELF::PF_X) != 0)};
553	if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
554	Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
555	BC->IsLinuxKernel = true;
556	break;
557	case ELF::PT_INTERP:
558	BC->HasInterpHeader = true;
559	break;
560	}
561	}
562
563	if (BC->IsLinuxKernel)
564	BC->outs() << "BOLT-INFO: Linux kernel binary detected\n";
565
566	for (const SectionRef &Section : InputFile->sections()) {
567	Expected<StringRef> SectionNameOrErr = Section.getName();
568	if (Error E = SectionNameOrErr.takeError())
569	return E;
570	StringRef SectionName = SectionNameOrErr.get();
571	if (SectionName == BC->getMainCodeSectionName()) {
572	BC->OldTextSectionAddress = Section.getAddress();
573	BC->OldTextSectionSize = Section.getSize();
574
575	Expected<StringRef> SectionContentsOrErr = Section.getContents();
576	if (Error E = SectionContentsOrErr.takeError())
577	return E;
578	StringRef SectionContents = SectionContentsOrErr.get();
579	BC->OldTextSectionOffset =
580	SectionContents.data() - InputFile->getData().data();
581	}
582
583	if (!opts::HeatmapMode &&
584	!(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&
585	(SectionName.starts_with(Prefix: getOrgSecPrefix()) ||
586	SectionName == getBOLTTextSectionName()))
587	return createStringError(
588	EC: errc::function_not_supported,
589	S: "BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize");
590	}
591
592	if (!NextAvailableAddress || !NextAvailableOffset)
593	return createStringError(EC: errc::executable_format_error,
594	S: "no PT_LOAD pheader seen");
595
596	BC->outs() << "BOLT-INFO: first alloc address is 0x"
597	<< Twine::utohexstr(Val: BC->FirstAllocAddress) << '\n';
598
599	FirstNonAllocatableOffset = NextAvailableOffset;
600
601	if (opts::CustomAllocationVMA) {
602	// If user specified a custom address where we should start writing new
603	// data, honor that.
604	NextAvailableAddress = opts::CustomAllocationVMA;
605	// Sanity check the user-supplied address and emit warnings if something
606	// seems off.
607	for (const ELF64LE::Phdr &Phdr : PHs) {
608	switch (Phdr.p_type) {
609	case ELF::PT_LOAD:
610	if (NextAvailableAddress >= Phdr.p_vaddr &&
611	NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) {
612	BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x"
613	<< Twine::utohexstr(Val: NextAvailableAddress)
614	<< " conflicts with ELF segment at 0x"
615	<< Twine::utohexstr(Val: Phdr.p_vaddr) << "\n";
616	}
617	}
618	}
619	}
620	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->PageAlign);
621	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: BC->PageAlign);
622
623	// Hugify: Additional huge page from left side due to
624	// weird ASLR mapping addresses (4KB aligned)
625	if (opts::Hugify && !BC->HasFixedLoadAddress) {
626	NextAvailableAddress += BC->PageAlign;
627	}
628
629	if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
630	// This is where the black magic happens. Creating PHDR table in a segment
631	// other than that containing ELF header is tricky. Some loaders and/or
632	// parts of loaders will apply e_phoff from ELF header assuming both are in
633	// the same segment, while others will do the proper calculation.
634	// We create the new PHDR table in such a way that both of the methods
635	// of loading and locating the table work. There's a slight file size
636	// overhead because of that.
637	//
638	// NB: bfd's strip command cannot do the above and will corrupt the
639	// binary during the process of stripping non-allocatable sections.
640	if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress)
641	NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress;
642	else
643	NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress;
644
645	assert(NextAvailableOffset ==
646	NextAvailableAddress - BC->FirstAllocAddress &&
647	"PHDR table address calculation error");
648
649	BC->outs() << "BOLT-INFO: creating new program header table at address 0x"
650	<< Twine::utohexstr(Val: NextAvailableAddress) << ", offset 0x"
651	<< Twine::utohexstr(Val: NextAvailableOffset) << '\n';
652
653	PHDRTableAddress = NextAvailableAddress;
654	PHDRTableOffset = NextAvailableOffset;
655
656	// Reserve space for 3 extra pheaders.
657	unsigned Phnum = Obj.getHeader().e_phnum;
658	Phnum += 3;
659
660	// Reserve two more pheaders to avoid having writeable and executable
661	// segment in instrumented binary.
662	if (opts::Instrument)
663	Phnum += 2;
664
665	NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
666	NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
667	}
668
669	// Align at cache line.
670	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 64);
671	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: 64);
672
673	NewTextSegmentAddress = NextAvailableAddress;
674	NewTextSegmentOffset = NextAvailableOffset;
675	BC->LayoutStartAddress = NextAvailableAddress;
676
677	// Tools such as objcopy can strip section contents but leave header
678	// entries. Check that at least .text is mapped in the file.
679	if (!getFileOffsetForAddress(Address: BC->OldTextSectionAddress))
680	return createStringError(EC: errc::executable_format_error,
681	S: "BOLT-ERROR: input binary is not a valid ELF "
682	"executable as its text section is not "
683	"mapped to a valid segment");
684	return Error::success();
685	}
686
687	Error RewriteInstance::run() {
688	assert(BC && "failed to create a binary context");
689
690	BC->outs() << "BOLT-INFO: Target architecture: "
691	<< Triple::getArchTypeName(
692	Kind: (llvm::Triple::ArchType)InputFile->getArch())
693	<< "\n";
694	BC->outs() << "BOLT-INFO: BOLT version: " << BoltRevision << "\n";
695
696	if (Error E = discoverStorage())
697	return E;
698	if (Error E = readSpecialSections())
699	return E;
700	adjustCommandLineOptions();
701	discoverFileObjects();
702
703	if (opts::Instrument && !BC->IsStaticExecutable)
704	if (Error E = discoverRtFiniAddress())
705	return E;
706
707	preprocessProfileData();
708
709	// Skip disassembling if we have a translation table and we are running an
710	// aggregation job.
711	if (opts::AggregateOnly && BAT->enabledFor(InputFile)) {
712	// YAML profile in BAT mode requires CFG for .bolt.org.text functions
713	if (!opts::SaveProfile.empty() ||
714	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) {
715	selectFunctionsToProcess();
716	disassembleFunctions();
717	processMetadataPreCFG();
718	buildFunctionsCFG();
719	}
720	processProfileData();
721	return Error::success();
722	}
723
724	selectFunctionsToProcess();
725
726	readDebugInfo();
727
728	disassembleFunctions();
729
730	processMetadataPreCFG();
731
732	buildFunctionsCFG();
733
734	processProfileData();
735
736	// Save input binary metadata if BAT section needs to be emitted
737	if (opts::EnableBAT)
738	BAT->saveMetadata(BC&: *BC);
739
740	postProcessFunctions();
741
742	processMetadataPostCFG();
743
744	if (opts::DiffOnly)
745	return Error::success();
746
747	if (opts::BinaryAnalysisMode) {
748	runBinaryAnalyses();
749	return Error::success();
750	}
751
752	preregisterSections();
753
754	runOptimizationPasses();
755
756	finalizeMetadataPreEmit();
757
758	emitAndLink();
759
760	updateMetadata();
761
762	if (opts::Instrument && !BC->IsStaticExecutable)
763	updateRtFiniReloc();
764
765	if (opts::OutputFilename == "/dev/null") {
766	BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n";
767	return Error::success();
768	} else if (BC->IsLinuxKernel) {
769	BC->errs() << "BOLT-WARNING: Linux kernel support is experimental\n";
770	}
771
772	// Rewrite allocatable contents and copy non-allocatable parts with mods.
773	rewriteFile();
774	return Error::success();
775	}
776
777	void RewriteInstance::discoverFileObjects() {
778	NamedRegionTimer T("discoverFileObjects", "discover file objects",
779	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
780
781	// For local symbols we want to keep track of associated FILE symbol name for
782	// disambiguation by combined name.
783	StringRef FileSymbolName;
784	bool SeenFileName = false;
785	struct SymbolRefHash {
786	size_t operator()(SymbolRef const &S) const {
787	return std::hash<decltype(DataRefImpl::p)>{}(S.getRawDataRefImpl().p);
788	}
789	};
790	std::unordered_map<SymbolRef, StringRef, SymbolRefHash> SymbolToFileName;
791	for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
792	Expected<StringRef> NameOrError = Symbol.getName();
793	if (NameOrError && NameOrError->starts_with(Prefix: "__asan_init")) {
794	BC->errs()
795	<< "BOLT-ERROR: input file was compiled or linked with sanitizer "
796	"support. Cannot optimize.\n";
797	exit(status: 1);
798	}
799	if (NameOrError && NameOrError->starts_with(Prefix: "__llvm_coverage_mapping")) {
800	BC->errs()
801	<< "BOLT-ERROR: input file was compiled or linked with coverage "
802	"support. Cannot optimize.\n";
803	exit(status: 1);
804	}
805
806	if (cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Undefined)
807	continue;
808
809	if (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_File) {
810	FileSymbols.emplace_back(args: Symbol);
811	StringRef Name =
812	cantFail(ValOrErr: std::move(NameOrError), Msg: "cannot get symbol name for file");
813	// Ignore Clang LTO artificial FILE symbol as it is not always generated,
814	// and this uncertainty is causing havoc in function name matching.
815	if (Name == "ld-temp.o")
816	continue;
817	FileSymbolName = Name;
818	SeenFileName = true;
819	continue;
820	}
821	if (!FileSymbolName.empty() &&
822	!(cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Global))
823	SymbolToFileName[Symbol] = FileSymbolName;
824	}
825
826	// Sort symbols in the file by value. Ignore symbols from non-allocatable
827	// sections. We memoize getAddress(), as it has rather high overhead.
828	struct SymbolInfo {
829	uint64_t Address;
830	SymbolRef Symbol;
831	};
832	std::vector<SymbolInfo> SortedSymbols;
833	auto isSymbolInMemory = [this](const SymbolRef &Sym) {
834	if (cantFail(ValOrErr: Sym.getType()) == SymbolRef::ST_File)
835	return false;
836	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Absolute)
837	return true;
838	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Undefined)
839	return false;
840	BinarySection Section(*BC, *cantFail(ValOrErr: Sym.getSection()));
841	return Section.isAllocatable();
842	};
843	auto checkSymbolInSection = [this](const SymbolInfo &S) {
844	// Sometimes, we encounter symbols with addresses outside their section. If
845	// such symbols happen to fall into another section, they can interfere with
846	// disassembly. Notably, this occurs with AArch64 marker symbols ($d and $t)
847	// that belong to .eh_frame, but end up pointing into .text.
848	// As a workaround, we ignore all symbols that lie outside their sections.
849	auto Section = cantFail(ValOrErr: S.Symbol.getSection());
850
851	// Accept all absolute symbols.
852	if (Section == InputFile->section_end())
853	return true;
854
855	uint64_t SecStart = Section->getAddress();
856	uint64_t SecEnd = SecStart + Section->getSize();
857	uint64_t SymEnd = S.Address + ELFSymbolRef(S.Symbol).getSize();
858	if (S.Address >= SecStart && SymEnd <= SecEnd)
859	return true;
860
861	auto SymType = cantFail(ValOrErr: S.Symbol.getType());
862	// Skip warnings for common benign cases.
863	if (opts::Verbosity < 1 && SymType == SymbolRef::ST_Other)
864	return false; // E.g. ELF::STT_TLS.
865
866	auto SymName = S.Symbol.getName();
867	auto SecName = cantFail(ValOrErr: S.Symbol.getSection())->getName();
868	BC->errs() << "BOLT-WARNING: ignoring symbol "
869	<< (SymName ? *SymName : "[unnamed]") << " at 0x"
870	<< Twine::utohexstr(Val: S.Address) << ", which lies outside "
871	<< (SecName ? *SecName : "[unnamed]") << "\n";
872
873	return false;
874	};
875	for (const SymbolRef &Symbol : InputFile->symbols())
876	if (isSymbolInMemory(Symbol)) {
877	SymbolInfo SymInfo{.Address: cantFail(ValOrErr: Symbol.getAddress()), .Symbol: Symbol};
878	if (checkSymbolInSection(SymInfo))
879	SortedSymbols.push_back(x: SymInfo);
880	}
881
882	auto CompareSymbols = [this](const SymbolInfo &A, const SymbolInfo &B) {
883	if (A.Address != B.Address)
884	return A.Address < B.Address;
885
886	const bool AMarker = BC->isMarker(Symbol: A.Symbol);
887	const bool BMarker = BC->isMarker(Symbol: B.Symbol);
888	if (AMarker || BMarker) {
889	return AMarker && !BMarker;
890	}
891
892	const auto AType = cantFail(ValOrErr: A.Symbol.getType());
893	const auto BType = cantFail(ValOrErr: B.Symbol.getType());
894	if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function)
895	return true;
896	if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug)
897	return true;
898
899	return false;
900	};
901	llvm::stable_sort(Range&: SortedSymbols, C: CompareSymbols);
902
903	auto LastSymbol = SortedSymbols.end();
904	if (!SortedSymbols.empty())
905	--LastSymbol;
906
907	// For aarch64, the ABI defines mapping symbols so we identify data in the
908	// code section (see IHI0056B). $d identifies data contents.
909	// Compilers usually merge multiple data objects in a single $d-$x interval,
910	// but we need every data object to be marked with $d. Because of that we
911	// create a vector of MarkerSyms with all locations of data objects.
912
913	struct MarkerSym {
914	uint64_t Address;
915	MarkerSymType Type;
916	};
917
918	std::vector<MarkerSym> SortedMarkerSymbols;
919	auto addExtraDataMarkerPerSymbol = [&]() {
920	bool IsData = false;
921	uint64_t LastAddr = 0;
922	for (const auto &SymInfo : SortedSymbols) {
923	if (LastAddr == SymInfo.Address) // don't repeat markers
924	continue;
925
926	MarkerSymType MarkerType = BC->getMarkerType(Symbol: SymInfo.Symbol);
927	if (MarkerType != MarkerSymType::NONE) {
928	SortedMarkerSymbols.push_back(x: MarkerSym{.Address: SymInfo.Address, .Type: MarkerType});
929	LastAddr = SymInfo.Address;
930	IsData = MarkerType == MarkerSymType::DATA;
931	continue;
932	}
933
934	if (IsData) {
935	SortedMarkerSymbols.push_back(x: {.Address: SymInfo.Address, .Type: MarkerSymType::DATA});
936	LastAddr = SymInfo.Address;
937	}
938	}
939	};
940
941	if (BC->isAArch64() || BC->isRISCV()) {
942	addExtraDataMarkerPerSymbol();
943	LastSymbol = std::stable_partition(
944	first: SortedSymbols.begin(), last: SortedSymbols.end(),
945	pred: [this](const SymbolInfo &S) { return !BC->isMarker(Symbol: S.Symbol); });
946	if (!SortedSymbols.empty())
947	--LastSymbol;
948	}
949
950	BinaryFunction *PreviousFunction = nullptr;
951	unsigned AnonymousId = 0;
952
953	const auto SortedSymbolsEnd =
954	LastSymbol == SortedSymbols.end() ? LastSymbol : std::next(x: LastSymbol);
955	for (auto Iter = SortedSymbols.begin(); Iter != SortedSymbolsEnd; ++Iter) {
956	const SymbolRef &Symbol = Iter->Symbol;
957	const uint64_t SymbolAddress = Iter->Address;
958	const auto SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
959	const SymbolRef::Type SymbolType = cantFail(ValOrErr: Symbol.getType());
960
961	if (SymbolType == SymbolRef::ST_File)
962	continue;
963
964	StringRef SymName = cantFail(ValOrErr: Symbol.getName(), Msg: "cannot get symbol name");
965	if (SymbolAddress == 0) {
966	if (opts::Verbosity >= 1 && SymbolType == SymbolRef::ST_Function)
967	BC->errs() << "BOLT-WARNING: function with 0 address seen\n";
968	continue;
969	}
970
971	// Ignore input hot markers unless in heatmap mode
972	if ((SymName == "__hot_start" || SymName == "__hot_end") &&
973	!opts::HeatmapMode)
974	continue;
975
976	FileSymRefs.emplace(args: SymbolAddress, args: Symbol);
977
978	// Skip section symbols that will be registered by disassemblePLT().
979	if (SymbolType == SymbolRef::ST_Debug) {
980	ErrorOr<BinarySection &> BSection =
981	BC->getSectionForAddress(Address: SymbolAddress);
982	if (BSection && getPLTSectionInfo(SectionName: BSection->getName()))
983	continue;
984	}
985
986	/// It is possible we are seeing a globalized local. LLVM might treat it as
987	/// a local if it has a "private global" prefix, e.g. ".L". Thus we have to
988	/// change the prefix to enforce global scope of the symbol.
989	std::string Name =
990	SymName.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix())
991	? "PG" + std::string(SymName)
992	: std::string(SymName);
993
994	// Disambiguate all local symbols before adding to symbol table.
995	// Since we don't know if we will see a global with the same name,
996	// always modify the local name.
997	//
998	// NOTE: the naming convention for local symbols should match
999	// the one we use for profile data.
1000	std::string UniqueName;
1001	std::string AlternativeName;
1002	if (Name.empty()) {
1003	UniqueName = "ANONYMOUS." + std::to_string(val: AnonymousId++);
1004	} else if (SymbolFlags & SymbolRef::SF_Global) {
1005	if (const BinaryData *BD = BC->getBinaryDataByName(Name)) {
1006	if (BD->getSize() == ELFSymbolRef(Symbol).getSize() &&
1007	BD->getAddress() == SymbolAddress) {
1008	if (opts::Verbosity > 1)
1009	BC->errs() << "BOLT-WARNING: ignoring duplicate global symbol "
1010	<< Name << "\n";
1011	// Ignore duplicate entry - possibly a bug in the linker
1012	continue;
1013	}
1014	BC->errs() << "BOLT-ERROR: bad input binary, global symbol \"" << Name
1015	<< "\" is not unique\n";
1016	exit(status: 1);
1017	}
1018	UniqueName = Name;
1019	} else {
1020	// If we have a local file name, we should create 2 variants for the
1021	// function name. The reason is that perf profile might have been
1022	// collected on a binary that did not have the local file name (e.g. as
1023	// a side effect of stripping debug info from the binary):
1024	//
1025	// primary: <function>/<id>
1026	// alternative: <function>/<file>/<id2>
1027	//
1028	// The <id> field is used for disambiguation of local symbols since there
1029	// could be identical function names coming from identical file names
1030	// (e.g. from different directories).
1031	std::string AltPrefix;
1032	auto SFI = SymbolToFileName.find(x: Symbol);
1033	if (SymbolType == SymbolRef::ST_Function && SFI != SymbolToFileName.end())
1034	AltPrefix = Name + "/" + std::string(SFI->second);
1035
1036	UniqueName = NR.uniquify(Name);
1037	if (!AltPrefix.empty())
1038	AlternativeName = NR.uniquify(Name: AltPrefix);
1039	}
1040
1041	uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1042	uint64_t SymbolAlignment = Symbol.getAlignment();
1043
1044	auto registerName = [&](uint64_t FinalSize) {
1045	// Register names even if it's not a function, e.g. for an entry point.
1046	BC->registerNameAtAddress(Name: UniqueName, Address: SymbolAddress, Size: FinalSize,
1047	Alignment: SymbolAlignment, Flags: SymbolFlags);
1048	if (!AlternativeName.empty())
1049	BC->registerNameAtAddress(Name: AlternativeName, Address: SymbolAddress, Size: FinalSize,
1050	Alignment: SymbolAlignment, Flags: SymbolFlags);
1051	};
1052
1053	section_iterator Section =
1054	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
1055	if (Section == InputFile->section_end()) {
1056	// Could be an absolute symbol. Used on RISC-V for __global_pointer$ so we
1057	// need to record it to handle relocations against it. For other instances
1058	// of absolute symbols, we record for pretty printing.
1059	LLVM_DEBUG(if (opts::Verbosity > 1) {
1060	dbgs() << "BOLT-INFO: absolute sym " << UniqueName << "\n";
1061	});
1062	registerName(SymbolSize);
1063	continue;
1064	}
1065
1066	if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) {
1067	registerName(SymbolSize);
1068	continue;
1069	}
1070
1071	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName
1072	<< " for function\n");
1073
1074	if (SymbolAddress == Section->getAddress() + Section->getSize()) {
1075	assert(SymbolSize == 0 &&
1076	"unexpect non-zero sized symbol at end of section");
1077	LLVM_DEBUG(
1078	dbgs()
1079	<< "BOLT-DEBUG: rejecting as symbol points to end of its section\n");
1080	registerName(SymbolSize);
1081	continue;
1082	}
1083
1084	if (!Section->isText() || Section->isVirtual()) {
1085	assert(SymbolType != SymbolRef::ST_Function &&
1086	"unexpected function inside non-code section");
1087	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code or "
1088	"is in nobits section\n");
1089	registerName(SymbolSize);
1090	continue;
1091	}
1092
1093	// Assembly functions could be ST_NONE with 0 size. Check that the
1094	// corresponding section is a code section and they are not inside any
1095	// other known function to consider them.
1096	//
1097	// Sometimes assembly functions are not marked as functions and neither are
1098	// their local labels. The only way to tell them apart is to look at
1099	// symbol scope - global vs local.
1100	if (PreviousFunction && SymbolType != SymbolRef::ST_Function) {
1101	if (PreviousFunction->containsAddress(PC: SymbolAddress)) {
1102	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1103	LLVM_DEBUG(dbgs()
1104	<< "BOLT-DEBUG: symbol is a function local symbol\n");
1105	} else if (SymbolAddress == PreviousFunction->getAddress() &&
1106	!SymbolSize) {
1107	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring symbol as a marker\n");
1108	} else if (opts::Verbosity > 1) {
1109	BC->errs() << "BOLT-WARNING: symbol " << UniqueName
1110	<< " seen in the middle of function " << *PreviousFunction
1111	<< ". Could be a new entry.\n";
1112	}
1113	registerName(SymbolSize);
1114	continue;
1115	} else if (PreviousFunction->getSize() == 0 &&
1116	PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1117	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: symbol is a function local symbol\n");
1118	registerName(SymbolSize);
1119	continue;
1120	}
1121	}
1122
1123	if (PreviousFunction && PreviousFunction->containsAddress(PC: SymbolAddress) &&
1124	PreviousFunction->getAddress() != SymbolAddress) {
1125	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1126	if (opts::Verbosity >= 1)
1127	BC->outs()
1128	<< "BOLT-INFO: skipping possibly another entry for function "
1129	<< *PreviousFunction << " : " << UniqueName << '\n';
1130	registerName(SymbolSize);
1131	} else {
1132	BC->outs() << "BOLT-INFO: using " << UniqueName
1133	<< " as another entry to "
1134	<< "function " << *PreviousFunction << '\n';
1135
1136	registerName(0);
1137
1138	PreviousFunction->addEntryPointAtOffset(Offset: SymbolAddress -
1139	PreviousFunction->getAddress());
1140
1141	// Remove the symbol from FileSymRefs so that we can skip it from
1142	// in the future.
1143	auto SI = llvm::find_if(
1144	Range: llvm::make_range(p: FileSymRefs.equal_range(x: SymbolAddress)),
1145	P: [&](auto SymIt) { return SymIt.second == Symbol; });
1146	assert(SI != FileSymRefs.end() && "symbol expected to be present");
1147	assert(SI->second == Symbol && "wrong symbol found");
1148	FileSymRefs.erase(position: SI);
1149	}
1150	continue;
1151	}
1152
1153	// Checkout for conflicts with function data from FDEs.
1154	bool IsSimple = true;
1155	auto FDEI = CFIRdWrt->getFDEs().lower_bound(x: SymbolAddress);
1156	if (FDEI != CFIRdWrt->getFDEs().end()) {
1157	const dwarf::FDE &FDE = *FDEI->second;
1158	if (FDEI->first != SymbolAddress) {
1159	// There's no matching starting address in FDE. Make sure the previous
1160	// FDE does not contain this address.
1161	if (FDEI != CFIRdWrt->getFDEs().begin()) {
1162	--FDEI;
1163	const dwarf::FDE &PrevFDE = *FDEI->second;
1164	uint64_t PrevStart = PrevFDE.getInitialLocation();
1165	uint64_t PrevLength = PrevFDE.getAddressRange();
1166	if (SymbolAddress > PrevStart &&
1167	SymbolAddress < PrevStart + PrevLength) {
1168	BC->errs() << "BOLT-ERROR: function " << UniqueName
1169	<< " is in conflict with FDE ["
1170	<< Twine::utohexstr(Val: PrevStart) << ", "
1171	<< Twine::utohexstr(Val: PrevStart + PrevLength)
1172	<< "). Skipping.\n";
1173	IsSimple = false;
1174	}
1175	}
1176	} else if (FDE.getAddressRange() != SymbolSize) {
1177	if (SymbolSize) {
1178	// Function addresses match but sizes differ.
1179	BC->errs() << "BOLT-WARNING: sizes differ for function " << UniqueName
1180	<< ". FDE : " << FDE.getAddressRange()
1181	<< "; symbol table : " << SymbolSize
1182	<< ". Using max size.\n";
1183	}
1184	SymbolSize = std::max(a: SymbolSize, b: FDE.getAddressRange());
1185	if (BC->getBinaryDataAtAddress(Address: SymbolAddress)) {
1186	BC->setBinaryDataSize(Address: SymbolAddress, Size: SymbolSize);
1187	} else {
1188	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: No BD @ 0x"
1189	<< Twine::utohexstr(SymbolAddress) << "\n");
1190	}
1191	}
1192	}
1193
1194	BinaryFunction *BF = nullptr;
1195	// Since function may not have yet obtained its real size, do a search
1196	// using the list of registered functions instead of calling
1197	// getBinaryFunctionAtAddress().
1198	auto BFI = BC->getBinaryFunctions().find(x: SymbolAddress);
1199	if (BFI != BC->getBinaryFunctions().end()) {
1200	BF = &BFI->second;
1201	// Duplicate the function name. Make sure everything matches before we add
1202	// an alternative name.
1203	if (SymbolSize != BF->getSize()) {
1204	if (opts::Verbosity >= 1) {
1205	if (SymbolSize && BF->getSize())
1206	BC->errs() << "BOLT-WARNING: size mismatch for duplicate entries "
1207	<< *BF << " and " << UniqueName << '\n';
1208	BC->outs() << "BOLT-INFO: adjusting size of function " << *BF
1209	<< " old " << BF->getSize() << " new " << SymbolSize
1210	<< "\n";
1211	}
1212	BF->setSize(std::max(a: SymbolSize, b: BF->getSize()));
1213	BC->setBinaryDataSize(Address: SymbolAddress, Size: BF->getSize());
1214	}
1215	BF->addAlternativeName(NewName: UniqueName);
1216	} else {
1217	ErrorOr<BinarySection &> Section =
1218	BC->getSectionForAddress(Address: SymbolAddress);
1219	// Skip symbols from invalid sections
1220	if (!Section) {
1221	BC->errs() << "BOLT-WARNING: " << UniqueName << " (0x"
1222	<< Twine::utohexstr(Val: SymbolAddress)
1223	<< ") does not have any section\n";
1224	continue;
1225	}
1226
1227	// Skip symbols from zero-sized sections.
1228	if (!Section->getSize())
1229	continue;
1230
1231	BF = BC->createBinaryFunction(Name: UniqueName, Section&: *Section, Address: SymbolAddress,
1232	Size: SymbolSize);
1233	if (!IsSimple)
1234	BF->setSimple(false);
1235	}
1236
1237	// Check if it's a cold function fragment.
1238	if (FunctionFragmentTemplate.match(String: SymName)) {
1239	static bool PrintedWarning = false;
1240	if (!PrintedWarning) {
1241	PrintedWarning = true;
1242	BC->errs() << "BOLT-WARNING: split function detected on input : "
1243	<< SymName;
1244	if (BC->HasRelocations)
1245	BC->errs() << ". The support is limited in relocation mode\n";
1246	else
1247	BC->errs() << '\n';
1248	}
1249	BC->HasSplitFunctions = true;
1250	BF->IsFragment = true;
1251	}
1252
1253	if (!AlternativeName.empty())
1254	BF->addAlternativeName(NewName: AlternativeName);
1255
1256	registerName(SymbolSize);
1257	PreviousFunction = BF;
1258	}
1259
1260	// Read dynamic relocation first as their presence affects the way we process
1261	// static relocations. E.g. we will ignore a static relocation at an address
1262	// that is a subject to dynamic relocation processing.
1263	processDynamicRelocations();
1264
1265	// Process PLT section.
1266	disassemblePLT();
1267
1268	// See if we missed any functions marked by FDE.
1269	for (const auto &FDEI : CFIRdWrt->getFDEs()) {
1270	const uint64_t Address = FDEI.first;
1271	const dwarf::FDE *FDE = FDEI.second;
1272	const BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address);
1273	if (BF)
1274	continue;
1275
1276	BF = BC->getBinaryFunctionContainingAddress(Address);
1277	if (BF) {
1278	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1279	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1280	<< ") conflicts with function " << *BF << '\n';
1281	continue;
1282	}
1283
1284	if (opts::Verbosity >= 1)
1285	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1286	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1287	<< ") has no corresponding symbol table entry\n";
1288
1289	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
1290	assert(Section && "cannot get section for address from FDE");
1291	std::string FunctionName =
1292	"__BOLT_FDE_FUNCat" + Twine::utohexstr(Val: Address).str();
1293	BC->createBinaryFunction(Name: FunctionName, Section&: *Section, Address,
1294	Size: FDE->getAddressRange());
1295	}
1296
1297	BC->setHasSymbolsWithFileName(SeenFileName);
1298
1299	// Now that all the functions were created - adjust their boundaries.
1300	adjustFunctionBoundaries();
1301
1302	// Annotate functions with code/data markers in AArch64
1303	for (auto ISym = SortedMarkerSymbols.begin();
1304	ISym != SortedMarkerSymbols.end(); ++ISym) {
1305
1306	auto *BF =
1307	BC->getBinaryFunctionContainingAddress(Address: ISym->Address, CheckPastEnd: true, UseMaxSize: true);
1308
1309	if (!BF) {
1310	// Stray marker
1311	continue;
1312	}
1313	const auto EntryOffset = ISym->Address - BF->getAddress();
1314	if (ISym->Type == MarkerSymType::CODE) {
1315	BF->markCodeAtOffset(Offset: EntryOffset);
1316	continue;
1317	}
1318	if (ISym->Type == MarkerSymType::DATA) {
1319	BF->markDataAtOffset(Offset: EntryOffset);
1320	BC->AddressToConstantIslandMap[ISym->Address] = BF;
1321	continue;
1322	}
1323	llvm_unreachable("Unknown marker");
1324	}
1325
1326	if (BC->isAArch64()) {
1327	// Check for dynamic relocations that might be contained in
1328	// constant islands.
1329	for (const BinarySection &Section : BC->allocatableSections()) {
1330	const uint64_t SectionAddress = Section.getAddress();
1331	for (const Relocation &Rel : Section.dynamicRelocations()) {
1332	const uint64_t RelAddress = SectionAddress + Rel.Offset;
1333	BinaryFunction *BF =
1334	BC->getBinaryFunctionContainingAddress(Address: RelAddress,
1335	/*CheckPastEnd*/ false,
1336	/*UseMaxSize*/ true);
1337	if (BF) {
1338	assert(Rel.isRelative() && "Expected relative relocation for island");
1339	BC->logBOLTErrorsAndQuitOnFatal(
1340	E: BF->markIslandDynamicRelocationAtAddress(Address: RelAddress));
1341	}
1342	}
1343	}
1344
1345	// The linker may omit data markers for absolute long veneers. Introduce
1346	// those markers artificially to assist the disassembler.
1347	for (BinaryFunction &BF :
1348	llvm::make_second_range(c&: BC->getBinaryFunctions())) {
1349	if (BF.getOneName().starts_with(Prefix: "__AArch64AbsLongThunk_") &&
1350	BF.getSize() == 16 && !BF.getSizeOfDataInCodeAt(Offset: 8)) {
1351	BC->errs() << "BOLT-WARNING: missing data marker detected in veneer "
1352	<< BF << '\n';
1353	BF.markDataAtOffset(Offset: 8);
1354	BC->AddressToConstantIslandMap[BF.getAddress() + 8] = &BF;
1355	}
1356	}
1357	}
1358
1359	if (!BC->IsLinuxKernel) {
1360	// Read all relocations now that we have binary functions mapped.
1361	processRelocations();
1362	}
1363
1364	registerFragments();
1365	FileSymbols.clear();
1366	FileSymRefs.clear();
1367
1368	discoverBOLTReserved();
1369	}
1370
1371	void RewriteInstance::discoverBOLTReserved() {
1372	BinaryData *StartBD = BC->getBinaryDataByName(Name: getBOLTReservedStart());
1373	BinaryData *EndBD = BC->getBinaryDataByName(Name: getBOLTReservedEnd());
1374	if (!StartBD != !EndBD) {
1375	BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: "
1376	<< getBOLTReservedStart() << ", " << getBOLTReservedEnd()
1377	<< '\n';
1378	exit(status: 1);
1379	}
1380
1381	if (!StartBD)
1382	return;
1383
1384	if (StartBD->getAddress() >= EndBD->getAddress()) {
1385	BC->errs() << "BOLT-ERROR: invalid reserved space boundaries\n";
1386	exit(status: 1);
1387	}
1388	BC->BOLTReserved = AddressRange(StartBD->getAddress(), EndBD->getAddress());
1389	BC->outs() << "BOLT-INFO: using reserved space for allocating new sections\n";
1390
1391	PHDRTableOffset = 0;
1392	PHDRTableAddress = 0;
1393	NewTextSegmentAddress = 0;
1394	NewTextSegmentOffset = 0;
1395	NextAvailableAddress = BC->BOLTReserved.start();
1396	}
1397
1398	Error RewriteInstance::discoverRtFiniAddress() {
1399	// Use DT_FINI if it's available.
1400	if (BC->FiniAddress) {
1401	BC->FiniFunctionAddress = BC->FiniAddress;
1402	return Error::success();
1403	}
1404
1405	if (!BC->FiniArrayAddress || !BC->FiniArraySize) {
1406	return createStringError(
1407	EC: std::errc::not_supported,
1408	Fmt: "Instrumentation needs either DT_FINI or DT_FINI_ARRAY");
1409	}
1410
1411	if (*BC->FiniArraySize < BC->AsmInfo->getCodePointerSize()) {
1412	return createStringError(EC: std::errc::not_supported,
1413	Fmt: "Need at least 1 DT_FINI_ARRAY slot");
1414	}
1415
1416	ErrorOr<BinarySection &> FiniArraySection =
1417	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1418	if (auto EC = FiniArraySection.getError())
1419	return errorCodeToError(EC);
1420
1421	if (const Relocation *Reloc = FiniArraySection->getDynamicRelocationAt(Offset: 0)) {
1422	BC->FiniFunctionAddress = Reloc->Addend;
1423	return Error::success();
1424	}
1425
1426	if (const Relocation *Reloc = FiniArraySection->getRelocationAt(Offset: 0)) {
1427	BC->FiniFunctionAddress = Reloc->Value;
1428	return Error::success();
1429	}
1430
1431	return createStringError(EC: std::errc::not_supported,
1432	Fmt: "No relocation for first DT_FINI_ARRAY slot");
1433	}
1434
1435	void RewriteInstance::updateRtFiniReloc() {
1436	// Updating DT_FINI is handled by patchELFDynamic.
1437	if (BC->FiniAddress)
1438	return;
1439
1440	const RuntimeLibrary *RT = BC->getRuntimeLibrary();
1441	if (!RT || !RT->getRuntimeFiniAddress())
1442	return;
1443
1444	assert(BC->FiniArrayAddress && BC->FiniArraySize &&
1445	"inconsistent .fini_array state");
1446
1447	ErrorOr<BinarySection &> FiniArraySection =
1448	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1449	assert(FiniArraySection && ".fini_array removed");
1450
1451	if (std::optional<Relocation> Reloc =
1452	FiniArraySection->takeDynamicRelocationAt(Offset: 0)) {
1453	assert(Reloc->Addend == BC->FiniFunctionAddress &&
1454	"inconsistent .fini_array dynamic relocation");
1455	Reloc->Addend = RT->getRuntimeFiniAddress();
1456	FiniArraySection->addDynamicRelocation(Reloc: *Reloc);
1457	}
1458
1459	// Update the static relocation by adding a pending relocation which will get
1460	// patched when flushPendingRelocations is called in rewriteFile. Note that
1461	// flushPendingRelocations will calculate the value to patch as
1462	// "Symbol + Addend". Since we don't have a symbol, just set the addend to the
1463	// desired value.
1464	FiniArraySection->addPendingRelocation(Rel: Relocation{
1465	/*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),
1466	/*Addend*/ RT->getRuntimeFiniAddress(), /*Value*/ 0});
1467	}
1468
1469	void RewriteInstance::registerFragments() {
1470	if (!BC->HasSplitFunctions ||
1471	opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
1472	return;
1473
1474	// Process fragments with ambiguous parents separately as they are typically a
1475	// vanishing minority of cases and require expensive symbol table lookups.
1476	std::vector<std::pair<StringRef, BinaryFunction *>> AmbiguousFragments;
1477	for (auto &BFI : BC->getBinaryFunctions()) {
1478	BinaryFunction &Function = BFI.second;
1479	if (!Function.isFragment())
1480	continue;
1481	for (StringRef Name : Function.getNames()) {
1482	StringRef BaseName = NR.restore(Name);
1483	const bool IsGlobal = BaseName == Name;
1484	SmallVector<StringRef> Matches;
1485	if (!FunctionFragmentTemplate.match(String: BaseName, Matches: &Matches))
1486	continue;
1487	StringRef ParentName = Matches[1];
1488	const BinaryData *BD = BC->getBinaryDataByName(Name: ParentName);
1489	const uint64_t NumPossibleLocalParents =
1490	NR.getUniquifiedNameCount(Name: ParentName);
1491	// The most common case: single local parent fragment.
1492	if (!BD && NumPossibleLocalParents == 1) {
1493	BD = BC->getBinaryDataByName(Name: NR.getUniqueName(Name: ParentName, ID: 1));
1494	} else if (BD && (!NumPossibleLocalParents || IsGlobal)) {
1495	// Global parent and either no local candidates (second most common), or
1496	// the fragment is global as well (uncommon).
1497	} else {
1498	// Any other case: need to disambiguate using FILE symbols.
1499	AmbiguousFragments.emplace_back(args&: ParentName, args: &Function);
1500	continue;
1501	}
1502	if (BD) {
1503	BinaryFunction *BF = BC->getFunctionForSymbol(Symbol: BD->getSymbol());
1504	if (BF) {
1505	BC->registerFragment(TargetFunction&: Function, Function&: *BF);
1506	continue;
1507	}
1508	}
1509	BC->errs() << "BOLT-ERROR: parent function not found for " << Function
1510	<< '\n';
1511	exit(status: 1);
1512	}
1513	}
1514
1515	if (AmbiguousFragments.empty())
1516	return;
1517
1518	if (!BC->hasSymbolsWithFileName()) {
1519	BC->errs() << "BOLT-ERROR: input file has split functions but does not "
1520	"have FILE symbols. If the binary was stripped, preserve "
1521	"FILE symbols with --keep-file-symbols strip option\n";
1522	exit(status: 1);
1523	}
1524
1525	// The first global symbol is identified by the symbol table sh_info value.
1526	// Used as local symbol search stopping point.
1527	auto *ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
1528	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
1529	auto *SymTab = llvm::find_if(Range: cantFail(ValOrErr: Obj.sections()), P: [](const auto &Sec) {
1530	return Sec.sh_type == ELF::SHT_SYMTAB;
1531	});
1532	assert(SymTab);
1533	// Symtab sh_info contains the value one greater than the symbol table index
1534	// of the last local symbol.
1535	ELFSymbolRef LocalSymEnd = ELF64LEFile->toSymbolRef(SymTable: SymTab, SymbolNum: SymTab->sh_info);
1536
1537	for (auto &Fragment : AmbiguousFragments) {
1538	const StringRef &ParentName = Fragment.first;
1539	BinaryFunction *BF = Fragment.second;
1540	const uint64_t Address = BF->getAddress();
1541
1542	// Get fragment's own symbol
1543	const auto SymIt = llvm::find_if(
1544	Range: llvm::make_range(p: FileSymRefs.equal_range(x: Address)), P: [&](auto SI) {
1545	StringRef Name = cantFail(SI.second.getName());
1546	return Name.contains(Other: ParentName);
1547	});
1548	if (SymIt == FileSymRefs.end()) {
1549	BC->errs()
1550	<< "BOLT-ERROR: symbol lookup failed for function at address 0x"
1551	<< Twine::utohexstr(Val: Address) << '\n';
1552	exit(status: 1);
1553	}
1554
1555	// Find containing FILE symbol
1556	ELFSymbolRef Symbol = SymIt->second;
1557	auto FSI = llvm::upper_bound(Range&: FileSymbols, Value&: Symbol);
1558	if (FSI == FileSymbols.begin()) {
1559	BC->errs() << "BOLT-ERROR: owning FILE symbol not found for symbol "
1560	<< cantFail(ValOrErr: Symbol.getName()) << '\n';
1561	exit(status: 1);
1562	}
1563
1564	ELFSymbolRef StopSymbol = LocalSymEnd;
1565	if (FSI != FileSymbols.end())
1566	StopSymbol = *FSI;
1567
1568	uint64_t ParentAddress{0};
1569
1570	// BOLT split fragment symbols are emitted just before the main function
1571	// symbol.
1572	for (ELFSymbolRef NextSymbol = Symbol; NextSymbol < StopSymbol;
1573	NextSymbol.moveNext()) {
1574	StringRef Name = cantFail(ValOrErr: NextSymbol.getName());
1575	if (Name == ParentName) {
1576	ParentAddress = cantFail(ValOrErr: NextSymbol.getValue());
1577	goto registerParent;
1578	}
1579	if (Name.starts_with(Prefix: ParentName))
1580	// With multi-way splitting, there are multiple fragments with different
1581	// suffixes. Parent follows the last fragment.
1582	continue;
1583	break;
1584	}
1585
1586	// Iterate over local file symbols and check symbol names to match parent.
1587	for (ELFSymbolRef Symbol(FSI[-1]); Symbol < StopSymbol; Symbol.moveNext()) {
1588	if (cantFail(ValOrErr: Symbol.getName()) == ParentName) {
1589	ParentAddress = cantFail(ValOrErr: Symbol.getAddress());
1590	break;
1591	}
1592	}
1593
1594	registerParent:
1595	// No local parent is found, use global parent function.
1596	if (!ParentAddress)
1597	if (BinaryData *ParentBD = BC->getBinaryDataByName(Name: ParentName))
1598	ParentAddress = ParentBD->getAddress();
1599
1600	if (BinaryFunction *ParentBF =
1601	BC->getBinaryFunctionAtAddress(Address: ParentAddress)) {
1602	BC->registerFragment(TargetFunction&: *BF, Function&: *ParentBF);
1603	continue;
1604	}
1605	BC->errs() << "BOLT-ERROR: parent function not found for " << *BF << '\n';
1606	exit(status: 1);
1607	}
1608	}
1609
1610	void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,
1611	uint64_t EntryAddress,
1612	uint64_t EntrySize) {
1613	if (!TargetAddress)
1614	return;
1615
1616	auto setPLTSymbol = [&](BinaryFunction *BF, StringRef Name) {
1617	const unsigned PtrSize = BC->AsmInfo->getCodePointerSize();
1618	MCSymbol *TargetSymbol = BC->registerNameAtAddress(
1619	Name: Name.str() + "@GOT", Address: TargetAddress, Size: PtrSize, Alignment: PtrSize);
1620	BF->setPLTSymbol(TargetSymbol);
1621	};
1622
1623	BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address: EntryAddress);
1624	if (BF && BC->isAArch64()) {
1625	// Handle IFUNC trampoline with symbol
1626	setPLTSymbol(BF, BF->getOneName());
1627	return;
1628	}
1629
1630	const Relocation *Rel = BC->getDynamicRelocationAt(Address: TargetAddress);
1631	if (!Rel)
1632	return;
1633
1634	MCSymbol *Symbol = Rel->Symbol;
1635	if (!Symbol) {
1636	if (BC->isRISCV() || !Rel->Addend || !Rel->isIRelative())
1637	return;
1638
1639	// IFUNC trampoline without symbol
1640	BinaryFunction *TargetBF = BC->getBinaryFunctionAtAddress(Address: Rel->Addend);
1641	if (!TargetBF) {
1642	BC->errs()
1643	<< "BOLT-WARNING: Expected BF to be presented as IFUNC resolver at "
1644	<< Twine::utohexstr(Val: Rel->Addend) << ", skipping\n";
1645	return;
1646	}
1647
1648	Symbol = TargetBF->getSymbol();
1649	}
1650
1651	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: EntryAddress);
1652	assert(Section && "cannot get section for address");
1653	if (!BF)
1654	BF = BC->createBinaryFunction(Name: Symbol->getName().str() + "@PLT", Section&: *Section,
1655	Address: EntryAddress, Size: 0, SymbolSize: EntrySize,
1656	Alignment: Section->getAlignment());
1657	else
1658	BF->addAlternativeName(NewName: Symbol->getName().str() + "@PLT");
1659	setPLTSymbol(BF, Symbol->getName());
1660	}
1661
1662	void RewriteInstance::disassemblePLTInstruction(const BinarySection &Section,
1663	uint64_t InstrOffset,
1664	MCInst &Instruction,
1665	uint64_t &InstrSize) {
1666	const uint64_t SectionAddress = Section.getAddress();
1667	const uint64_t SectionSize = Section.getSize();
1668	StringRef PLTContents = Section.getContents();
1669	ArrayRef<uint8_t> PLTData(
1670	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1671
1672	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1673	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1674	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1675	CStream&: nulls())) {
1676	BC->errs()
1677	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1678	<< Section.getName() << formatv(Fmt: " at offset {0:x}\n", Vals&: InstrOffset);
1679	exit(status: 1);
1680	}
1681	}
1682
1683	void RewriteInstance::disassemblePLTSectionAArch64(BinarySection &Section) {
1684	const uint64_t SectionAddress = Section.getAddress();
1685	const uint64_t SectionSize = Section.getSize();
1686
1687	uint64_t InstrOffset = 0;
1688	// Locate new plt entry
1689	while (InstrOffset < SectionSize) {
1690	InstructionListType Instructions;
1691	MCInst Instruction;
1692	uint64_t EntryOffset = InstrOffset;
1693	uint64_t EntrySize = 0;
1694	uint64_t InstrSize;
1695	// Loop through entry instructions
1696	while (InstrOffset < SectionSize) {
1697	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1698	EntrySize += InstrSize;
1699	if (!BC->MIB->isIndirectBranch(Inst: Instruction)) {
1700	Instructions.emplace_back(args&: Instruction);
1701	InstrOffset += InstrSize;
1702	continue;
1703	}
1704
1705	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1706	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1707	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1708
1709	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1710	break;
1711	}
1712
1713	// Branch instruction
1714	InstrOffset += InstrSize;
1715
1716	// Skip nops if any
1717	while (InstrOffset < SectionSize) {
1718	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1719	if (!BC->MIB->isNoop(Inst: Instruction))
1720	break;
1721
1722	InstrOffset += InstrSize;
1723	}
1724	}
1725	}
1726
1727	void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) {
1728	const uint64_t SectionAddress = Section.getAddress();
1729	const uint64_t SectionSize = Section.getSize();
1730	StringRef PLTContents = Section.getContents();
1731	ArrayRef<uint8_t> PLTData(
1732	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1733
1734	auto disassembleInstruction = [&](uint64_t InstrOffset, MCInst &Instruction,
1735	uint64_t &InstrSize) {
1736	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1737	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1738	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1739	CStream&: nulls())) {
1740	BC->errs()
1741	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1742	<< Section.getName() << " at offset 0x"
1743	<< Twine::utohexstr(Val: InstrOffset) << '\n';
1744	exit(status: 1);
1745	}
1746	};
1747
1748	// Skip the first special entry since no relocation points to it.
1749	uint64_t InstrOffset = 32;
1750
1751	while (InstrOffset < SectionSize) {
1752	InstructionListType Instructions;
1753	MCInst Instruction;
1754	const uint64_t EntryOffset = InstrOffset;
1755	const uint64_t EntrySize = 16;
1756	uint64_t InstrSize;
1757
1758	while (InstrOffset < EntryOffset + EntrySize) {
1759	disassembleInstruction(InstrOffset, Instruction, InstrSize);
1760	Instructions.emplace_back(args&: Instruction);
1761	InstrOffset += InstrSize;
1762	}
1763
1764	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1765	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1766	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1767
1768	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1769	}
1770	}
1771
1772	void RewriteInstance::disassemblePLTSectionX86(BinarySection &Section,
1773	uint64_t EntrySize) {
1774	const uint64_t SectionAddress = Section.getAddress();
1775	const uint64_t SectionSize = Section.getSize();
1776
1777	for (uint64_t EntryOffset = 0; EntryOffset + EntrySize <= SectionSize;
1778	EntryOffset += EntrySize) {
1779	MCInst Instruction;
1780	uint64_t InstrSize, InstrOffset = EntryOffset;
1781	while (InstrOffset < EntryOffset + EntrySize) {
1782	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1783	// Check if the entry size needs adjustment.
1784	if (EntryOffset == 0 && BC->MIB->isTerminateBranch(Inst: Instruction) &&
1785	EntrySize == 8)
1786	EntrySize = 16;
1787
1788	if (BC->MIB->isIndirectBranch(Inst: Instruction))
1789	break;
1790
1791	InstrOffset += InstrSize;
1792	}
1793
1794	if (InstrOffset + InstrSize > EntryOffset + EntrySize)
1795	continue;
1796
1797	uint64_t TargetAddress;
1798	if (!BC->MIB->evaluateMemOperandTarget(Inst: Instruction, Target&: TargetAddress,
1799	Address: SectionAddress + InstrOffset,
1800	Size: InstrSize)) {
1801	BC->errs() << "BOLT-ERROR: error evaluating PLT instruction at offset 0x"
1802	<< Twine::utohexstr(Val: SectionAddress + InstrOffset) << '\n';
1803	exit(status: 1);
1804	}
1805
1806	createPLTBinaryFunction(TargetAddress, EntryAddress: SectionAddress + EntryOffset,
1807	EntrySize);
1808	}
1809	}
1810
1811	void RewriteInstance::disassemblePLT() {
1812	auto analyzeOnePLTSection = [&](BinarySection &Section, uint64_t EntrySize) {
1813	if (BC->isAArch64())
1814	return disassemblePLTSectionAArch64(Section);
1815	if (BC->isRISCV())
1816	return disassemblePLTSectionRISCV(Section);
1817	if (BC->isX86())
1818	return disassemblePLTSectionX86(Section, EntrySize);
1819	llvm_unreachable("Unmplemented PLT");
1820	};
1821
1822	for (BinarySection &Section : BC->allocatableSections()) {
1823	const PLTSectionInfo *PLTSI = getPLTSectionInfo(SectionName: Section.getName());
1824	if (!PLTSI)
1825	continue;
1826
1827	analyzeOnePLTSection(Section, PLTSI->EntrySize);
1828
1829	BinaryFunction *PltBF;
1830	auto BFIter = BC->getBinaryFunctions().find(x: Section.getAddress());
1831	if (BFIter != BC->getBinaryFunctions().end()) {
1832	PltBF = &BFIter->second;
1833	} else {
1834	// If we did not register any function at the start of the section,
1835	// then it must be a general PLT entry. Add a function at the location.
1836	PltBF = BC->createBinaryFunction(
1837	Name: "__BOLT_PSEUDO_" + Section.getName().str(), Section,
1838	Address: Section.getAddress(), Size: 0, SymbolSize: PLTSI->EntrySize, Alignment: Section.getAlignment());
1839	}
1840	PltBF->setPseudo(true);
1841	}
1842	}
1843
1844	void RewriteInstance::adjustFunctionBoundaries() {
1845	for (auto BFI = BC->getBinaryFunctions().begin(),
1846	BFE = BC->getBinaryFunctions().end();
1847	BFI != BFE; ++BFI) {
1848	BinaryFunction &Function = BFI->second;
1849	const BinaryFunction *NextFunction = nullptr;
1850	if (std::next(x: BFI) != BFE)
1851	NextFunction = &std::next(x: BFI)->second;
1852
1853	// Check if there's a symbol or a function with a larger address in the
1854	// same section. If there is - it determines the maximum size for the
1855	// current function. Otherwise, it is the size of a containing section
1856	// the defines it.
1857	//
1858	// NOTE: ignore some symbols that could be tolerated inside the body
1859	// of a function.
1860	auto NextSymRefI = FileSymRefs.upper_bound(x: Function.getAddress());
1861	while (NextSymRefI != FileSymRefs.end()) {
1862	SymbolRef &Symbol = NextSymRefI->second;
1863	const uint64_t SymbolAddress = NextSymRefI->first;
1864	const uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1865
1866	if (NextFunction && SymbolAddress >= NextFunction->getAddress())
1867	break;
1868
1869	if (!Function.isSymbolValidInScope(Symbol, SymbolSize))
1870	break;
1871
1872	// Skip basic block labels. This happens on RISC-V with linker relaxation
1873	// enabled because every branch needs a relocation and corresponding
1874	// symbol. We don't want to add such symbols as entry points.
1875	const auto PrivateLabelPrefix = BC->AsmInfo->getPrivateLabelPrefix();
1876	if (!PrivateLabelPrefix.empty() &&
1877	cantFail(ValOrErr: Symbol.getName()).starts_with(Prefix: PrivateLabelPrefix)) {
1878	++NextSymRefI;
1879	continue;
1880	}
1881
1882	// This is potentially another entry point into the function.
1883	uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();
1884	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "
1885	<< Function << " at offset 0x"
1886	<< Twine::utohexstr(EntryOffset) << '\n');
1887	Function.addEntryPointAtOffset(Offset: EntryOffset);
1888
1889	++NextSymRefI;
1890	}
1891
1892	// Function runs at most till the end of the containing section.
1893	uint64_t NextObjectAddress = Function.getOriginSection()->getEndAddress();
1894	// Or till the next object marked by a symbol.
1895	if (NextSymRefI != FileSymRefs.end())
1896	NextObjectAddress = std::min(a: NextSymRefI->first, b: NextObjectAddress);
1897
1898	// Or till the next function not marked by a symbol.
1899	if (NextFunction)
1900	NextObjectAddress =
1901	std::min(a: NextFunction->getAddress(), b: NextObjectAddress);
1902
1903	const uint64_t MaxSize = NextObjectAddress - Function.getAddress();
1904	if (MaxSize < Function.getSize()) {
1905	BC->errs() << "BOLT-ERROR: symbol seen in the middle of the function "
1906	<< Function << ". Skipping.\n";
1907	Function.setSimple(false);
1908	Function.setMaxSize(Function.getSize());
1909	continue;
1910	}
1911	Function.setMaxSize(MaxSize);
1912	if (!Function.getSize() && Function.isSimple()) {
1913	// Some assembly functions have their size set to 0, use the max
1914	// size as their real size.
1915	if (opts::Verbosity >= 1)
1916	BC->outs() << "BOLT-INFO: setting size of function " << Function
1917	<< " to " << Function.getMaxSize() << " (was 0)\n";
1918	Function.setSize(Function.getMaxSize());
1919	}
1920	}
1921	}
1922
1923	void RewriteInstance::relocateEHFrameSection() {
1924	assert(EHFrameSection && "Non-empty .eh_frame section expected.");
1925
1926	BinarySection *RelocatedEHFrameSection =
1927	getSection(Name: ".relocated" + getEHFrameSectionName());
1928	assert(RelocatedEHFrameSection &&
1929	"Relocated eh_frame section should be preregistered.");
1930	DWARFDataExtractor DE(EHFrameSection->getContents(),
1931	BC->AsmInfo->isLittleEndian(),
1932	BC->AsmInfo->getCodePointerSize());
1933	auto createReloc = [&](uint64_t Value, uint64_t Offset, uint64_t DwarfType) {
1934	if (DwarfType == dwarf::DW_EH_PE_omit)
1935	return;
1936
1937	// Only fix references that are relative to other locations.
1938	if (!(DwarfType & dwarf::DW_EH_PE_pcrel) &&
1939	!(DwarfType & dwarf::DW_EH_PE_textrel) &&
1940	!(DwarfType & dwarf::DW_EH_PE_funcrel) &&
1941	!(DwarfType & dwarf::DW_EH_PE_datarel))
1942	return;
1943
1944	if (!(DwarfType & dwarf::DW_EH_PE_sdata4))
1945	return;
1946
1947	uint32_t RelType;
1948	switch (DwarfType & 0x0f) {
1949	default:
1950	llvm_unreachable("unsupported DWARF encoding type");
1951	case dwarf::DW_EH_PE_sdata4:
1952	case dwarf::DW_EH_PE_udata4:
1953	RelType = Relocation::getPC32();
1954	Offset -= 4;
1955	break;
1956	case dwarf::DW_EH_PE_sdata8:
1957	case dwarf::DW_EH_PE_udata8:
1958	RelType = Relocation::getPC64();
1959	Offset -= 8;
1960	break;
1961	}
1962
1963	// Create a relocation against an absolute value since the goal is to
1964	// preserve the contents of the section independent of the new values
1965	// of referenced symbols.
1966	RelocatedEHFrameSection->addRelocation(Offset, Symbol: nullptr, Type: RelType, Addend: Value);
1967	};
1968
1969	Error E = EHFrameParser::parse(Data: DE, EHFrameAddress: EHFrameSection->getAddress(), PatcherCallback: createReloc);
1970	check_error(E: std::move(E), Message: "failed to patch EH frame");
1971	}
1972
1973	Error RewriteInstance::readSpecialSections() {
1974	NamedRegionTimer T("readSpecialSections", "read special sections",
1975	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
1976
1977	bool HasTextRelocations = false;
1978	bool HasSymbolTable = false;
1979	bool HasDebugInfo = false;
1980
1981	// Process special sections.
1982	for (const SectionRef &Section : InputFile->sections()) {
1983	Expected<StringRef> SectionNameOrErr = Section.getName();
1984	check_error(E: SectionNameOrErr.takeError(), Message: "cannot get section name");
1985	StringRef SectionName = *SectionNameOrErr;
1986
1987	if (Error E = Section.getContents().takeError())
1988	return E;
1989	BC->registerSection(Section);
1990	LLVM_DEBUG(
1991	dbgs() << "BOLT-DEBUG: registering section " << SectionName << " @ 0x"
1992	<< Twine::utohexstr(Section.getAddress()) << ":0x"
1993	<< Twine::utohexstr(Section.getAddress() + Section.getSize())
1994	<< "\n");
1995	if (isDebugSection(SectionName))
1996	HasDebugInfo = true;
1997	}
1998
1999	// Set IsRelro section attribute based on PT_GNU_RELRO segment.
2000	markGnuRelroSections();
2001
2002	if (HasDebugInfo && !opts::UpdateDebugSections && !opts::AggregateOnly) {
2003	BC->errs() << "BOLT-WARNING: debug info will be stripped from the binary. "
2004	"Use -update-debug-sections to keep it.\n";
2005	}
2006
2007	HasTextRelocations = (bool)BC->getUniqueSectionByName(
2008	SectionName: ".rela" + std::string(BC->getMainCodeSectionName()));
2009	HasSymbolTable = (bool)BC->getUniqueSectionByName(SectionName: ".symtab");
2010	EHFrameSection = BC->getUniqueSectionByName(SectionName: ".eh_frame");
2011
2012	if (ErrorOr<BinarySection &> BATSec =
2013	BC->getUniqueSectionByName(SectionName: BoltAddressTranslation::SECTION_NAME)) {
2014	BC->HasBATSection = true;
2015	// Do not read BAT when plotting a heatmap
2016	if (opts::HeatmapMode != opts::HeatmapModeKind::HM_Exclusive) {
2017	if (std::error_code EC = BAT->parse(OS&: BC->outs(), Buf: BATSec->getContents())) {
2018	BC->errs() << "BOLT-ERROR: failed to parse BOLT address translation "
2019	"table.\n";
2020	exit(status: 1);
2021	}
2022	}
2023	}
2024
2025	if (opts::PrintSections) {
2026	BC->outs() << "BOLT-INFO: Sections from original binary:\n";
2027	BC->printSections(OS&: BC->outs());
2028	}
2029
2030	if (opts::RelocationMode == cl::BOU_TRUE && !HasTextRelocations) {
2031	BC->errs()
2032	<< "BOLT-ERROR: relocations against code are missing from the input "
2033	"file. Cannot proceed in relocations mode (-relocs).\n";
2034	exit(status: 1);
2035	}
2036
2037	BC->HasRelocations =
2038	HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE);
2039
2040	if (BC->IsLinuxKernel && BC->HasRelocations) {
2041	BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";
2042	BC->HasRelocations = false;
2043	}
2044
2045	BC->IsStripped = !HasSymbolTable;
2046
2047	if (BC->IsStripped && !opts::AllowStripped) {
2048	BC->errs()
2049	<< "BOLT-ERROR: stripped binaries are not supported. If you know "
2050	"what you're doing, use --allow-stripped to proceed";
2051	exit(status: 1);
2052	}
2053
2054	// Force non-relocation mode for heatmap generation
2055	if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)
2056	BC->HasRelocations = false;
2057
2058	if (BC->HasRelocations)
2059	BC->outs() << "BOLT-INFO: enabling " << (opts::StrictMode ? "strict " : "")
2060	<< "relocation mode\n";
2061
2062	// Read EH frame for function boundaries info.
2063	Expected<const DWARFDebugFrame *> EHFrameOrError = BC->DwCtx->getEHFrame();
2064	if (!EHFrameOrError)
2065	report_error(Message: "expected valid eh_frame section", E: EHFrameOrError.takeError());
2066	CFIRdWrt.reset(p: new CFIReaderWriter(*BC, *EHFrameOrError.get()));
2067
2068	processSectionMetadata();
2069
2070	// Read .dynamic/PT_DYNAMIC.
2071	return readELFDynamic();
2072	}
2073
2074	void RewriteInstance::adjustCommandLineOptions() {
2075	if (BC->isAArch64() && !BC->HasRelocations)
2076	BC->errs() << "BOLT-WARNING: non-relocation mode for AArch64 is not fully "
2077	"supported\n";
2078
2079	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
2080	RtLibrary->adjustCommandLineOptions(BC: *BC);
2081
2082	if (BC->isX86() && BC->MAB->allowAutoPadding()) {
2083	if (!BC->HasRelocations) {
2084	BC->errs()
2085	<< "BOLT-ERROR: cannot apply mitigations for Intel JCC erratum in "
2086	"non-relocation mode\n";
2087	exit(status: 1);
2088	}
2089	BC->outs()
2090	<< "BOLT-WARNING: using mitigation for Intel JCC erratum, layout "
2091	"may take several minutes\n";
2092	}
2093
2094	if (opts::SplitEH && !BC->HasRelocations) {
2095	BC->errs() << "BOLT-WARNING: disabling -split-eh in non-relocation mode\n";
2096	opts::SplitEH = false;
2097	}
2098
2099	if (opts::StrictMode && !BC->HasRelocations) {
2100	BC->errs()
2101	<< "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "
2102	"mode\n";
2103	opts::StrictMode = false;
2104	}
2105
2106	if (BC->HasRelocations && opts::AggregateOnly &&
2107	!opts::StrictMode.getNumOccurrences()) {
2108	BC->outs() << "BOLT-INFO: enabling strict relocation mode for aggregation "
2109	"purposes\n";
2110	opts::StrictMode = true;
2111	}
2112
2113	if (!BC->HasRelocations &&
2114	opts::ReorderFunctions != ReorderFunctions::RT_NONE) {
2115	BC->errs() << "BOLT-ERROR: function reordering only works when "
2116	<< "relocations are enabled\n";
2117	exit(status: 1);
2118	}
2119
2120	if (!BC->HasRelocations &&
2121	opts::ICF == IdenticalCodeFolding::ICFLevel::Safe) {
2122	BC->errs() << "BOLT-ERROR: binary built without relocations. Safe ICF is "
2123	"not supported\n";
2124	exit(status: 1);
2125	}
2126
2127	if (opts::Instrument ||
2128	(opts::ReorderFunctions != ReorderFunctions::RT_NONE &&
2129	!opts::HotText.getNumOccurrences())) {
2130	opts::HotText = true;
2131	} else if (opts::HotText && !BC->HasRelocations) {
2132	BC->errs() << "BOLT-WARNING: hot text is disabled in non-relocation mode\n";
2133	opts::HotText = false;
2134	}
2135
2136	if (opts::Instrument && opts::UseGnuStack) {
2137	BC->errs() << "BOLT-ERROR: cannot avoid having writeable and executable "
2138	"segment in instrumented binary if program headers will be "
2139	"updated in place\n";
2140	exit(status: 1);
2141	}
2142
2143	if (opts::HotText && opts::HotTextMoveSections.getNumOccurrences() == 0) {
2144	opts::HotTextMoveSections.addValue(V: ".stub");
2145	opts::HotTextMoveSections.addValue(V: ".mover");
2146	opts::HotTextMoveSections.addValue(V: ".never_hugify");
2147	}
2148
2149	if (opts::UseOldText && !BC->OldTextSectionAddress) {
2150	BC->errs()
2151	<< "BOLT-WARNING: cannot use old .text as the section was not found"
2152	"\n";
2153	opts::UseOldText = false;
2154	}
2155	if (opts::UseOldText && !BC->HasRelocations) {
2156	BC->errs() << "BOLT-WARNING: cannot use old .text in non-relocation mode\n";
2157	opts::UseOldText = false;
2158	}
2159
2160	if (!opts::AlignText.getNumOccurrences())
2161	opts::AlignText = BC->PageAlign;
2162
2163	if (opts::AlignText < opts::AlignFunctions)
2164	opts::AlignText = (unsigned)opts::AlignFunctions;
2165
2166	if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode &&
2167	!opts::UseOldText)
2168	opts::Lite = true;
2169
2170	if (opts::Lite && opts::UseOldText) {
2171	BC->errs() << "BOLT-WARNING: cannot combine -lite with -use-old-text. "
2172	"Disabling -use-old-text.\n";
2173	opts::UseOldText = false;
2174	}
2175
2176	if (opts::Lite && opts::StrictMode) {
2177	BC->errs()
2178	<< "BOLT-ERROR: -strict and -lite cannot be used at the same time\n";
2179	exit(status: 1);
2180	}
2181
2182	if (opts::Lite)
2183	BC->outs() << "BOLT-INFO: enabling lite mode\n";
2184
2185	if (BC->IsLinuxKernel) {
2186	if (!opts::KeepNops.getNumOccurrences())
2187	opts::KeepNops = true;
2188
2189	// Linux kernel may resume execution after a trap instruction in some cases.
2190	if (!opts::TerminalTrap.getNumOccurrences())
2191	opts::TerminalTrap = false;
2192	}
2193	}
2194
2195	namespace {
2196	template <typename ELFT>
2197	int64_t getRelocationAddend(const ELFObjectFile<ELFT> *Obj,
2198	const RelocationRef &RelRef) {
2199	using ELFShdrTy = typename ELFT::Shdr;
2200	using Elf_Rela = typename ELFT::Rela;
2201	int64_t Addend = 0;
2202	const ELFFile<ELFT> &EF = Obj->getELFFile();
2203	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2204	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2205	switch (RelocationSection->sh_type) {
2206	default:
2207	llvm_unreachable("unexpected relocation section type");
2208	case ELF::SHT_REL:
2209	break;
2210	case ELF::SHT_RELA: {
2211	const Elf_Rela *RelA = Obj->getRela(Rel);
2212	Addend = RelA->r_addend;
2213	break;
2214	}
2215	}
2216
2217	return Addend;
2218	}
2219
2220	int64_t getRelocationAddend(const ELFObjectFileBase *Obj,
2221	const RelocationRef &Rel) {
2222	return getRelocationAddend(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2223	}
2224
2225	template <typename ELFT>
2226	uint32_t getRelocationSymbol(const ELFObjectFile<ELFT> *Obj,
2227	const RelocationRef &RelRef) {
2228	using ELFShdrTy = typename ELFT::Shdr;
2229	uint32_t Symbol = 0;
2230	const ELFFile<ELFT> &EF = Obj->getELFFile();
2231	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2232	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2233	switch (RelocationSection->sh_type) {
2234	default:
2235	llvm_unreachable("unexpected relocation section type");
2236	case ELF::SHT_REL:
2237	Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL());
2238	break;
2239	case ELF::SHT_RELA:
2240	Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL());
2241	break;
2242	}
2243
2244	return Symbol;
2245	}
2246
2247	uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj,
2248	const RelocationRef &Rel) {
2249	return getRelocationSymbol(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2250	}
2251	} // anonymous namespace
2252
2253	bool RewriteInstance::analyzeRelocation(
2254	const RelocationRef &Rel, uint32_t &RType, std::string &SymbolName,
2255	bool &IsSectionRelocation, uint64_t &SymbolAddress, int64_t &Addend,
2256	uint64_t &ExtractedValue, bool &Skip) const {
2257	Skip = false;
2258	if (!Relocation::isSupported(Type: RType))
2259	return false;
2260
2261	auto IsWeakReference = [](const SymbolRef &Symbol) {
2262	Expected<uint32_t> SymFlagsOrErr = Symbol.getFlags();
2263	if (!SymFlagsOrErr)
2264	return false;
2265	return (*SymFlagsOrErr & SymbolRef::SF_Undefined) &&
2266	(*SymFlagsOrErr & SymbolRef::SF_Weak);
2267	};
2268
2269	const bool IsAArch64 = BC->isAArch64();
2270
2271	const size_t RelSize = Relocation::getSizeForType(Type: RType);
2272
2273	ErrorOr<uint64_t> Value =
2274	BC->getUnsignedValueAtAddress(Address: Rel.getOffset(), Size: RelSize);
2275	assert(Value && "failed to extract relocated value");
2276
2277	ExtractedValue = Relocation::extractValue(Type: RType, Contents: *Value, PC: Rel.getOffset());
2278	Addend = getRelocationAddend(Obj: InputFile, Rel);
2279
2280	const bool IsPCRelative = Relocation::isPCRelative(Type: RType);
2281	const uint64_t PCRelOffset = IsPCRelative && !IsAArch64 ? Rel.getOffset() : 0;
2282	bool SkipVerification = false;
2283	auto SymbolIter = Rel.getSymbol();
2284	if (SymbolIter == InputFile->symbol_end()) {
2285	SymbolAddress = ExtractedValue - Addend + PCRelOffset;
2286	MCSymbol *RelSymbol =
2287	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "RELSYMat");
2288	SymbolName = std::string(RelSymbol->getName());
2289	IsSectionRelocation = false;
2290	} else {
2291	const SymbolRef &Symbol = *SymbolIter;
2292	SymbolName = std::string(cantFail(ValOrErr: Symbol.getName()));
2293	SymbolAddress = cantFail(ValOrErr: Symbol.getAddress());
2294	SkipVerification = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Other);
2295	// Section symbols are marked as ST_Debug.
2296	IsSectionRelocation = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Debug);
2297	// Check for PLT entry registered with symbol name
2298	if (!SymbolAddress && !IsWeakReference(Symbol) &&
2299	(IsAArch64 || BC->isRISCV())) {
2300	const BinaryData *BD = BC->getPLTBinaryDataByName(Name: SymbolName);
2301	SymbolAddress = BD ? BD->getAddress() : 0;
2302	}
2303	}
2304	// For PIE or dynamic libs, the linker may choose not to put the relocation
2305	// result at the address if it is a X86_64_64 one because it will emit a
2306	// dynamic relocation (X86_RELATIVE) for the dynamic linker and loader to
2307	// resolve it at run time. The static relocation result goes as the addend
2308	// of the dynamic relocation in this case. We can't verify these cases.
2309	// FIXME: perhaps we can try to find if it really emitted a corresponding
2310	// RELATIVE relocation at this offset with the correct value as the addend.
2311	if (!BC->HasFixedLoadAddress && RelSize == 8)
2312	SkipVerification = true;
2313
2314	if (IsSectionRelocation && !IsAArch64) {
2315	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2316	assert(Section && "section expected for section relocation");
2317	SymbolName = "section " + std::string(Section->getName());
2318	// Convert section symbol relocations to regular relocations inside
2319	// non-section symbols.
2320	if (Section->containsAddress(Address: ExtractedValue) && !IsPCRelative) {
2321	SymbolAddress = ExtractedValue;
2322	Addend = 0;
2323	} else {
2324	Addend = ExtractedValue - (SymbolAddress - PCRelOffset);
2325	}
2326	}
2327
2328	// GOT relocation can cause the underlying instruction to be modified by the
2329	// linker, resulting in the extracted value being different from the actual
2330	// symbol. It's also possible to have a GOT entry for a symbol defined in the
2331	// binary. In the latter case, the instruction can be using the GOT version
2332	// causing the extracted value mismatch. Similar cases can happen for TLS.
2333	// Pass the relocation information as is to the disassembler and let it decide
2334	// how to use it for the operand symbolization.
2335	if (Relocation::isGOT(Type: RType) || Relocation::isTLS(Type: RType)) {
2336	SkipVerification = true;
2337	} else if (!SymbolAddress) {
2338	assert(!IsSectionRelocation);
2339	if (ExtractedValue || Addend == 0 || IsPCRelative) {
2340	SymbolAddress =
2341	truncateToSize(Value: ExtractedValue - Addend + PCRelOffset, Bytes: RelSize);
2342	} else {
2343	// This is weird case. The extracted value is zero but the addend is
2344	// non-zero and the relocation is not pc-rel. Using the previous logic,
2345	// the SymbolAddress would end up as a huge number. Seen in
2346	// exceptions_pic.test.
2347	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocation @ 0x"
2348	<< Twine::utohexstr(Rel.getOffset())
2349	<< " value does not match addend for "
2350	<< "relocation to undefined symbol.\n");
2351	return true;
2352	}
2353	}
2354
2355	auto verifyExtractedValue = [&]() {
2356	if (SkipVerification)
2357	return true;
2358
2359	if (IsAArch64 || BC->isRISCV())
2360	return true;
2361
2362	if (SymbolName == "__hot_start" || SymbolName == "__hot_end")
2363	return true;
2364
2365	if (RType == ELF::R_X86_64_PLT32)
2366	return true;
2367
2368	return truncateToSize(Value: ExtractedValue, Bytes: RelSize) ==
2369	truncateToSize(Value: SymbolAddress + Addend - PCRelOffset, Bytes: RelSize);
2370	};
2371
2372	(void)verifyExtractedValue;
2373	assert(verifyExtractedValue() && "mismatched extracted relocation value");
2374
2375	return true;
2376	}
2377
2378	void RewriteInstance::processDynamicRelocations() {
2379	// Read .relr.dyn section containing compressed R_*_RELATIVE relocations.
2380	if (DynamicRelrSize > 0) {
2381	ErrorOr<BinarySection &> DynamicRelrSectionOrErr =
2382	BC->getSectionForAddress(Address: *DynamicRelrAddress);
2383	if (!DynamicRelrSectionOrErr)
2384	report_error(Message: "unable to find section corresponding to DT_RELR",
2385	EC: DynamicRelrSectionOrErr.getError());
2386	if (DynamicRelrSectionOrErr->getSize() != DynamicRelrSize)
2387	report_error(Message: "section size mismatch for DT_RELRSZ",
2388	EC: errc::executable_format_error);
2389	readDynamicRelrRelocations(Section&: *DynamicRelrSectionOrErr);
2390	}
2391
2392	// Read relocations for PLT - DT_JMPREL.
2393	if (PLTRelocationsSize > 0) {
2394	ErrorOr<BinarySection &> PLTRelSectionOrErr =
2395	BC->getSectionForAddress(Address: *PLTRelocationsAddress);
2396	if (!PLTRelSectionOrErr)
2397	report_error(Message: "unable to find section corresponding to DT_JMPREL",
2398	EC: PLTRelSectionOrErr.getError());
2399	if (PLTRelSectionOrErr->getSize() != PLTRelocationsSize)
2400	report_error(Message: "section size mismatch for DT_PLTRELSZ",
2401	EC: errc::executable_format_error);
2402	readDynamicRelocations(Section: PLTRelSectionOrErr->getSectionRef(),
2403	/*IsJmpRel*/ true);
2404	}
2405
2406	// The rest of dynamic relocations - DT_RELA.
2407	// The static executable might have .rela.dyn secion and not have PT_DYNAMIC
2408	if (!DynamicRelocationsSize && BC->IsStaticExecutable) {
2409	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2410	BC->getUniqueSectionByName(SectionName: getRelaDynSectionName());
2411	if (DynamicRelSectionOrErr) {
2412	DynamicRelocationsAddress = DynamicRelSectionOrErr->getAddress();
2413	DynamicRelocationsSize = DynamicRelSectionOrErr->getSize();
2414	const SectionRef &SectionRef = DynamicRelSectionOrErr->getSectionRef();
2415	DynamicRelativeRelocationsCount = std::distance(
2416	first: SectionRef.relocation_begin(), last: SectionRef.relocation_end());
2417	}
2418	}
2419
2420	if (DynamicRelocationsSize > 0) {
2421	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2422	BC->getSectionForAddress(Address: *DynamicRelocationsAddress);
2423	if (!DynamicRelSectionOrErr)
2424	report_error(Message: "unable to find section corresponding to DT_RELA",
2425	EC: DynamicRelSectionOrErr.getError());
2426	auto DynamicRelSectionSize = DynamicRelSectionOrErr->getSize();
2427	// On RISC-V DT_RELASZ seems to include both .rela.dyn and .rela.plt
2428	if (DynamicRelocationsSize == DynamicRelSectionSize + PLTRelocationsSize)
2429	DynamicRelocationsSize = DynamicRelSectionSize;
2430	if (DynamicRelSectionSize != DynamicRelocationsSize)
2431	report_error(Message: "section size mismatch for DT_RELASZ",
2432	EC: errc::executable_format_error);
2433	readDynamicRelocations(Section: DynamicRelSectionOrErr->getSectionRef(),
2434	/*IsJmpRel*/ false);
2435	}
2436	}
2437
2438	void RewriteInstance::processRelocations() {
2439	if (!BC->HasRelocations)
2440	return;
2441
2442	for (const SectionRef &Section : InputFile->sections()) {
2443	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2444	if (SecIter == InputFile->section_end())
2445	continue;
2446	if (BinarySection(*BC, Section).isAllocatable())
2447	continue;
2448
2449	readRelocations(Section);
2450	}
2451
2452	if (NumFailedRelocations)
2453	BC->errs() << "BOLT-WARNING: Failed to analyze " << NumFailedRelocations
2454	<< " relocations\n";
2455	}
2456
2457	void RewriteInstance::readDynamicRelocations(const SectionRef &Section,
2458	bool IsJmpRel) {
2459	assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected");
2460
2461	LLVM_DEBUG({
2462	StringRef SectionName = cantFail(Section.getName());
2463	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2464	<< ":\n";
2465	});
2466
2467	for (const RelocationRef &Rel : Section.relocations()) {
2468	const uint32_t RType = Relocation::getType(Rel);
2469	if (Relocation::isNone(Type: RType))
2470	continue;
2471
2472	StringRef SymbolName = "<none>";
2473	MCSymbol *Symbol = nullptr;
2474	uint64_t SymbolAddress = 0;
2475	const uint64_t Addend = getRelocationAddend(Obj: InputFile, Rel);
2476
2477	symbol_iterator SymbolIter = Rel.getSymbol();
2478	if (SymbolIter != InputFile->symbol_end()) {
2479	SymbolName = cantFail(ValOrErr: SymbolIter->getName());
2480	BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName);
2481	Symbol = BD ? BD->getSymbol()
2482	: BC->getOrCreateUndefinedGlobalSymbol(Name: SymbolName);
2483	SymbolAddress = cantFail(ValOrErr: SymbolIter->getAddress());
2484	(void)SymbolAddress;
2485	}
2486
2487	LLVM_DEBUG(
2488	SmallString<16> TypeName;
2489	Rel.getTypeName(TypeName);
2490	dbgs() << "BOLT-DEBUG: dynamic relocation at 0x"
2491	<< Twine::utohexstr(Rel.getOffset()) << " : " << TypeName
2492	<< " : " << SymbolName << " : " << Twine::utohexstr(SymbolAddress)
2493	<< " : + 0x" << Twine::utohexstr(Addend) << '\n'
2494	);
2495
2496	if (IsJmpRel)
2497	IsJmpRelocation[RType] = true;
2498
2499	if (Symbol)
2500	SymbolIndex[Symbol] = getRelocationSymbol(Obj: InputFile, Rel);
2501
2502	const uint64_t ReferencedAddress = SymbolAddress + Addend;
2503	BinaryFunction *Func =
2504	BC->getBinaryFunctionContainingAddress(Address: ReferencedAddress);
2505
2506	if (Relocation::isRelative(Type: RType) && SymbolAddress == 0) {
2507	if (Func) {
2508	if (!Func->isInConstantIsland(Address: ReferencedAddress)) {
2509	if (const uint64_t ReferenceOffset =
2510	ReferencedAddress - Func->getAddress()) {
2511	Func->addEntryPointAtOffset(Offset: ReferenceOffset);
2512	}
2513	} else {
2514	BC->errs() << "BOLT-ERROR: referenced address at 0x"
2515	<< Twine::utohexstr(Val: ReferencedAddress)
2516	<< " is in constant island of function " << *Func << "\n";
2517	exit(status: 1);
2518	}
2519	}
2520	} else if (Relocation::isRelative(Type: RType) && SymbolAddress != 0) {
2521	BC->errs() << "BOLT-ERROR: symbol address non zero for RELATIVE "
2522	"relocation type\n";
2523	exit(status: 1);
2524	}
2525
2526	BC->addDynamicRelocation(Address: Rel.getOffset(), Symbol, Type: RType, Addend);
2527	}
2528	}
2529
2530	void RewriteInstance::readDynamicRelrRelocations(BinarySection &Section) {
2531	assert(Section.isAllocatable() && "allocatable expected");
2532
2533	LLVM_DEBUG({
2534	StringRef SectionName = Section.getName();
2535	dbgs() << "BOLT-DEBUG: reading relocations in section " << SectionName
2536	<< ":\n";
2537	});
2538
2539	const uint32_t RType = Relocation::getRelative();
2540	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
2541	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
2542
2543	auto ExtractAddendValue = [&](uint64_t Address) -> uint64_t {
2544	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
2545	assert(Section && "cannot get section for data address from RELR");
2546	DataExtractor DE = DataExtractor(Section->getContents(),
2547	BC->AsmInfo->isLittleEndian(), PSize);
2548	uint64_t Offset = Address - Section->getAddress();
2549	return DE.getUnsigned(offset_ptr: &Offset, byte_size: PSize);
2550	};
2551
2552	auto AddRelocation = [&](uint64_t Address) {
2553	uint64_t Addend = ExtractAddendValue(Address);
2554	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: R_*_RELATIVE relocation at 0x"
2555	<< Twine::utohexstr(Address) << " to 0x"
2556	<< Twine::utohexstr(Addend) << '\n';);
2557	BC->addDynamicRelocation(Address, Symbol: nullptr, Type: RType, Addend);
2558	};
2559
2560	DataExtractor DE = DataExtractor(Section.getContents(),
2561	BC->AsmInfo->isLittleEndian(), PSize);
2562	uint64_t Offset = 0, Address = 0;
2563	uint64_t RelrCount = DynamicRelrSize / DynamicRelrEntrySize;
2564	while (RelrCount--) {
2565	assert(DE.isValidOffset(Offset));
2566	uint64_t Entry = DE.getUnsigned(offset_ptr: &Offset, byte_size: DynamicRelrEntrySize);
2567	if ((Entry & 1) == 0) {
2568	AddRelocation(Entry);
2569	Address = Entry + PSize;
2570	} else {
2571	const uint64_t StartAddress = Address;
2572	while (Entry >>= 1) {
2573	if (Entry & 1)
2574	AddRelocation(Address);
2575
2576	Address += PSize;
2577	}
2578
2579	Address = StartAddress + MaxDelta;
2580	}
2581	}
2582	}
2583
2584	void RewriteInstance::printRelocationInfo(const RelocationRef &Rel,
2585	StringRef SymbolName,
2586	uint64_t SymbolAddress,
2587	uint64_t Addend,
2588	uint64_t ExtractedValue) const {
2589	SmallString<16> TypeName;
2590	Rel.getTypeName(Result&: TypeName);
2591	const uint64_t Address = SymbolAddress + Addend;
2592	const uint64_t Offset = Rel.getOffset();
2593	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2594	BinaryFunction *Func =
2595	BC->getBinaryFunctionContainingAddress(Address: Offset, CheckPastEnd: false, UseMaxSize: BC->isAArch64());
2596	dbgs() << formatv(Fmt: "Relocation: offset = {0:x}; type = {1}; value = {2:x}; ",
2597	Vals: Offset, Vals&: TypeName, Vals&: ExtractedValue)
2598	<< formatv(Fmt: "symbol = {0} ({1}); symbol address = {2:x}; ", Vals&: SymbolName,
2599	Vals: Section ? Section->getName() : "", Vals&: SymbolAddress)
2600	<< formatv(Fmt: "addend = {0:x}; address = {1:x}; in = ", Vals&: Addend, Vals: Address);
2601	if (Func)
2602	dbgs() << Func->getPrintName();
2603	else
2604	dbgs() << BC->getSectionForAddress(Address: Rel.getOffset())->getName();
2605	dbgs() << '\n';
2606	}
2607
2608	void RewriteInstance::readRelocations(const SectionRef &Section) {
2609	LLVM_DEBUG({
2610	StringRef SectionName = cantFail(Section.getName());
2611	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2612	<< ":\n";
2613	});
2614	if (BinarySection(*BC, Section).isAllocatable()) {
2615	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring runtime relocations\n");
2616	return;
2617	}
2618	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2619	assert(SecIter != InputFile->section_end() && "relocated section expected");
2620	SectionRef RelocatedSection = *SecIter;
2621
2622	StringRef RelocatedSectionName = cantFail(ValOrErr: RelocatedSection.getName());
2623	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is "
2624	<< RelocatedSectionName << '\n');
2625
2626	if (!BinarySection(*BC, RelocatedSection).isAllocatable()) {
2627	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocations against "
2628	<< "non-allocatable section\n");
2629	return;
2630	}
2631	const bool SkipRelocs = StringSwitch<bool>(RelocatedSectionName)
2632	.Cases(S0: ".plt", S1: ".rela.plt", S2: ".got.plt",
2633	S3: ".eh_frame", S4: ".gcc_except_table", Value: true)
2634	.Default(Value: false);
2635	if (SkipRelocs) {
2636	LLVM_DEBUG(
2637	dbgs() << "BOLT-DEBUG: ignoring relocations against known section\n");
2638	return;
2639	}
2640
2641	for (const RelocationRef &Rel : Section.relocations())
2642	handleRelocation(RelocatedSection, Rel);
2643	}
2644
2645	void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
2646	const RelocationRef &Rel) {
2647	const bool IsAArch64 = BC->isAArch64();
2648	const bool IsX86 = BC->isX86();
2649	const bool IsFromCode = RelocatedSection.isText();
2650	const bool IsWritable = BinarySection(*BC, RelocatedSection).isWritable();
2651
2652	SmallString<16> TypeName;
2653	Rel.getTypeName(Result&: TypeName);
2654	uint32_t RType = Relocation::getType(Rel);
2655	if (Relocation::skipRelocationType(Type: RType))
2656	return;
2657
2658	// Adjust the relocation type as the linker might have skewed it.
2659	if (IsX86 && (RType & ELF::R_X86_64_converted_reloc_bit)) {
2660	if (opts::Verbosity >= 1)
2661	dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n";
2662	RType &= ~ELF::R_X86_64_converted_reloc_bit;
2663	}
2664
2665	if (Relocation::isTLS(Type: RType)) {
2666	// No special handling required for TLS relocations on X86.
2667	if (IsX86)
2668	return;
2669
2670	// The non-got related TLS relocations on AArch64 and RISC-V also could be
2671	// skipped.
2672	if (!Relocation::isGOT(Type: RType))
2673	return;
2674	}
2675
2676	if (!IsAArch64 && BC->getDynamicRelocationAt(Address: Rel.getOffset())) {
2677	LLVM_DEBUG({
2678	dbgs() << formatv("BOLT-DEBUG: address {0:x} has a ", Rel.getOffset())
2679	<< "dynamic relocation against it. Ignoring static relocation.\n";
2680	});
2681	return;
2682	}
2683
2684	std::string SymbolName;
2685	uint64_t SymbolAddress;
2686	int64_t Addend;
2687	uint64_t ExtractedValue;
2688	bool IsSectionRelocation;
2689	bool Skip;
2690	if (!analyzeRelocation(Rel, RType, SymbolName, IsSectionRelocation,
2691	SymbolAddress, Addend, ExtractedValue, Skip)) {
2692	LLVM_DEBUG({
2693	dbgs() << "BOLT-WARNING: failed to analyze relocation @ offset = "
2694	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2695	});
2696	++NumFailedRelocations;
2697	return;
2698	}
2699
2700	if (Skip) {
2701	LLVM_DEBUG({
2702	dbgs() << "BOLT-DEBUG: skipping relocation @ offset = "
2703	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2704	});
2705	return;
2706	}
2707
2708	if (!IsFromCode && !IsWritable && (IsX86 || IsAArch64) &&
2709	Relocation::isPCRelative(Type: RType)) {
2710	BinaryData *BD = BC->getBinaryDataContainingAddress(Address: Rel.getOffset());
2711	if (BD && (BD->nameStartsWith(Prefix: "_ZTV") || // vtable
2712	BD->nameStartsWith(Prefix: "_ZTCN"))) { // construction vtable
2713	BinaryFunction *BF = BC->getBinaryFunctionContainingAddress(
2714	Address: SymbolAddress, /*CheckPastEnd*/ false, /*UseMaxSize*/ true);
2715	if (BF) {
2716	if (BF->getAddress() != SymbolAddress) {
2717	BC->errs()
2718	<< "BOLT-ERROR: the virtual function table entry at offset 0x"
2719	<< Twine::utohexstr(Val: Rel.getOffset())
2720	<< " points to the middle of a function @ 0x"
2721	<< Twine::utohexstr(Val: BF->getAddress()) << "\n";
2722	exit(status: 1);
2723	}
2724	BC->addRelocation(Address: Rel.getOffset(), Symbol: BF->getSymbol(), Type: RType, Addend,
2725	Value: ExtractedValue);
2726	return;
2727	}
2728	}
2729	}
2730
2731	const uint64_t Address = SymbolAddress + Addend;
2732
2733	LLVM_DEBUG({
2734	dbgs() << "BOLT-DEBUG: ";
2735	printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend, ExtractedValue);
2736	});
2737
2738	BinaryFunction *ContainingBF = nullptr;
2739	if (IsFromCode) {
2740	ContainingBF =
2741	BC->getBinaryFunctionContainingAddress(Address: Rel.getOffset(),
2742	/*CheckPastEnd*/ false,
2743	/*UseMaxSize*/ true);
2744	assert(ContainingBF && "cannot find function for address in code");
2745	if (!IsAArch64 && !ContainingBF->containsAddress(PC: Rel.getOffset())) {
2746	if (opts::Verbosity >= 1)
2747	BC->outs() << formatv(
2748	Fmt: "BOLT-INFO: {0} has relocations in padding area\n", Vals&: *ContainingBF);
2749	ContainingBF->setSize(ContainingBF->getMaxSize());
2750	ContainingBF->setSimple(false);
2751	return;
2752	}
2753	}
2754
2755	MCSymbol *ReferencedSymbol = nullptr;
2756	if (!IsSectionRelocation) {
2757	if (BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName)) {
2758	ReferencedSymbol = BD->getSymbol();
2759	} else if (BC->isGOTSymbol(SymName: SymbolName)) {
2760	if (BinaryData *BD = BC->getGOTSymbol())
2761	ReferencedSymbol = BD->getSymbol();
2762	} else if (BinaryData *BD = BC->getBinaryDataAtAddress(Address: SymbolAddress)) {
2763	ReferencedSymbol = BD->getSymbol();
2764	}
2765	}
2766
2767	ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};
2768	symbol_iterator SymbolIter = Rel.getSymbol();
2769	if (SymbolIter != InputFile->symbol_end()) {
2770	SymbolRef Symbol = *SymbolIter;
2771	section_iterator Section =
2772	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
2773	if (Section != InputFile->section_end()) {
2774	Expected<StringRef> SectionName = Section->getName();
2775	if (SectionName && !SectionName->empty())
2776	ReferencedSection = BC->getUniqueSectionByName(SectionName: *SectionName);
2777	} else if (BC->isRISCV() && ReferencedSymbol && ContainingBF &&
2778	(cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Absolute)) {
2779	// This might be a relocation for an ABS symbols like __global_pointer$ on
2780	// RISC-V
2781	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol,
2782	RelType: Relocation::getType(Rel), Addend: 0,
2783	Value: cantFail(ValOrErr: Symbol.getValue()));
2784	return;
2785	}
2786	}
2787
2788	if (!ReferencedSection)
2789	ReferencedSection = BC->getSectionForAddress(Address: SymbolAddress);
2790
2791	const bool IsToCode = ReferencedSection && ReferencedSection->isText();
2792
2793	// Special handling of PC-relative relocations.
2794	if (IsX86 && Relocation::isPCRelative(Type: RType)) {
2795	if (!IsFromCode && IsToCode) {
2796	// PC-relative relocations from data to code are tricky since the
2797	// original information is typically lost after linking, even with
2798	// '--emit-relocs'. Such relocations are normally used by PIC-style
2799	// jump tables and they reference both the jump table and jump
2800	// targets by computing the difference between the two. If we blindly
2801	// apply the relocation, it will appear that it references an arbitrary
2802	// location in the code, possibly in a different function from the one
2803	// containing the jump table.
2804	//
2805	// For that reason, we only register the fact that there is a
2806	// PC-relative relocation at a given address against the code.
2807	// The actual referenced label/address will be determined during jump
2808	// table analysis.
2809	BC->addPCRelativeDataRelocation(Address: Rel.getOffset());
2810	} else if (ContainingBF && !IsSectionRelocation && ReferencedSymbol) {
2811	// If we know the referenced symbol, register the relocation from
2812	// the code. It's required to properly handle cases where
2813	// "symbol + addend" references an object different from "symbol".
2814	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
2815	Addend, Value: ExtractedValue);
2816	} else {
2817	LLVM_DEBUG({
2818	dbgs() << "BOLT-DEBUG: not creating PC-relative relocation at"
2819	<< formatv("{0:x} for {1}\n", Rel.getOffset(), SymbolName);
2820	});
2821	}
2822
2823	return;
2824	}
2825
2826	bool ForceRelocation = BC->forceSymbolRelocations(SymbolName);
2827	if ((BC->isAArch64() || BC->isRISCV()) && Relocation::isGOT(Type: RType))
2828	ForceRelocation = true;
2829
2830	if (!ReferencedSection && !ForceRelocation) {
2831	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: cannot determine referenced section.\n");
2832	return;
2833	}
2834
2835	// Occasionally we may see a reference past the last byte of the function
2836	// typically as a result of __builtin_unreachable(). Check it here.
2837	BinaryFunction *ReferencedBF = BC->getBinaryFunctionContainingAddress(
2838	Address, /*CheckPastEnd*/ true, /*UseMaxSize*/ IsAArch64);
2839
2840	if (!IsSectionRelocation) {
2841	if (BinaryFunction *BF =
2842	BC->getBinaryFunctionContainingAddress(Address: SymbolAddress)) {
2843	if (BF != ReferencedBF) {
2844	// It's possible we are referencing a function without referencing any
2845	// code, e.g. when taking a bitmask action on a function address.
2846	BC->errs()
2847	<< "BOLT-WARNING: non-standard function reference (e.g. bitmask)"
2848	<< formatv(Fmt: " detected against function {0} from ", Vals&: *BF);
2849	if (IsFromCode)
2850	BC->errs() << formatv(Fmt: "function {0}\n", Vals&: *ContainingBF);
2851	else
2852	BC->errs() << formatv(Fmt: "data section at {0:x}\n", Vals: Rel.getOffset());
2853	LLVM_DEBUG(printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend,
2854	ExtractedValue));
2855	ReferencedBF = BF;
2856	}
2857	}
2858	} else if (ReferencedBF) {
2859	assert(ReferencedSection && "section expected for section relocation");
2860	if (*ReferencedBF->getOriginSection() != *ReferencedSection) {
2861	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring false function reference\n");
2862	ReferencedBF = nullptr;
2863	}
2864	}
2865
2866	// Workaround for a member function pointer de-virtualization bug. We check
2867	// if a non-pc-relative relocation in the code is pointing to (fptr - 1).
2868	if (IsToCode && ContainingBF && !Relocation::isPCRelative(Type: RType) &&
2869	(!ReferencedBF || (ReferencedBF->getAddress() != Address))) {
2870	if (const BinaryFunction *RogueBF =
2871	BC->getBinaryFunctionAtAddress(Address: Address + 1)) {
2872	// Do an extra check that the function was referenced previously.
2873	// It's a linear search, but it should rarely happen.
2874	auto CheckReloc = [&](const Relocation &Rel) {
2875	return Rel.Symbol == RogueBF->getSymbol() &&
2876	!Relocation::isPCRelative(Type: Rel.Type);
2877	};
2878	bool Found = llvm::any_of(
2879	Range: llvm::make_second_range(c&: ContainingBF->Relocations), P: CheckReloc);
2880
2881	if (Found) {
2882	BC->errs()
2883	<< "BOLT-WARNING: detected possible compiler de-virtualization "
2884	"bug: -1 addend used with non-pc-relative relocation against "
2885	<< formatv(Fmt: "function {0} in function {1}\n", Vals: *RogueBF,
2886	Vals&: *ContainingBF);
2887	return;
2888	}
2889	}
2890	}
2891
2892	if (ForceRelocation && !ReferencedBF) {
2893	// Create the relocation symbol if it's not defined in the binary.
2894	if (SymbolAddress == 0)
2895	ReferencedSymbol = BC->registerNameAtAddress(Name: SymbolName, Address: 0, Size: 0, Alignment: 0);
2896
2897	LLVM_DEBUG(
2898	dbgs() << "BOLT-DEBUG: forcing relocation against symbol "
2899	<< (ReferencedSymbol ? ReferencedSymbol->getName() : "<none>")
2900	<< " with addend " << Addend << '\n');
2901	} else if (ReferencedBF) {
2902	ReferencedSymbol = ReferencedBF->getSymbol();
2903	uint64_t RefFunctionOffset = 0;
2904
2905	// Adjust the point of reference to a code location inside a function.
2906	if (ReferencedBF->containsAddress(PC: Address, /*UseMaxSize = */ true)) {
2907	RefFunctionOffset = Address - ReferencedBF->getAddress();
2908	if (Relocation::isInstructionReference(Type: RType)) {
2909	// Instruction labels are created while disassembling so we just leave
2910	// the symbol empty for now. Since the extracted value is typically
2911	// unrelated to the referenced symbol (e.g., %pcrel_lo in RISC-V
2912	// references an instruction but the patched value references the low
2913	// bits of a data address), we set the extracted value to the symbol
2914	// address in order to be able to correctly reconstruct the reference
2915	// later.
2916	ReferencedSymbol = nullptr;
2917	ExtractedValue = Address;
2918	} else if (RefFunctionOffset) {
2919	if (ContainingBF && ContainingBF != ReferencedBF) {
2920	ReferencedSymbol =
2921	ReferencedBF->addEntryPointAtOffset(Offset: RefFunctionOffset);
2922	} else {
2923	ReferencedSymbol =
2924	ReferencedBF->getOrCreateLocalLabel(Address,
2925	/*CreatePastEnd =*/true);
2926
2927	// If ContainingBF != nullptr, it equals ReferencedBF (see
2928	// if-condition above) so we're handling a relocation from a function
2929	// to itself. RISC-V uses such relocations for branches, for example.
2930	// These should not be registered as externally references offsets.
2931	if (!ContainingBF)
2932	ReferencedBF->registerReferencedOffset(Offset: RefFunctionOffset);
2933	}
2934	if (opts::Verbosity > 1 &&
2935	BinarySection(*BC, RelocatedSection).isWritable())
2936	BC->errs()
2937	<< "BOLT-WARNING: writable reference into the middle of the "
2938	<< formatv(Fmt: "function {0} detected at address {1:x}\n",
2939	Vals&: *ReferencedBF, Vals: Rel.getOffset());
2940	}
2941	SymbolAddress = Address;
2942	Addend = 0;
2943	}
2944	LLVM_DEBUG({
2945	dbgs() << " referenced function " << *ReferencedBF;
2946	if (Address != ReferencedBF->getAddress())
2947	dbgs() << formatv(" at offset {0:x}", RefFunctionOffset);
2948	dbgs() << '\n';
2949	});
2950	} else {
2951	if (IsToCode && SymbolAddress) {
2952	// This can happen e.g. with PIC-style jump tables.
2953	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: no corresponding function for "
2954	"relocation against code\n");
2955	}
2956
2957	// In AArch64 there are zero reasons to keep a reference to the
2958	// "original" symbol plus addend. The original symbol is probably just a
2959	// section symbol. If we are here, this means we are probably accessing
2960	// data, so it is imperative to keep the original address.
2961	if (IsAArch64) {
2962	SymbolName = formatv(Fmt: "SYMBOLat{0:x}", Vals: Address);
2963	SymbolAddress = Address;
2964	Addend = 0;
2965	}
2966
2967	if (BinaryData *BD = BC->getBinaryDataContainingAddress(Address: SymbolAddress)) {
2968	// Note: this assertion is trying to check sanity of BinaryData objects
2969	// but AArch64 and RISCV has inferred and incomplete object locations
2970	// coming from GOT/TLS or any other non-trivial relocation (that requires
2971	// creation of sections and whose symbol address is not really what should
2972	// be encoded in the instruction). So we essentially disabled this check
2973	// for AArch64 and live with bogus names for objects.
2974	assert((IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
2975	BD->nameStartsWith(SymbolName) ||
2976	BD->nameStartsWith("PG" + SymbolName) ||
2977	(BD->nameStartsWith("ANONYMOUS") &&
2978	(BD->getSectionName().starts_with(".plt") ||
2979	BD->getSectionName().ends_with(".plt")))) &&
2980	"BOLT symbol names of all non-section relocations must match up "
2981	"with symbol names referenced in the relocation");
2982
2983	if (IsSectionRelocation)
2984	BC->markAmbiguousRelocations(BD&: *BD, Address);
2985
2986	ReferencedSymbol = BD->getSymbol();
2987	Addend += (SymbolAddress - BD->getAddress());
2988	SymbolAddress = BD->getAddress();
2989	assert(Address == SymbolAddress + Addend);
2990	} else {
2991	// These are mostly local data symbols but undefined symbols
2992	// in relocation sections can get through here too, from .plt.
2993	assert(
2994	(IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
2995	BC->getSectionNameForAddress(SymbolAddress)->starts_with(".plt")) &&
2996	"known symbols should not resolve to anonymous locals");
2997
2998	if (IsSectionRelocation) {
2999	ReferencedSymbol =
3000	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "SYMBOLat");
3001	} else {
3002	SymbolRef Symbol = *Rel.getSymbol();
3003	const uint64_t SymbolSize =
3004	IsAArch64 ? 0 : ELFSymbolRef(Symbol).getSize();
3005	const uint64_t SymbolAlignment = IsAArch64 ? 1 : Symbol.getAlignment();
3006	const uint32_t SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
3007	std::string Name;
3008	if (SymbolFlags & SymbolRef::SF_Global) {
3009	Name = SymbolName;
3010	} else {
3011	if (StringRef(SymbolName)
3012	.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix()))
3013	Name = NR.uniquify(Name: "PG" + SymbolName);
3014	else
3015	Name = NR.uniquify(Name: SymbolName);
3016	}
3017	ReferencedSymbol = BC->registerNameAtAddress(
3018	Name, Address: SymbolAddress, Size: SymbolSize, Alignment: SymbolAlignment, Flags: SymbolFlags);
3019	}
3020
3021	if (IsSectionRelocation) {
3022	BinaryData *BD = BC->getBinaryDataByName(Name: ReferencedSymbol->getName());
3023	BC->markAmbiguousRelocations(BD&: *BD, Address);
3024	}
3025	}
3026	}
3027
3028	auto checkMaxDataRelocations = [&]() {
3029	++NumDataRelocations;
3030	LLVM_DEBUG(if (opts::MaxDataRelocations &&
3031	NumDataRelocations + 1 == opts::MaxDataRelocations) {
3032	dbgs() << "BOLT-DEBUG: processing ending on data relocation "
3033	<< NumDataRelocations << ": ";
3034	printRelocationInfo(Rel, ReferencedSymbol->getName(), SymbolAddress,
3035	Addend, ExtractedValue);
3036	});
3037
3038	return (!opts::MaxDataRelocations ||
3039	NumDataRelocations < opts::MaxDataRelocations);
3040	};
3041
3042	if ((ReferencedSection && refersToReorderedSection(Section: ReferencedSection)) ||
3043	(opts::ForceToDataRelocations && checkMaxDataRelocations()) ||
3044	// RISC-V has ADD/SUB data-to-data relocations
3045	BC->isRISCV())
3046	ForceRelocation = true;
3047
3048	if (IsFromCode)
3049	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
3050	Addend, Value: ExtractedValue);
3051	else if (IsToCode || ForceRelocation)
3052	BC->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, Type: RType, Addend,
3053	Value: ExtractedValue);
3054	else
3055	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");
3056	}
3057
3058	static BinaryFunction *getInitFunctionIfStaticBinary(BinaryContext &BC) {
3059	// Workaround for https://github.com/llvm/llvm-project/issues/100096
3060	// ("[BOLT] GOT array pointer incorrectly rewritten"). In aarch64
3061	// static glibc binaries, the .init section's _init function pointer can
3062	// alias with a data pointer for the end of an array. GOT rewriting
3063	// currently can't detect this and updates the data pointer to the
3064	// moved _init, causing a runtime crash. Skipping _init on the other
3065	// hand should be harmless.
3066	if (!BC.IsStaticExecutable)
3067	return nullptr;
3068	const BinaryData *BD = BC.getBinaryDataByName(Name: "_init");
3069	if (!BD || BD->getSectionName() != ".init")
3070	return nullptr;
3071	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: skip _init in for GOT workaround.\n");
3072	return BC.getBinaryFunctionAtAddress(Address: BD->getAddress());
3073	}
3074
3075	void RewriteInstance::selectFunctionsToProcess() {
3076	// Extend the list of functions to process or skip from a file.
3077	auto populateFunctionNames = [](cl::opt<std::string> &FunctionNamesFile,
3078	cl::list<std::string> &FunctionNames) {
3079	if (FunctionNamesFile.empty())
3080	return;
3081	std::ifstream FuncsFile(FunctionNamesFile, std::ios::in);
3082	std::string FuncName;
3083	while (std::getline(is&: FuncsFile, str&: FuncName))
3084	FunctionNames.push_back(value: FuncName);
3085	};
3086	populateFunctionNames(opts::FunctionNamesFile, opts::ForceFunctionNames);
3087	populateFunctionNames(opts::SkipFunctionNamesFile, opts::SkipFunctionNames);
3088	populateFunctionNames(opts::FunctionNamesFileNR, opts::ForceFunctionNamesNR);
3089
3090	// Make a set of functions to process to speed up lookups.
3091	std::unordered_set<std::string> ForceFunctionsNR(
3092	opts::ForceFunctionNamesNR.begin(), opts::ForceFunctionNamesNR.end());
3093
3094	if ((!opts::ForceFunctionNames.empty() ||
3095	!opts::ForceFunctionNamesNR.empty()) &&
3096	!opts::SkipFunctionNames.empty()) {
3097	BC->errs()
3098	<< "BOLT-ERROR: cannot select functions to process and skip at the "
3099	"same time. Please use only one type of selection.\n";
3100	exit(status: 1);
3101	}
3102
3103	uint64_t LiteThresholdExecCount = 0;
3104	if (opts::LiteThresholdPct) {
3105	if (opts::LiteThresholdPct > 100)
3106	opts::LiteThresholdPct = 100;
3107
3108	std::vector<const BinaryFunction *> TopFunctions;
3109	for (auto &BFI : BC->getBinaryFunctions()) {
3110	const BinaryFunction &Function = BFI.second;
3111	if (ProfileReader->mayHaveProfileData(BF: Function))
3112	TopFunctions.push_back(x: &Function);
3113	}
3114	llvm::sort(
3115	C&: TopFunctions, Comp: [](const BinaryFunction *A, const BinaryFunction *B) {
3116	return A->getKnownExecutionCount() < B->getKnownExecutionCount();
3117	});
3118
3119	size_t Index = TopFunctions.size() * opts::LiteThresholdPct / 100;
3120	if (Index)
3121	--Index;
3122	LiteThresholdExecCount = TopFunctions[Index]->getKnownExecutionCount();
3123	BC->outs() << "BOLT-INFO: limiting processing to functions with at least "
3124	<< LiteThresholdExecCount << " invocations\n";
3125	}
3126	LiteThresholdExecCount = std::max(
3127	a: LiteThresholdExecCount, b: static_cast<uint64_t>(opts::LiteThresholdCount));
3128
3129	StringSet<> ReorderFunctionsUserSet;
3130	StringSet<> ReorderFunctionsLTOCommonSet;
3131	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
3132	std::vector<std::string> FunctionNames;
3133	BC->logBOLTErrorsAndQuitOnFatal(
3134	E: ReorderFunctions::readFunctionOrderFile(FunctionNames));
3135	for (const std::string &Function : FunctionNames) {
3136	ReorderFunctionsUserSet.insert(key: Function);
3137	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name: Function))
3138	ReorderFunctionsLTOCommonSet.insert(key: *LTOCommonName);
3139	}
3140	}
3141
3142	uint64_t NumFunctionsToProcess = 0;
3143	auto mustSkip = [&](const BinaryFunction &Function) {
3144	if (opts::MaxFunctions.getNumOccurrences() &&
3145	NumFunctionsToProcess >= opts::MaxFunctions)
3146	return true;
3147	for (std::string &Name : opts::SkipFunctionNames)
3148	if (Function.hasNameRegex(NameRegex: Name))
3149	return true;
3150
3151	return false;
3152	};
3153
3154	auto shouldProcess = [&](const BinaryFunction &Function) {
3155	if (mustSkip(Function))
3156	return false;
3157
3158	// If the list is not empty, only process functions from the list.
3159	if (!opts::ForceFunctionNames.empty() || !ForceFunctionsNR.empty()) {
3160	// Regex check (-funcs and -funcs-file options).
3161	for (std::string &Name : opts::ForceFunctionNames)
3162	if (Function.hasNameRegex(NameRegex: Name))
3163	return true;
3164
3165	// Non-regex check (-funcs-no-regex and -funcs-file-no-regex).
3166	for (const StringRef Name : Function.getNames())
3167	if (ForceFunctionsNR.count(x: Name.str()))
3168	return true;
3169
3170	return false;
3171	}
3172
3173	if (opts::Lite) {
3174	// Forcibly include functions specified in the -function-order file.
3175	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
3176	for (const StringRef Name : Function.getNames())
3177	if (ReorderFunctionsUserSet.contains(key: Name))
3178	return true;
3179	for (const StringRef Name : Function.getNames())
3180	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name))
3181	if (ReorderFunctionsLTOCommonSet.contains(key: *LTOCommonName))
3182	return true;
3183	}
3184
3185	if (ProfileReader && !ProfileReader->mayHaveProfileData(BF: Function))
3186	return false;
3187
3188	if (Function.getKnownExecutionCount() < LiteThresholdExecCount)
3189	return false;
3190	}
3191
3192	return true;
3193	};
3194
3195	if (BinaryFunction *Init = getInitFunctionIfStaticBinary(BC&: *BC))
3196	Init->setIgnored();
3197
3198	for (auto &BFI : BC->getBinaryFunctions()) {
3199	BinaryFunction &Function = BFI.second;
3200
3201	// Pseudo functions are explicitly marked by us not to be processed.
3202	if (Function.isPseudo()) {
3203	Function.IsIgnored = true;
3204	Function.HasExternalRefRelocations = true;
3205	continue;
3206	}
3207
3208	// Decide what to do with fragments after parent functions are processed.
3209	if (Function.isFragment())
3210	continue;
3211
3212	if (!shouldProcess(Function)) {
3213	if (opts::Verbosity >= 1) {
3214	BC->outs() << "BOLT-INFO: skipping processing " << Function
3215	<< " per user request\n";
3216	}
3217	Function.setIgnored();
3218	} else {
3219	++NumFunctionsToProcess;
3220	if (opts::MaxFunctions.getNumOccurrences() &&
3221	NumFunctionsToProcess == opts::MaxFunctions)
3222	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3223	}
3224	}
3225
3226	if (!BC->HasSplitFunctions)
3227	return;
3228
3229	// Fragment overrides:
3230	// - If the fragment must be skipped, then the parent must be skipped as well.
3231	// Otherwise, fragment should follow the parent function:
3232	// - if the parent is skipped, skip fragment,
3233	// - if the parent is processed, process the fragment(s) as well.
3234	for (auto &BFI : BC->getBinaryFunctions()) {
3235	BinaryFunction &Function = BFI.second;
3236	if (!Function.isFragment())
3237	continue;
3238	if (mustSkip(Function)) {
3239	for (BinaryFunction *Parent : Function.ParentFragments) {
3240	if (opts::Verbosity >= 1) {
3241	BC->outs() << "BOLT-INFO: skipping processing " << *Parent
3242	<< " together with fragment function\n";
3243	}
3244	Parent->setIgnored();
3245	--NumFunctionsToProcess;
3246	}
3247	Function.setIgnored();
3248	continue;
3249	}
3250
3251	bool IgnoredParent =
3252	llvm::any_of(Range&: Function.ParentFragments, P: [&](BinaryFunction *Parent) {
3253	return Parent->isIgnored();
3254	});
3255	if (IgnoredParent) {
3256	if (opts::Verbosity >= 1) {
3257	BC->outs() << "BOLT-INFO: skipping processing " << Function
3258	<< " together with parent function\n";
3259	}
3260	Function.setIgnored();
3261	} else {
3262	++NumFunctionsToProcess;
3263	if (opts::Verbosity >= 1) {
3264	BC->outs() << "BOLT-INFO: processing " << Function
3265	<< " as a sibling of non-ignored function\n";
3266	}
3267	if (opts::MaxFunctions && NumFunctionsToProcess == opts::MaxFunctions)
3268	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3269	}
3270	}
3271	}
3272
3273	void RewriteInstance::readDebugInfo() {
3274	NamedRegionTimer T("readDebugInfo", "read debug info", TimerGroupName,
3275	TimerGroupDesc, opts::TimeRewrite);
3276	if (!opts::UpdateDebugSections)
3277	return;
3278
3279	BC->preprocessDebugInfo();
3280	}
3281
3282	void RewriteInstance::preprocessProfileData() {
3283	if (!ProfileReader)
3284	return;
3285
3286	NamedRegionTimer T("preprocessprofile", "pre-process profile data",
3287	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3288
3289	BC->outs() << "BOLT-INFO: pre-processing profile using "
3290	<< ProfileReader->getReaderName() << '\n';
3291
3292	if (BAT->enabledFor(InputFile)) {
3293	BC->outs() << "BOLT-INFO: profile collection done on a binary already "
3294	"processed by BOLT\n";
3295	ProfileReader->setBAT(&*BAT);
3296	}
3297
3298	if (Error E = ProfileReader->preprocessProfile(BC&: *BC))
3299	report_error(Message: "cannot pre-process profile", E: std::move(E));
3300
3301	if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName() &&
3302	!opts::AllowStripped) {
3303	BC->errs()
3304	<< "BOLT-ERROR: input binary does not have local file symbols "
3305	"but profile data includes function names with embedded file "
3306	"names. It appears that the input binary was stripped while a "
3307	"profiled binary was not. If you know what you are doing and "
3308	"wish to proceed, use -allow-stripped option.\n";
3309	exit(status: 1);
3310	}
3311	}
3312
3313	void RewriteInstance::initializeMetadataManager() {
3314	if (BC->IsLinuxKernel)
3315	MetadataManager.registerRewriter(Rewriter: createLinuxKernelRewriter(*BC));
3316
3317	MetadataManager.registerRewriter(Rewriter: createBuildIDRewriter(*BC));
3318
3319	MetadataManager.registerRewriter(Rewriter: createPseudoProbeRewriter(*BC));
3320
3321	MetadataManager.registerRewriter(Rewriter: createSDTRewriter(*BC));
3322	}
3323
3324	void RewriteInstance::processSectionMetadata() {
3325	NamedRegionTimer T("processmetadata-section", "process section metadata",
3326	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3327	initializeMetadataManager();
3328
3329	MetadataManager.runSectionInitializers();
3330	}
3331
3332	void RewriteInstance::processMetadataPreCFG() {
3333	NamedRegionTimer T("processmetadata-precfg", "process metadata pre-CFG",
3334	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3335	MetadataManager.runInitializersPreCFG();
3336
3337	processProfileDataPreCFG();
3338	}
3339
3340	void RewriteInstance::processMetadataPostCFG() {
3341	NamedRegionTimer T("processmetadata-postcfg", "process metadata post-CFG",
3342	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3343	MetadataManager.runInitializersPostCFG();
3344	}
3345
3346	void RewriteInstance::processProfileDataPreCFG() {
3347	if (!ProfileReader)
3348	return;
3349
3350	NamedRegionTimer T("processprofile-precfg", "process profile data pre-CFG",
3351	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3352
3353	if (Error E = ProfileReader->readProfilePreCFG(BC&: *BC))
3354	report_error(Message: "cannot read profile pre-CFG", E: std::move(E));
3355	}
3356
3357	void RewriteInstance::processProfileData() {
3358	if (!ProfileReader)
3359	return;
3360
3361	NamedRegionTimer T("processprofile", "process profile data", TimerGroupName,
3362	TimerGroupDesc, opts::TimeRewrite);
3363
3364	if (Error E = ProfileReader->readProfile(BC&: *BC))
3365	report_error(Message: "cannot read profile", E: std::move(E));
3366
3367	if (opts::PrintProfile || opts::PrintAll) {
3368	for (auto &BFI : BC->getBinaryFunctions()) {
3369	BinaryFunction &Function = BFI.second;
3370	if (Function.empty())
3371	continue;
3372
3373	Function.print(OS&: BC->outs(), Annotation: "after attaching profile");
3374	}
3375	}
3376
3377	if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) {
3378	YAMLProfileWriter PW(opts::SaveProfile);
3379	PW.writeProfile(RI: *this);
3380	}
3381	if (opts::AggregateOnly &&
3382	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML &&
3383	!BAT->enabledFor(InputFile)) {
3384	YAMLProfileWriter PW(opts::OutputFilename);
3385	PW.writeProfile(RI: *this);
3386	}
3387
3388	// Release memory used by profile reader.
3389	ProfileReader.reset();
3390
3391	if (opts::AggregateOnly) {
3392	PrintProgramStats PPS(&*BAT);
3393	BC->logBOLTErrorsAndQuitOnFatal(E: PPS.runOnFunctions(BC&: *BC));
3394	TimerGroup::printAll(OS&: outs());
3395	exit(status: 0);
3396	}
3397	}
3398
3399	void RewriteInstance::disassembleFunctions() {
3400	NamedRegionTimer T("disassembleFunctions", "disassemble functions",
3401	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3402	for (auto &BFI : BC->getBinaryFunctions()) {
3403	BinaryFunction &Function = BFI.second;
3404
3405	ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
3406	if (!FunctionData) {
3407	BC->errs() << "BOLT-ERROR: corresponding section is non-executable or "
3408	<< "empty for function " << Function << '\n';
3409	exit(status: 1);
3410	}
3411
3412	// Treat zero-sized functions as non-simple ones.
3413	if (Function.getSize() == 0) {
3414	Function.setSimple(false);
3415	continue;
3416	}
3417
3418	// Offset of the function in the file.
3419	const auto *FileBegin =
3420	reinterpret_cast<const uint8_t *>(InputFile->getData().data());
3421	Function.setFileOffset(FunctionData->begin() - FileBegin);
3422
3423	if (!shouldDisassemble(BF: Function)) {
3424	NamedRegionTimer T("scan", "scan functions", "buildfuncs",
3425	"Scan Binary Functions", opts::TimeBuild);
3426	Function.scanExternalRefs();
3427	Function.setSimple(false);
3428	continue;
3429	}
3430
3431	bool DisasmFailed{false};
3432	handleAllErrors(E: Function.disassemble(), Handlers: [&](const BOLTError &E) {
3433	DisasmFailed = true;
3434	if (E.isFatal()) {
3435	E.log(OS&: BC->errs());
3436	exit(status: 1);
3437	}
3438	if (opts::processAllFunctions()) {
3439	BC->errs() << BC->generateBugReportMessage(
3440	Message: "function cannot be properly disassembled. "
3441	"Unable to continue in relocation mode.",
3442	Function);
3443	exit(status: 1);
3444	}
3445	if (opts::Verbosity >= 1)
3446	BC->outs() << "BOLT-INFO: could not disassemble function " << Function
3447	<< ". Will ignore.\n";
3448	// Forcefully ignore the function.
3449	Function.scanExternalRefs();
3450	Function.setIgnored();
3451	});
3452
3453	if (DisasmFailed)
3454	continue;
3455
3456	if (opts::PrintAll || opts::PrintDisasm)
3457	Function.print(OS&: BC->outs(), Annotation: "after disassembly");
3458	}
3459
3460	BC->processInterproceduralReferences();
3461	BC->populateJumpTables();
3462
3463	for (auto &BFI : BC->getBinaryFunctions()) {
3464	BinaryFunction &Function = BFI.second;
3465
3466	if (!shouldDisassemble(BF: Function))
3467	continue;
3468
3469	Function.postProcessEntryPoints();
3470	Function.postProcessJumpTables();
3471	}
3472
3473	BC->clearJumpTableTempData();
3474	BC->adjustCodePadding();
3475
3476	for (auto &BFI : BC->getBinaryFunctions()) {
3477	BinaryFunction &Function = BFI.second;
3478
3479	if (!shouldDisassemble(BF: Function))
3480	continue;
3481
3482	if (!Function.isSimple()) {
3483	assert((!BC->HasRelocations || Function.getSize() == 0 ||
3484	Function.hasIndirectTargetToSplitFragment()) &&
3485	"unexpected non-simple function in relocation mode");
3486	continue;
3487	}
3488
3489	// Fill in CFI information for this function
3490	if (!Function.trapsOnEntry() && !CFIRdWrt->fillCFIInfoFor(Function)) {
3491	if (BC->HasRelocations) {
3492	BC->errs() << BC->generateBugReportMessage(Message: "unable to fill CFI.",
3493	Function);
3494	exit(status: 1);
3495	} else {
3496	BC->errs() << "BOLT-WARNING: unable to fill CFI for function "
3497	<< Function << ". Skipping.\n";
3498	Function.setSimple(false);
3499	continue;
3500	}
3501	}
3502
3503	// Parse LSDA.
3504	if (Function.getLSDAAddress() != 0 &&
3505	!BC->getFragmentsToSkip().count(x: &Function)) {
3506	ErrorOr<BinarySection &> LSDASection =
3507	BC->getSectionForAddress(Address: Function.getLSDAAddress());
3508	check_error(EC: LSDASection.getError(), Message: "failed to get LSDA section");
3509	ArrayRef<uint8_t> LSDAData = ArrayRef<uint8_t>(
3510	LSDASection->getData(), LSDASection->getContents().size());
3511	BC->logBOLTErrorsAndQuitOnFatal(
3512	E: Function.parseLSDA(LSDAData, LSDAAddress: LSDASection->getAddress()));
3513	}
3514	}
3515	}
3516
3517	void RewriteInstance::buildFunctionsCFG() {
3518	NamedRegionTimer T("buildCFG", "buildCFG", "buildfuncs",
3519	"Build Binary Functions", opts::TimeBuild);
3520
3521	// Create annotation indices to allow lock-free execution
3522	BC->MIB->getOrCreateAnnotationIndex(Name: "JTIndexReg");
3523	BC->MIB->getOrCreateAnnotationIndex(Name: "NOP");
3524
3525	ParallelUtilities::WorkFuncWithAllocTy WorkFun =
3526	[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {
3527	bool HadErrors{false};
3528	handleAllErrors(E: BF.buildCFG(AllocId), Handlers: [&](const BOLTError &E) {
3529	if (!E.getMessage().empty())
3530	E.log(OS&: BC->errs());
3531	if (E.isFatal())
3532	exit(status: 1);
3533	HadErrors = true;
3534	});
3535
3536	if (HadErrors)
3537	return;
3538
3539	if (opts::PrintAll) {
3540	auto L = BC->scopeLock();
3541	BF.print(OS&: BC->outs(), Annotation: "while building cfg");
3542	}
3543	};
3544
3545	ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
3546	return !shouldDisassemble(BF) || !BF.isSimple();
3547	};
3548
3549	ParallelUtilities::runOnEachFunctionWithUniqueAllocId(
3550	BC&: *BC, SchedPolicy: ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFunction: WorkFun,
3551	SkipPredicate, LogName: "disassembleFunctions-buildCFG",
3552	/*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll);
3553
3554	BC->postProcessSymbolTable();
3555	}
3556
3557	void RewriteInstance::postProcessFunctions() {
3558	// We mark fragments as non-simple here, not during disassembly,
3559	// So we can build their CFGs.
3560	BC->skipMarkedFragments();
3561	BC->clearFragmentsToSkip();
3562
3563	BC->TotalScore = 0;
3564	BC->SumExecutionCount = 0;
3565	for (auto &BFI : BC->getBinaryFunctions()) {
3566	BinaryFunction &Function = BFI.second;
3567
3568	// Set function as non-simple if it has dynamic relocations
3569	// in constant island, we don't want this function to be optimized
3570	// e.g. function splitting is unsupported.
3571	if (Function.hasDynamicRelocationAtIsland())
3572	Function.setSimple(false);
3573
3574	if (Function.empty())
3575	continue;
3576
3577	Function.postProcessCFG();
3578
3579	if (opts::PrintAll || opts::PrintCFG)
3580	Function.print(OS&: BC->outs(), Annotation: "after building cfg");
3581
3582	if (opts::DumpDotAll)
3583	Function.dumpGraphForPass(Annotation: "00_build-cfg");
3584
3585	if (opts::PrintLoopInfo) {
3586	Function.calculateLoopInfo();
3587	Function.printLoopInfo(OS&: BC->outs());
3588	}
3589
3590	BC->TotalScore += Function.getFunctionScore();
3591	BC->SumExecutionCount += Function.getKnownExecutionCount();
3592	}
3593
3594	if (opts::PrintGlobals) {
3595	BC->outs() << "BOLT-INFO: Global symbols:\n";
3596	BC->printGlobalSymbols(OS&: BC->outs());
3597	}
3598	}
3599
3600	void RewriteInstance::runOptimizationPasses() {
3601	NamedRegionTimer T("runOptimizationPasses", "run optimization passes",
3602	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3603	BC->logBOLTErrorsAndQuitOnFatal(E: BinaryFunctionPassManager::runAllPasses(BC&: *BC));
3604	}
3605
3606	void RewriteInstance::runBinaryAnalyses() {
3607	NamedRegionTimer T("runBinaryAnalyses", "run binary analysis passes",
3608	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3609	BinaryFunctionPassManager Manager(*BC);
3610	// FIXME: add a pass that warns about which functions do not have CFG,
3611	// and therefore, analysis is most likely to be less accurate.
3612	using GSK = opts::GadgetScannerKind;
3613	using PAuthScanner = PAuthGadgetScanner::Analysis;
3614
3615	// If no command line option was given, act as if "all" was specified.
3616	bool RunAll = !opts::GadgetScannersToRun.getBits() ||
3617	opts::GadgetScannersToRun.isSet(V: GSK::GS_ALL);
3618
3619	if (RunAll || opts::GadgetScannersToRun.isSet(V: GSK::GS_PAUTH)) {
3620	Manager.registerPass(
3621	Pass: std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/args: false));
3622	} else if (RunAll || opts::GadgetScannersToRun.isSet(V: GSK::GS_PACRET)) {
3623	Manager.registerPass(
3624	Pass: std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/args: true));
3625	}
3626
3627	BC->logBOLTErrorsAndQuitOnFatal(E: Manager.runPasses());
3628	}
3629
3630	void RewriteInstance::preregisterSections() {
3631	// Preregister sections before emission to set their order in the output.
3632	const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true,
3633	/*IsText*/ false,
3634	/*IsAllocatable*/ true);
3635	if (BinarySection *EHFrameSection = getSection(Name: getEHFrameSectionName())) {
3636	// New .eh_frame.
3637	BC->registerOrUpdateSection(Name: getNewSecPrefix() + getEHFrameSectionName(),
3638	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3639	// Fully register a relocatable copy of the original .eh_frame.
3640	BC->registerSection(SectionName: ".relocated.eh_frame", OriginalSection: *EHFrameSection);
3641	}
3642	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".gcc_except_table",
3643	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3644	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata", ELFType: ELF::SHT_PROGBITS,
3645	ELFFlags: ROFlags);
3646	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata.cold",
3647	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3648	}
3649
3650	void RewriteInstance::emitAndLink() {
3651	NamedRegionTimer T("emitAndLink", "emit and link", TimerGroupName,
3652	TimerGroupDesc, opts::TimeRewrite);
3653
3654	SmallString<0> ObjectBuffer;
3655	raw_svector_ostream OS(ObjectBuffer);
3656
3657	// Implicitly MCObjectStreamer takes ownership of MCAsmBackend (MAB)
3658	// and MCCodeEmitter (MCE). ~MCObjectStreamer() will delete these
3659	// two instances.
3660	std::unique_ptr<MCStreamer> Streamer = BC->createStreamer(OS);
3661
3662	if (EHFrameSection) {
3663	if (opts::UseOldText || opts::StrictMode) {
3664	// The section is going to be regenerated from scratch.
3665	// Empty the contents, but keep the section reference.
3666	EHFrameSection->clearContents();
3667	} else {
3668	// Make .eh_frame relocatable.
3669	relocateEHFrameSection();
3670	}
3671	}
3672
3673	emitBinaryContext(Streamer&: *Streamer, BC&: *BC, OrgSecPrefix: getOrgSecPrefix());
3674
3675	Streamer->finish();
3676	if (Streamer->getContext().hadError()) {
3677	BC->errs() << "BOLT-ERROR: Emission failed.\n";
3678	exit(status: 1);
3679	}
3680
3681	if (opts::KeepTmp) {
3682	SmallString<128> OutObjectPath;
3683	sys::fs::getPotentiallyUniqueTempFileName(Prefix: "output", Suffix: "o", ResultPath&: OutObjectPath);
3684	std::error_code EC;
3685	raw_fd_ostream FOS(OutObjectPath, EC);
3686	check_error(EC, Message: "cannot create output object file");
3687	FOS << ObjectBuffer;
3688	BC->outs()
3689	<< "BOLT-INFO: intermediary output object file saved for debugging "
3690	"purposes: "
3691	<< OutObjectPath << "\n";
3692	}
3693
3694	ErrorOr<BinarySection &> TextSection =
3695	BC->getUniqueSectionByName(SectionName: BC->getMainCodeSectionName());
3696	if (BC->HasRelocations && TextSection)
3697	BC->renameSection(Section&: *TextSection,
3698	NewName: getOrgSecPrefix() + BC->getMainCodeSectionName());
3699
3700	//////////////////////////////////////////////////////////////////////////////
3701	// Assign addresses to new sections.
3702	//////////////////////////////////////////////////////////////////////////////
3703
3704	// Get output object as ObjectFile.
3705	std::unique_ptr<MemoryBuffer> ObjectMemBuffer =
3706	MemoryBuffer::getMemBuffer(InputData: ObjectBuffer, BufferName: "in-memory object file", RequiresNullTerminator: false);
3707
3708	auto EFMM = std::make_unique<ExecutableFileMemoryManager>(args&: *BC);
3709	EFMM->setNewSecPrefix(getNewSecPrefix());
3710	EFMM->setOrgSecPrefix(getOrgSecPrefix());
3711
3712	Linker = std::make_unique<JITLinkLinker>(args&: *BC, args: std::move(EFMM));
3713	Linker->loadObject(Obj: ObjectMemBuffer->getMemBufferRef(),
3714	MapSections: [this](auto MapSection) { mapFileSections(MapSection); });
3715
3716	// Update output addresses based on the new section map and
3717	// layout. Only do this for the object created by ourselves.
3718	updateOutputValues(Linker: *Linker);
3719
3720	if (opts::UpdateDebugSections) {
3721	DebugInfoRewriter->updateLineTableOffsets(
3722	Asm: static_cast<MCObjectStreamer &>(*Streamer).getAssembler());
3723	}
3724
3725	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {
3726	StartLinkingRuntimeLib = true;
3727	RtLibrary->link(BC&: *BC, ToolPath, Linker&: *Linker, MapSections: [this](auto MapSection) {
3728	// Map newly registered sections.
3729	this->mapAllocatableSections(MapSection);
3730	});
3731	}
3732
3733	// Once the code is emitted, we can rename function sections to actual
3734	// output sections and de-register sections used for emission.
3735	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
3736	ErrorOr<BinarySection &> Section = Function->getCodeSection();
3737	if (Section &&
3738	(Function->getImageAddress() == 0 || Function->getImageSize() == 0))
3739	continue;
3740
3741	// Restore origin section for functions that were emitted or supposed to
3742	// be emitted to patch sections.
3743	if (Section)
3744	BC->deregisterSection(Section&: *Section);
3745	assert(Function->getOriginSectionName() && "expected origin section");
3746	Function->CodeSectionName = Function->getOriginSectionName()->str();
3747	for (const FunctionFragment &FF :
3748	Function->getLayout().getSplitFragments()) {
3749	if (ErrorOr<BinarySection &> ColdSection =
3750	Function->getCodeSection(Fragment: FF.getFragmentNum()))
3751	BC->deregisterSection(Section&: *ColdSection);
3752	}
3753	if (Function->getLayout().isSplit())
3754	Function->setColdCodeSectionName(getBOLTTextSectionName());
3755	}
3756
3757	if (opts::PrintCacheMetrics) {
3758	BC->outs() << "BOLT-INFO: cache metrics after emitting functions:\n";
3759	CacheMetrics::printAll(OS&: BC->outs(), BinaryFunctions: BC->getSortedFunctions());
3760	}
3761	}
3762
3763	void RewriteInstance::finalizeMetadataPreEmit() {
3764	NamedRegionTimer T("finalizemetadata-preemit", "finalize metadata pre-emit",
3765	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3766	MetadataManager.runFinalizersPreEmit();
3767	}
3768
3769	void RewriteInstance::updateMetadata() {
3770	NamedRegionTimer T("updatemetadata-postemit", "update metadata post-emit",
3771	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3772	MetadataManager.runFinalizersAfterEmit();
3773
3774	if (opts::UpdateDebugSections) {
3775	NamedRegionTimer T("updateDebugInfo", "update debug info", TimerGroupName,
3776	TimerGroupDesc, opts::TimeRewrite);
3777	DebugInfoRewriter->updateDebugInfo();
3778	}
3779
3780	if (opts::WriteBoltInfoSection)
3781	addBoltInfoSection();
3782	}
3783
3784	void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
3785	BC->deregisterUnusedSections();
3786
3787	// If no new .eh_frame was written, remove relocated original .eh_frame.
3788	BinarySection *RelocatedEHFrameSection =
3789	getSection(Name: ".relocated" + getEHFrameSectionName());
3790	if (RelocatedEHFrameSection && RelocatedEHFrameSection->hasValidSectionID()) {
3791	BinarySection *NewEHFrameSection =
3792	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
3793	if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) {
3794	// JITLink will still have to process relocations for the section, hence
3795	// we need to assign it the address that wouldn't result in relocation
3796	// processing failure.
3797	MapSection(*RelocatedEHFrameSection, NextAvailableAddress);
3798	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
3799	}
3800	}
3801
3802	mapCodeSections(MapSection);
3803
3804	// Map the rest of the sections.
3805	mapAllocatableSections(MapSection);
3806
3807	if (!BC->BOLTReserved.empty()) {
3808	const uint64_t AllocatedSize =
3809	NextAvailableAddress - BC->BOLTReserved.start();
3810	if (BC->BOLTReserved.size() < AllocatedSize) {
3811	BC->errs() << "BOLT-ERROR: reserved space (" << BC->BOLTReserved.size()
3812	<< " byte" << (BC->BOLTReserved.size() == 1 ? "" : "s")
3813	<< ") is smaller than required for new allocations ("
3814	<< AllocatedSize << " bytes)\n";
3815	exit(status: 1);
3816	}
3817	}
3818	}
3819
3820	std::vector<BinarySection *> RewriteInstance::getCodeSections() {
3821	std::vector<BinarySection *> CodeSections;
3822	for (BinarySection &Section : BC->textSections())
3823	if (Section.hasValidSectionID())
3824	CodeSections.emplace_back(args: &Section);
3825
3826	auto compareSections = [&](const BinarySection *A, const BinarySection *B) {
3827	// If both A and B have names starting with ".text.cold", then
3828	// - if opts::HotFunctionsAtEnd is true, we want order
3829	// ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
3830	// - if opts::HotFunctionsAtEnd is false, we want order
3831	// ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
3832	if (A->getName().starts_with(Prefix: BC->getColdCodeSectionName()) &&
3833	B->getName().starts_with(Prefix: BC->getColdCodeSectionName())) {
3834	if (A->getName().size() != B->getName().size())
3835	return (opts::HotFunctionsAtEnd)
3836	? (A->getName().size() > B->getName().size())
3837	: (A->getName().size() < B->getName().size());
3838	return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName())
3839	: (A->getName() < B->getName());
3840	}
3841
3842	// Place movers before anything else.
3843	if (A->getName() == BC->getHotTextMoverSectionName())
3844	return true;
3845	if (B->getName() == BC->getHotTextMoverSectionName())
3846	return false;
3847
3848	// Depending on opts::HotFunctionsAtEnd, place main and warm sections in
3849	// order.
3850	if (opts::HotFunctionsAtEnd) {
3851	if (B->getName() == BC->getMainCodeSectionName())
3852	return true;
3853	if (A->getName() == BC->getMainCodeSectionName())
3854	return false;
3855	return (B->getName() == BC->getWarmCodeSectionName());
3856	} else {
3857	if (A->getName() == BC->getMainCodeSectionName())
3858	return true;
3859	if (B->getName() == BC->getMainCodeSectionName())
3860	return false;
3861	return (A->getName() == BC->getWarmCodeSectionName());
3862	}
3863	};
3864
3865	// Determine the order of sections.
3866	llvm::stable_sort(Range&: CodeSections, C: compareSections);
3867
3868	return CodeSections;
3869	}
3870
3871	void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
3872	if (BC->HasRelocations) {
3873	// Map sections for functions with pre-assigned addresses.
3874	for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
3875	const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
3876	if (!OutputAddress)
3877	continue;
3878
3879	ErrorOr<BinarySection &> FunctionSection =
3880	InjectedFunction->getCodeSection();
3881	assert(FunctionSection && "function should have section");
3882	FunctionSection->setOutputAddress(OutputAddress);
3883	MapSection(*FunctionSection, OutputAddress);
3884	InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
3885	InjectedFunction->setImageSize(FunctionSection->getOutputSize());
3886	}
3887
3888	// Populate the list of sections to be allocated.
3889	std::vector<BinarySection *> CodeSections = getCodeSections();
3890
3891	// Remove sections that were pre-allocated (patch sections).
3892	llvm::erase_if(C&: CodeSections, P: [](BinarySection *Section) {
3893	return Section->getOutputAddress();
3894	});
3895	LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
3896	for (const BinarySection *Section : CodeSections)
3897	dbgs() << Section->getName() << '\n';
3898	);
3899
3900	uint64_t PaddingSize = 0; // size of padding required at the end
3901
3902	// Allocate sections starting at a given Address.
3903	auto allocateAt = [&](uint64_t Address) {
3904	const char *LastNonColdSectionName = BC->HasWarmSection
3905	? BC->getWarmCodeSectionName()
3906	: BC->getMainCodeSectionName();
3907	for (BinarySection *Section : CodeSections) {
3908	Address = alignTo(Value: Address, Align: Section->getAlignment());
3909	Section->setOutputAddress(Address);
3910	Address += Section->getOutputSize();
3911
3912	// Hugify: Additional huge page from right side due to
3913	// weird ASLR mapping addresses (4KB aligned)
3914	if (opts::Hugify && !BC->HasFixedLoadAddress &&
3915	Section->getName() == LastNonColdSectionName)
3916	Address = alignTo(Value: Address, Align: Section->getAlignment());
3917	}
3918
3919	// Make sure we allocate enough space for huge pages.
3920	ErrorOr<BinarySection &> TextSection =
3921	BC->getUniqueSectionByName(SectionName: LastNonColdSectionName);
3922	if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
3923	uint64_t HotTextEnd =
3924	TextSection->getOutputAddress() + TextSection->getOutputSize();
3925	HotTextEnd = alignTo(Value: HotTextEnd, Align: BC->PageAlign);
3926	if (HotTextEnd > Address) {
3927	PaddingSize = HotTextEnd - Address;
3928	Address = HotTextEnd;
3929	}
3930	}
3931	return Address;
3932	};
3933
3934	// Check if we can fit code in the original .text
3935	bool AllocationDone = false;
3936	if (opts::UseOldText) {
3937	const uint64_t CodeSize =
3938	allocateAt(BC->OldTextSectionAddress) - BC->OldTextSectionAddress;
3939
3940	if (CodeSize <= BC->OldTextSectionSize) {
3941	BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
3942	<< Twine::utohexstr(Val: opts::AlignText) << " alignment\n";
3943	AllocationDone = true;
3944	} else {
3945	BC->errs()
3946	<< "BOLT-WARNING: original .text too small to fit the new code"
3947	<< " using 0x" << Twine::utohexstr(Val: opts::AlignText)
3948	<< " alignment. " << CodeSize << " bytes needed, have "
3949	<< BC->OldTextSectionSize << " bytes available.\n";
3950	opts::UseOldText = false;
3951	}
3952	}
3953
3954	if (!AllocationDone)
3955	NextAvailableAddress = allocateAt(NextAvailableAddress);
3956
3957	// Do the mapping for ORC layer based on the allocation.
3958	for (BinarySection *Section : CodeSections) {
3959	LLVM_DEBUG(
3960	dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
3961	<< Twine::utohexstr(Section->getAllocAddress()) << " to 0x"
3962	<< Twine::utohexstr(Section->getOutputAddress()) << '\n');
3963	MapSection(*Section, Section->getOutputAddress());
3964	Section->setOutputFileOffset(
3965	getFileOffsetForAddress(Address: Section->getOutputAddress()));
3966	}
3967
3968	// Check if we need to insert a padding section for hot text.
3969	if (PaddingSize && !opts::UseOldText)
3970	BC->outs() << "BOLT-INFO: padding code to 0x"
3971	<< Twine::utohexstr(Val: NextAvailableAddress)
3972	<< " to accommodate hot text\n";
3973
3974	return;
3975	}
3976
3977	// Processing in non-relocation mode.
3978	uint64_t NewTextSectionStartAddress = NextAvailableAddress;
3979
3980	for (auto &BFI : BC->getBinaryFunctions()) {
3981	BinaryFunction &Function = BFI.second;
3982	if (!Function.isEmitted())
3983	continue;
3984
3985	ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();
3986	assert(FuncSection && "cannot find section for function");
3987	FuncSection->setOutputAddress(Function.getAddress());
3988	LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
3989	<< Twine::utohexstr(FuncSection->getAllocAddress())
3990	<< " to 0x" << Twine::utohexstr(Function.getAddress())
3991	<< '\n');
3992	MapSection(*FuncSection, Function.getAddress());
3993	Function.setImageAddress(FuncSection->getAllocAddress());
3994	Function.setImageSize(FuncSection->getOutputSize());
3995	assert(Function.getImageSize() <= Function.getMaxSize() &&
3996	"Unexpected large function");
3997
3998	if (!Function.isSplit())
3999	continue;
4000
4001	assert(Function.getLayout().isHotColdSplit() &&
4002	"Cannot allocate more than two fragments per function in "
4003	"non-relocation mode.");
4004
4005	FunctionFragment &FF =
4006	Function.getLayout().getFragment(Num: FragmentNum::cold());
4007	ErrorOr<BinarySection &> ColdSection =
4008	Function.getCodeSection(Fragment: FF.getFragmentNum());
4009	assert(ColdSection && "cannot find section for cold part");
4010	// Cold fragments are aligned at 16 bytes.
4011	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 16);
4012	FF.setAddress(NextAvailableAddress);
4013	FF.setImageAddress(ColdSection->getAllocAddress());
4014	FF.setImageSize(ColdSection->getOutputSize());
4015	FF.setFileOffset(getFileOffsetForAddress(Address: NextAvailableAddress));
4016	ColdSection->setOutputAddress(FF.getAddress());
4017
4018	LLVM_DEBUG(
4019	dbgs() << formatv(
4020	"BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n",
4021	FF.getImageAddress(), FF.getAddress(), FF.getImageSize()));
4022	MapSection(*ColdSection, FF.getAddress());
4023
4024	NextAvailableAddress += FF.getImageSize();
4025	}
4026
4027	// Add the new text section aggregating all existing code sections.
4028	// This is pseudo-section that serves a purpose of creating a corresponding
4029	// entry in section header table.
4030	const uint64_t NewTextSectionSize =
4031	NextAvailableAddress - NewTextSectionStartAddress;
4032	if (NewTextSectionSize) {
4033	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
4034	/*IsText=*/true,
4035	/*IsAllocatable=*/true);
4036	BinarySection &Section =
4037	BC->registerOrUpdateSection(Name: getBOLTTextSectionName(),
4038	ELFType: ELF::SHT_PROGBITS,
4039	ELFFlags: Flags,
4040	/*Data=*/nullptr,
4041	Size: NewTextSectionSize,
4042	Alignment: 16);
4043	Section.setOutputAddress(NewTextSectionStartAddress);
4044	Section.setOutputFileOffset(
4045	getFileOffsetForAddress(Address: NewTextSectionStartAddress));
4046	}
4047	}
4048
4049	void RewriteInstance::mapAllocatableSections(
4050	BOLTLinker::SectionMapper MapSection) {
4051
4052	if (opts::UseOldText || opts::StrictMode) {
4053	auto tryRewriteSection = [&](BinarySection &OldSection,
4054	BinarySection &NewSection) {
4055	if (OldSection.getSize() < NewSection.getOutputSize())
4056	return;
4057
4058	BC->outs() << "BOLT-INFO: rewriting " << OldSection.getName()
4059	<< " in-place\n";
4060
4061	NewSection.setOutputAddress(OldSection.getAddress());
4062	NewSection.setOutputFileOffset(OldSection.getInputFileOffset());
4063	MapSection(NewSection, OldSection.getAddress());
4064
4065	// Pad contents with zeros.
4066	NewSection.addPadding(PaddingSize: OldSection.getSize() - NewSection.getOutputSize());
4067
4068	// Prevent the original section name from appearing in the section header
4069	// table.
4070	OldSection.setAnonymous(true);
4071	};
4072
4073	if (EHFrameSection) {
4074	BinarySection *NewEHFrameSection =
4075	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
4076	assert(NewEHFrameSection && "New contents expected for .eh_frame");
4077	tryRewriteSection(*EHFrameSection, *NewEHFrameSection);
4078	}
4079	BinarySection *EHSection = getSection(Name: ".gcc_except_table");
4080	BinarySection *NewEHSection =
4081	getSection(Name: getNewSecPrefix() + ".gcc_except_table");
4082	if (EHSection) {
4083	assert(NewEHSection && "New contents expected for .gcc_except_table");
4084	tryRewriteSection(*EHSection, *NewEHSection);
4085	}
4086	}
4087
4088	// Allocate read-only sections first, then writable sections.
4089	enum : uint8_t { ST_READONLY, ST_READWRITE };
4090	for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) {
4091	const uint64_t LastNextAvailableAddress = NextAvailableAddress;
4092	if (SType == ST_READWRITE) {
4093	// Align R+W segment to regular page size
4094	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->RegularPageSize);
4095	NewWritableSegmentAddress = NextAvailableAddress;
4096	}
4097
4098	for (BinarySection &Section : BC->allocatableSections()) {
4099	if (Section.isLinkOnly())
4100	continue;
4101
4102	if (!Section.hasValidSectionID())
4103	continue;
4104
4105	if (Section.isWritable() == (SType == ST_READONLY))
4106	continue;
4107
4108	if (Section.getOutputAddress()) {
4109	LLVM_DEBUG({
4110	dbgs() << "BOLT-DEBUG: section " << Section.getName()
4111	<< " is already mapped at 0x"
4112	<< Twine::utohexstr(Section.getOutputAddress()) << '\n';
4113	});
4114	continue;
4115	}
4116
4117	if (Section.hasSectionRef()) {
4118	LLVM_DEBUG({
4119	dbgs() << "BOLT-DEBUG: mapping original section " << Section.getName()
4120	<< " to 0x" << Twine::utohexstr(Section.getAddress()) << '\n';
4121	});
4122	Section.setOutputAddress(Section.getAddress());
4123	Section.setOutputFileOffset(Section.getInputFileOffset());
4124	MapSection(Section, Section.getAddress());
4125	} else {
4126	uint64_t Alignment = Section.getAlignment();
4127	if (opts::Instrument && StartLinkingRuntimeLib) {
4128	Alignment = BC->RegularPageSize;
4129	StartLinkingRuntimeLib = false;
4130	}
4131	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: Alignment);
4132
4133	LLVM_DEBUG({
4134	dbgs() << "BOLT-DEBUG: mapping section " << Section.getName()
4135	<< " (0x" << Twine::utohexstr(Section.getAllocAddress())
4136	<< ") to 0x" << Twine::utohexstr(NextAvailableAddress) << ":0x"
4137	<< Twine::utohexstr(NextAvailableAddress +
4138	Section.getOutputSize())
4139	<< '\n';
4140	});
4141
4142	MapSection(Section, NextAvailableAddress);
4143	Section.setOutputAddress(NextAvailableAddress);
4144	Section.setOutputFileOffset(
4145	getFileOffsetForAddress(Address: NextAvailableAddress));
4146
4147	NextAvailableAddress += Section.getOutputSize();
4148	}
4149	}
4150
4151	if (SType == ST_READONLY) {
4152	if (PHDRTableAddress) {
4153	// Segment size includes the size of the PHDR area.
4154	NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
4155	} else if (NewTextSegmentAddress) {
4156	// Existing PHDR table would be updated.
4157	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
4158	}
4159	} else if (SType == ST_READWRITE) {
4160	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
4161	// Restore NextAvailableAddress if no new writable sections
4162	if (!NewWritableSegmentSize)
4163	NextAvailableAddress = LastNextAvailableAddress;
4164	}
4165	}
4166	}
4167
4168	void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
4169	if (std::optional<AddressMap> Map = AddressMap::parse(BC&: *BC))
4170	BC->setIOAddressMap(std::move(*Map));
4171
4172	for (BinaryFunction *Function : BC->getAllBinaryFunctions())
4173	Function->updateOutputValues(Linker);
4174	}
4175
4176	void RewriteInstance::patchELFPHDRTable() {
4177	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
4178	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
4179	raw_fd_ostream &OS = Out->os();
4180
4181	// Write/re-write program headers.
4182	Phnum = Obj.getHeader().e_phnum;
4183	if (PHDRTableOffset) {
4184	// Writing new pheader table and adding one new entry for R+X segment.
4185	Phnum += 1;
4186	if (NewWritableSegmentSize) {
4187	// Adding one more entry for R+W segment.
4188	Phnum += 1;
4189	}
4190	} else {
4191	assert(!PHDRTableAddress && "unexpected address for program header table");
4192	PHDRTableOffset = Obj.getHeader().e_phoff;
4193	if (NewWritableSegmentSize) {
4194	BC->errs() << "BOLT-ERROR: unable to add writable segment\n";
4195	exit(status: 1);
4196	}
4197	}
4198
4199	if (opts::Instrument)
4200	Phnum += 2;
4201
4202	// NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
4203	// last segments size based on the NextAvailableAddress variable.
4204	if (!NewWritableSegmentSize) {
4205	if (PHDRTableAddress)
4206	NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
4207	else if (NewTextSegmentAddress)
4208	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
4209	} else {
4210	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
4211	}
4212
4213	const uint64_t SavedPos = OS.tell();
4214	OS.seek(off: PHDRTableOffset);
4215
4216	auto createNewPhdrs = [&]() {
4217	SmallVector<ELF64LEPhdrTy, 3> NewPhdrs;
4218	ELF64LEPhdrTy NewPhdr;
4219	NewPhdr.p_type = ELF::PT_LOAD;
4220	if (PHDRTableAddress) {
4221	NewPhdr.p_offset = PHDRTableOffset;
4222	NewPhdr.p_vaddr = PHDRTableAddress;
4223	NewPhdr.p_paddr = PHDRTableAddress;
4224	} else {
4225	NewPhdr.p_offset = NewTextSegmentOffset;
4226	NewPhdr.p_vaddr = NewTextSegmentAddress;
4227	NewPhdr.p_paddr = NewTextSegmentAddress;
4228	}
4229	NewPhdr.p_filesz = NewTextSegmentSize;
4230	NewPhdr.p_memsz = NewTextSegmentSize;
4231	NewPhdr.p_flags = ELF::PF_X | ELF::PF_R;
4232	NewPhdr.p_align = BC->PageAlign;
4233
4234	if (!opts::Instrument) {
4235	NewPhdrs.push_back(Elt: NewPhdr);
4236	} else {
4237	ErrorOr<BinarySection &> Sec =
4238	BC->getUniqueSectionByName(SectionName: ".bolt.instr.counters");
4239	assert(Sec && "expected one and only one `.bolt.instr.counters` section");
4240	const uint64_t Addr = Sec->getOutputAddress();
4241	const uint64_t Offset = Sec->getOutputFileOffset();
4242	const uint64_t Size = Sec->getOutputSize();
4243	assert(Addr > NewPhdr.p_vaddr &&
4244	Addr + Size < NewPhdr.p_vaddr + NewPhdr.p_memsz &&
4245	"`.bolt.instr.counters` section is expected to be included in the "
4246	"new text sgement");
4247
4248	// Set correct size for the previous header since we are breaking the
4249	// new text segment into three segments.
4250	uint64_t Delta = Addr - NewPhdr.p_vaddr;
4251	NewPhdr.p_filesz = Delta;
4252	NewPhdr.p_memsz = Delta;
4253	NewPhdrs.push_back(Elt: NewPhdr);
4254
4255	// Create a program header for a RW segment that includes the
4256	// `.bolt.instr.counters` section only.
4257	ELF64LEPhdrTy NewPhdrRWSegment;
4258	NewPhdrRWSegment.p_type = ELF::PT_LOAD;
4259	NewPhdrRWSegment.p_offset = Offset;
4260	NewPhdrRWSegment.p_vaddr = Addr;
4261	NewPhdrRWSegment.p_paddr = Addr;
4262	NewPhdrRWSegment.p_filesz = Size;
4263	NewPhdrRWSegment.p_memsz = Size;
4264	NewPhdrRWSegment.p_flags = ELF::PF_R | ELF::PF_W;
4265	NewPhdrRWSegment.p_align = BC->RegularPageSize;
4266	NewPhdrs.push_back(Elt: NewPhdrRWSegment);
4267
4268	// Create a program header for a RX segment that includes all the RX
4269	// sections from runtime library.
4270	ELF64LEPhdrTy NewPhdrRXSegment;
4271	NewPhdrRXSegment.p_type = ELF::PT_LOAD;
4272	const uint64_t AddrRX = alignTo(Value: Addr + Size, Align: BC->RegularPageSize);
4273	const uint64_t OffsetRX = alignTo(Value: Offset + Size, Align: BC->RegularPageSize);
4274	const uint64_t SizeRX = NewTextSegmentSize - (AddrRX - NewPhdr.p_paddr);
4275	NewPhdrRXSegment.p_offset = OffsetRX;
4276	NewPhdrRXSegment.p_vaddr = AddrRX;
4277	NewPhdrRXSegment.p_paddr = AddrRX;
4278	NewPhdrRXSegment.p_filesz = SizeRX;
4279	NewPhdrRXSegment.p_memsz = SizeRX;
4280	NewPhdrRXSegment.p_flags = ELF::PF_X | ELF::PF_R;
4281	NewPhdrRXSegment.p_align = BC->RegularPageSize;
4282	NewPhdrs.push_back(Elt: NewPhdrRXSegment);
4283	}
4284
4285	return NewPhdrs;
4286	};
4287
4288	auto writeNewSegmentPhdrs = [&]() {
4289	if (PHDRTableAddress || NewTextSegmentSize) {
4290	SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
4291	OS.write(Ptr: reinterpret_cast<const char *>(NewPhdrs.data()),
4292	Size: sizeof(ELF64LE::Phdr) * NewPhdrs.size());
4293	}
4294
4295	if (NewWritableSegmentSize) {
4296	ELF64LEPhdrTy NewPhdr;
4297	NewPhdr.p_type = ELF::PT_LOAD;
4298	NewPhdr.p_offset = getFileOffsetForAddress(Address: NewWritableSegmentAddress);
4299	NewPhdr.p_vaddr = NewWritableSegmentAddress;
4300	NewPhdr.p_paddr = NewWritableSegmentAddress;
4301	NewPhdr.p_filesz = NewWritableSegmentSize;
4302	NewPhdr.p_memsz = NewWritableSegmentSize;
4303	NewPhdr.p_align = BC->RegularPageSize;
4304	NewPhdr.p_flags = ELF::PF_R | ELF::PF_W;
4305	OS.write(Ptr: reinterpret_cast<const char *>(&NewPhdr), Size: sizeof(NewPhdr));
4306	}
4307	};
4308
4309	bool ModdedGnuStack = false;
4310	bool AddedSegment = false;
4311
4312	// Copy existing program headers with modifications.
4313	for (const ELF64LE::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers())) {
4314	ELF64LE::Phdr NewPhdr = Phdr;
4315	switch (Phdr.p_type) {
4316	case ELF::PT_PHDR:
4317	if (PHDRTableAddress) {
4318	NewPhdr.p_offset = PHDRTableOffset;
4319	NewPhdr.p_vaddr = PHDRTableAddress;
4320	NewPhdr.p_paddr = PHDRTableAddress;
4321	NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum;
4322	NewPhdr.p_memsz = sizeof(NewPhdr) * Phnum;
4323	}
4324	break;
4325	case ELF::PT_GNU_EH_FRAME: {
4326	ErrorOr<BinarySection &> EHFrameHdrSec = BC->getUniqueSectionByName(
4327	SectionName: getNewSecPrefix() + getEHFrameHdrSectionName());
4328	if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() &&
4329	EHFrameHdrSec->isFinalized()) {
4330	NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset();
4331	NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress();
4332	NewPhdr.p_paddr = EHFrameHdrSec->getOutputAddress();
4333	NewPhdr.p_filesz = EHFrameHdrSec->getOutputSize();
4334	NewPhdr.p_memsz = EHFrameHdrSec->getOutputSize();
4335	}
4336	break;
4337	}
4338	case ELF::PT_GNU_STACK:
4339	if (opts::UseGnuStack) {
4340	// Overwrite the header with the new segment header.
4341	assert(!opts::Instrument);
4342	SmallVector<ELF64LE::Phdr, 3> NewPhdrs = createNewPhdrs();
4343	assert(NewPhdrs.size() == 1 &&
4344	"expect exactly one program header was created");
4345	NewPhdr = NewPhdrs[0];
4346	ModdedGnuStack = true;
4347	}
4348	break;
4349	case ELF::PT_DYNAMIC:
4350	if (!opts::UseGnuStack) {
4351	// Insert new headers before DYNAMIC.
4352	writeNewSegmentPhdrs();
4353	AddedSegment = true;
4354	}
4355	break;
4356	}
4357	OS.write(Ptr: reinterpret_cast<const char *>(&NewPhdr), Size: sizeof(NewPhdr));
4358	}
4359
4360	if (!opts::UseGnuStack && !AddedSegment) {
4361	// Append new headers to the end of the table.
4362	writeNewSegmentPhdrs();
4363	}
4364
4365	if (opts::UseGnuStack && !ModdedGnuStack) {
4366	BC->errs()
4367	<< "BOLT-ERROR: could not find PT_GNU_STACK program header to modify\n";
4368	exit(status: 1);
4369	}
4370
4371	OS.seek(off: SavedPos);
4372	}
4373
4374	namespace {
4375
4376	/// Write padding to \p OS such that its current \p Offset becomes aligned
4377	/// at \p Alignment. Return new (aligned) offset.
4378	uint64_t appendPadding(raw_pwrite_stream &OS, uint64_t Offset,
4379	uint64_t Alignment) {
4380	if (!Alignment)
4381	return Offset;
4382
4383	const uint64_t PaddingSize =
4384	offsetToAlignment(Value: Offset, Alignment: llvm::Align(Alignment));
4385	for (unsigned I = 0; I < PaddingSize; ++I)
4386	OS.write(C: (unsigned char)0);
4387	return Offset + PaddingSize;
4388	}
4389
4390	}
4391
4392	void RewriteInstance::rewriteNoteSections() {
4393	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
4394	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
4395	raw_fd_ostream &OS = Out->os();
4396
4397	uint64_t NextAvailableOffset = std::max(
4398	a: getFileOffsetForAddress(Address: NextAvailableAddress), b: FirstNonAllocatableOffset);
4399	OS.seek(off: NextAvailableOffset);
4400
4401	// Copy over non-allocatable section contents and update file offsets.
4402	for (const ELF64LE::Shdr &Section : cantFail(ValOrErr: Obj.sections())) {
4403	if (Section.sh_type == ELF::SHT_NULL)
4404	continue;
4405	if (Section.sh_flags & ELF::SHF_ALLOC)
4406	continue;
4407
4408	SectionRef SecRef = ELF64LEFile->toSectionRef(Sec: &Section);
4409	BinarySection *BSec = BC->getSectionForSectionRef(Section: SecRef);
4410	assert(BSec && !BSec->isAllocatable() &&
4411	"Matching non-allocatable BinarySection should exist.");
4412
4413	StringRef SectionName =
4414	cantFail(ValOrErr: Obj.getSectionName(Section), Msg: "cannot get section name");
4415	if (shouldStrip(Section, SectionName))
4416	continue;
4417
4418	// Insert padding as needed.
4419	NextAvailableOffset =
4420	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.sh_addralign);
4421
4422	// New section size.
4423	uint64_t Size = 0;
4424	bool DataWritten = false;
4425	// Copy over section contents unless it's one of the sections we overwrite.
4426	if (!willOverwriteSection(SectionName)) {
4427	Size = Section.sh_size;
4428	StringRef Dataref = InputFile->getData().substr(Start: Section.sh_offset, N: Size);
4429	std::string Data;
4430	if (BSec->getPatcher()) {
4431	Data = BSec->getPatcher()->patchBinary(BinaryContents: Dataref);
4432	Dataref = StringRef(Data);
4433	}
4434
4435	// Section was expanded, so need to treat it as overwrite.
4436	if (Size != Dataref.size()) {
4437	BSec = &BC->registerOrUpdateNoteSection(
4438	Name: SectionName, Data: copyByteArray(Buffer: Dataref), Size: Dataref.size());
4439	Size = 0;
4440	} else {
4441	OS << Dataref;
4442	DataWritten = true;
4443
4444	// Add padding as the section extension might rely on the alignment.
4445	Size = appendPadding(OS, Offset: Size, Alignment: Section.sh_addralign);
4446	}
4447	}
4448
4449	// Perform section post-processing.
4450	assert(BSec->getAlignment() <= Section.sh_addralign &&
4451	"alignment exceeds value in file");
4452
4453	if (BSec->getAllocAddress()) {
4454	assert(!DataWritten && "Writing section twice.");
4455	(void)DataWritten;
4456	Size += BSec->write(OS);
4457	}
4458
4459	BSec->setOutputFileOffset(NextAvailableOffset);
4460	BSec->flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
4461	return getNewValueForSymbol(Name: S->getName());
4462	});
4463
4464	// Section contents are no longer needed, but we need to update the size so
4465	// that it will be reflected in the section header table.
4466	BSec->updateContents(NewData: nullptr, NewSize: Size);
4467
4468	NextAvailableOffset += Size;
4469	}
4470
4471	// Write new note sections.
4472	for (BinarySection &Section : BC->nonAllocatableSections()) {
4473	if (Section.getOutputFileOffset() || !Section.getAllocAddress())
4474	continue;
4475
4476	assert(!Section.hasPendingRelocations() && "cannot have pending relocs");
4477
4478	NextAvailableOffset =
4479	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.getAlignment());
4480	Section.setOutputFileOffset(NextAvailableOffset);
4481
4482	LLVM_DEBUG(
4483	dbgs() << "BOLT-DEBUG: writing out new section " << Section.getName()
4484	<< " of size " << Section.getOutputSize() << " at offset 0x"
4485	<< Twine::utohexstr(Section.getOutputFileOffset()) << '\n');
4486
4487	NextAvailableOffset += Section.write(OS);
4488	}
4489	}
4490
4491	template <typename ELFT>
4492	void RewriteInstance::finalizeSectionStringTable(ELFObjectFile<ELFT> *File) {
4493	// Pre-populate section header string table.
4494	for (const BinarySection &Section : BC->sections())
4495	if (!Section.isAnonymous())
4496	SHStrTab.add(S: Section.getOutputName());
4497	SHStrTab.finalize();
4498
4499	const size_t SHStrTabSize = SHStrTab.getSize();
4500	uint8_t *DataCopy = new uint8_t[SHStrTabSize];
4501	memset(s: DataCopy, c: 0, n: SHStrTabSize);
4502	SHStrTab.write(Buf: DataCopy);
4503	BC->registerOrUpdateNoteSection(Name: ".shstrtab",
4504	Data: DataCopy,
4505	Size: SHStrTabSize,
4506	/*Alignment=*/1,
4507	/*IsReadOnly=*/true,
4508	ELFType: ELF::SHT_STRTAB);
4509	}
4510
4511	void RewriteInstance::addBoltInfoSection() {
4512	std::string DescStr;
4513	raw_string_ostream DescOS(DescStr);
4514
4515	DescOS << "BOLT revision: " << BoltRevision << ", "
4516	<< "command line:";
4517	for (int I = 0; I < Argc; ++I)
4518	DescOS << " " << Argv[I];
4519
4520	// Encode as GNU GOLD VERSION so it is easily printable by 'readelf -n'
4521	const std::string BoltInfo =
4522	BinarySection::encodeELFNote(NameStr: "GNU", DescStr, Type: 4 /*NT_GNU_GOLD_VERSION*/);
4523	BC->registerOrUpdateNoteSection(Name: ".note.bolt_info", Data: copyByteArray(Buffer: BoltInfo),
4524	Size: BoltInfo.size(),
4525	/*Alignment=*/1,
4526	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4527	}
4528
4529	void RewriteInstance::addBATSection() {
4530	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME, Data: nullptr,
4531	Size: 0,
4532	/*Alignment=*/1,
4533	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4534	}
4535
4536	void RewriteInstance::encodeBATSection() {
4537	std::string DescStr;
4538	raw_string_ostream DescOS(DescStr);
4539
4540	BAT->write(BC: *BC, OS&: DescOS);
4541
4542	const std::string BoltInfo =
4543	BinarySection::encodeELFNote(NameStr: "BOLT", DescStr, Type: BinarySection::NT_BOLT_BAT);
4544	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME,
4545	Data: copyByteArray(Buffer: BoltInfo), Size: BoltInfo.size(),
4546	/*Alignment=*/1,
4547	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4548	BC->outs() << "BOLT-INFO: BAT section size (bytes): " << BoltInfo.size()
4549	<< '\n';
4550	}
4551
4552	template <typename ELFShdrTy>
4553	bool RewriteInstance::shouldStrip(const ELFShdrTy &Section,
4554	StringRef SectionName) {
4555	// Strip non-allocatable relocation sections.
4556	if (!(Section.sh_flags & ELF::SHF_ALLOC) && Section.sh_type == ELF::SHT_RELA)
4557	return true;
4558
4559	// Strip debug sections if not updating them.
4560	if (isDebugSection(SectionName) && !opts::UpdateDebugSections)
4561	return true;
4562
4563	// Strip symtab section if needed
4564	if (opts::RemoveSymtab && Section.sh_type == ELF::SHT_SYMTAB)
4565	return true;
4566
4567	return false;
4568	}
4569
4570	template <typename ELFT>
4571	std::vector<typename object::ELFObjectFile<ELFT>::Elf_Shdr>
4572	RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
4573	std::vector<uint32_t> &NewSectionIndex) {
4574	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4575	const ELFFile<ELFT> &Obj = File->getELFFile();
4576	typename ELFT::ShdrRange Sections = cantFail(Obj.sections());
4577
4578	// Keep track of section header entries attached to the corresponding section.
4579	std::vector<std::pair<BinarySection *, ELFShdrTy>> OutputSections;
4580	auto addSection = [&](const ELFShdrTy &Section, BinarySection &BinSec) {
4581	ELFShdrTy NewSection = Section;
4582	NewSection.sh_name = SHStrTab.getOffset(S: BinSec.getOutputName());
4583	OutputSections.emplace_back(&BinSec, std::move(NewSection));
4584	};
4585
4586	// Copy over entries for original allocatable sections using modified name.
4587	for (const ELFShdrTy &Section : Sections) {
4588	// Always ignore this section.
4589	if (Section.sh_type == ELF::SHT_NULL) {
4590	OutputSections.emplace_back(nullptr, Section);
4591	continue;
4592	}
4593
4594	if (!(Section.sh_flags & ELF::SHF_ALLOC))
4595	continue;
4596
4597	SectionRef SecRef = File->toSectionRef(&Section);
4598	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4599	assert(BinSec && "Matching BinarySection should exist.");
4600
4601	// Exclude anonymous sections.
4602	if (BinSec->isAnonymous())
4603	continue;
4604
4605	addSection(Section, *BinSec);
4606	}
4607
4608	for (BinarySection &Section : BC->allocatableSections()) {
4609	if (!Section.isFinalized())
4610	continue;
4611
4612	if (Section.hasSectionRef() || Section.isAnonymous()) {
4613	if (opts::Verbosity)
4614	BC->outs() << "BOLT-INFO: not writing section header for section "
4615	<< Section.getOutputName() << '\n';
4616	continue;
4617	}
4618
4619	if (opts::Verbosity >= 1)
4620	BC->outs() << "BOLT-INFO: writing section header for "
4621	<< Section.getOutputName() << '\n';
4622	ELFShdrTy NewSection;
4623	NewSection.sh_type = ELF::SHT_PROGBITS;
4624	NewSection.sh_addr = Section.getOutputAddress();
4625	NewSection.sh_offset = Section.getOutputFileOffset();
4626	NewSection.sh_size = Section.getOutputSize();
4627	NewSection.sh_entsize = 0;
4628	NewSection.sh_flags = Section.getELFFlags();
4629	NewSection.sh_link = 0;
4630	NewSection.sh_info = 0;
4631	NewSection.sh_addralign = Section.getAlignment();
4632	addSection(NewSection, Section);
4633	}
4634
4635	// Sort all allocatable sections by their offset.
4636	llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) {
4637	return A.second.sh_offset < B.second.sh_offset;
4638	});
4639
4640	// Fix section sizes to prevent overlapping.
4641	ELFShdrTy *PrevSection = nullptr;
4642	BinarySection *PrevBinSec = nullptr;
4643	for (auto &SectionKV : OutputSections) {
4644	ELFShdrTy &Section = SectionKV.second;
4645
4646	// Ignore NOBITS sections as they don't take any space in the file.
4647	if (Section.sh_type == ELF::SHT_NOBITS)
4648	continue;
4649
4650	// Note that address continuity is not guaranteed as sections could be
4651	// placed in different loadable segments.
4652	if (PrevSection &&
4653	PrevSection->sh_offset + PrevSection->sh_size > Section.sh_offset) {
4654	if (opts::Verbosity > 1)
4655	BC->outs() << "BOLT-INFO: adjusting size for section "
4656	<< PrevBinSec->getOutputName() << '\n';
4657	PrevSection->sh_size = Section.sh_offset - PrevSection->sh_offset;
4658	}
4659
4660	PrevSection = &Section;
4661	PrevBinSec = SectionKV.first;
4662	}
4663
4664	uint64_t LastFileOffset = 0;
4665
4666	// Copy over entries for non-allocatable sections performing necessary
4667	// adjustments.
4668	for (const ELFShdrTy &Section : Sections) {
4669	if (Section.sh_type == ELF::SHT_NULL)
4670	continue;
4671	if (Section.sh_flags & ELF::SHF_ALLOC)
4672	continue;
4673
4674	StringRef SectionName =
4675	cantFail(Obj.getSectionName(Section), "cannot get section name");
4676
4677	if (shouldStrip(Section, SectionName))
4678	continue;
4679
4680	SectionRef SecRef = File->toSectionRef(&Section);
4681	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4682	assert(BinSec && "Matching BinarySection should exist.");
4683
4684	ELFShdrTy NewSection = Section;
4685	NewSection.sh_offset = BinSec->getOutputFileOffset();
4686	NewSection.sh_size = BinSec->getOutputSize();
4687
4688	if (NewSection.sh_type == ELF::SHT_SYMTAB)
4689	NewSection.sh_info = NumLocalSymbols;
4690
4691	addSection(NewSection, *BinSec);
4692
4693	LastFileOffset = BinSec->getOutputFileOffset();
4694	}
4695
4696	// Create entries for new non-allocatable sections.
4697	for (BinarySection &Section : BC->nonAllocatableSections()) {
4698	if (Section.getOutputFileOffset() <= LastFileOffset)
4699	continue;
4700
4701	if (opts::Verbosity >= 1)
4702	BC->outs() << "BOLT-INFO: writing section header for "
4703	<< Section.getOutputName() << '\n';
4704
4705	ELFShdrTy NewSection;
4706	NewSection.sh_type = Section.getELFType();
4707	NewSection.sh_addr = 0;
4708	NewSection.sh_offset = Section.getOutputFileOffset();
4709	NewSection.sh_size = Section.getOutputSize();
4710	NewSection.sh_entsize = 0;
4711	NewSection.sh_flags = Section.getELFFlags();
4712	NewSection.sh_link = 0;
4713	NewSection.sh_info = 0;
4714	NewSection.sh_addralign = Section.getAlignment();
4715
4716	addSection(NewSection, Section);
4717	}
4718
4719	// Assign indices to sections.
4720	for (uint32_t Index = 1; Index < OutputSections.size(); ++Index)
4721	OutputSections[Index].first->setIndex(Index);
4722
4723	// Update section index mapping
4724	NewSectionIndex.clear();
4725	NewSectionIndex.resize(Sections.size(), 0);
4726	for (const ELFShdrTy &Section : Sections) {
4727	if (Section.sh_type == ELF::SHT_NULL)
4728	continue;
4729
4730	size_t OrgIndex = std::distance(Sections.begin(), &Section);
4731
4732	SectionRef SecRef = File->toSectionRef(&Section);
4733	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4734	assert(BinSec && "BinarySection should exist for an input section.");
4735
4736	// Some sections are stripped
4737	if (!BinSec->hasValidIndex())
4738	continue;
4739
4740	NewSectionIndex[OrgIndex] = BinSec->getIndex();
4741	}
4742
4743	std::vector<ELFShdrTy> SectionsOnly(OutputSections.size());
4744	llvm::copy(llvm::make_second_range(OutputSections), SectionsOnly.begin());
4745
4746	return SectionsOnly;
4747	}
4748
4749	// Rewrite section header table inserting new entries as needed. The sections
4750	// header table size itself may affect the offsets of other sections,
4751	// so we are placing it at the end of the binary.
4752	//
4753	// As we rewrite entries we need to track how many sections were inserted
4754	// as it changes the sh_link value. We map old indices to new ones for
4755	// existing sections.
4756	template <typename ELFT>
4757	void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {
4758	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4759	using ELFEhdrTy = typename ELFObjectFile<ELFT>::Elf_Ehdr;
4760	raw_fd_ostream &OS = Out->os();
4761	const ELFFile<ELFT> &Obj = File->getELFFile();
4762
4763	// Mapping from old section indices to new ones
4764	std::vector<uint32_t> NewSectionIndex;
4765	std::vector<ELFShdrTy> OutputSections =
4766	getOutputSections(File, NewSectionIndex);
4767	LLVM_DEBUG(
4768	dbgs() << "BOLT-DEBUG: old to new section index mapping:\n";
4769	for (uint64_t I = 0; I < NewSectionIndex.size(); ++I)
4770	dbgs() << " " << I << " -> " << NewSectionIndex[I] << '\n';
4771	);
4772
4773	// Align starting address for section header table. There's no architecutal
4774	// need to align this, it is just for pleasant human readability.
4775	uint64_t SHTOffset = OS.tell();
4776	SHTOffset = appendPadding(OS, Offset: SHTOffset, Alignment: 16);
4777
4778	// Write all section header entries while patching section references.
4779	for (ELFShdrTy &Section : OutputSections) {
4780	Section.sh_link = NewSectionIndex[Section.sh_link];
4781	if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA)
4782	Section.sh_info = NewSectionIndex[Section.sh_info];
4783	OS.write(Ptr: reinterpret_cast<const char *>(&Section), Size: sizeof(Section));
4784	}
4785
4786	// Fix ELF header.
4787	ELFEhdrTy NewEhdr = Obj.getHeader();
4788
4789	if (BC->HasRelocations) {
4790	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
4791	NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();
4792	else
4793	NewEhdr.e_entry = getNewFunctionAddress(OldAddress: NewEhdr.e_entry);
4794	assert((NewEhdr.e_entry || !Obj.getHeader().e_entry) &&
4795	"cannot find new address for entry point");
4796	}
4797	if (PHDRTableOffset) {
4798	NewEhdr.e_phoff = PHDRTableOffset;
4799	NewEhdr.e_phnum = Phnum;
4800	}
4801	NewEhdr.e_shoff = SHTOffset;
4802	NewEhdr.e_shnum = OutputSections.size();
4803	NewEhdr.e_shstrndx = NewSectionIndex[NewEhdr.e_shstrndx];
4804	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewEhdr), Size: sizeof(NewEhdr), Offset: 0);
4805	}
4806
4807	template <typename ELFT, typename WriteFuncTy, typename StrTabFuncTy>
4808	void RewriteInstance::updateELFSymbolTable(
4809	ELFObjectFile<ELFT> *File, bool IsDynSym,
4810	const typename object::ELFObjectFile<ELFT>::Elf_Shdr &SymTabSection,
4811	const std::vector<uint32_t> &NewSectionIndex, WriteFuncTy Write,
4812	StrTabFuncTy AddToStrTab) {
4813	const ELFFile<ELFT> &Obj = File->getELFFile();
4814	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
4815
4816	StringRef StringSection =
4817	cantFail(Obj.getStringTableForSymtab(SymTabSection));
4818
4819	unsigned NumHotTextSymsUpdated = 0;
4820	unsigned NumHotDataSymsUpdated = 0;
4821
4822	std::map<const BinaryFunction *, uint64_t> IslandSizes;
4823	auto getConstantIslandSize = [&IslandSizes](const BinaryFunction &BF) {
4824	auto Itr = IslandSizes.find(x: &BF);
4825	if (Itr != IslandSizes.end())
4826	return Itr->second;
4827	return IslandSizes[&BF] = BF.estimateConstantIslandSize();
4828	};
4829
4830	// Symbols for the new symbol table.
4831	std::vector<ELFSymTy> Symbols;
4832
4833	bool EmittedColdFileSymbol = false;
4834
4835	auto getNewSectionIndex = [&](uint32_t OldIndex) {
4836	// For dynamic symbol table, the section index could be wrong on the input,
4837	// and its value is ignored by the runtime if it's different from
4838	// SHN_UNDEF and SHN_ABS.
4839	// However, we still need to update dynamic symbol table, so return a
4840	// section index, even though the index is broken.
4841	if (IsDynSym && OldIndex >= NewSectionIndex.size())
4842	return OldIndex;
4843
4844	assert(OldIndex < NewSectionIndex.size() && "section index out of bounds");
4845	const uint32_t NewIndex = NewSectionIndex[OldIndex];
4846
4847	// We may have stripped the section that dynsym was referencing due to
4848	// the linker bug. In that case return the old index avoiding marking
4849	// the symbol as undefined.
4850	if (IsDynSym && NewIndex != OldIndex && NewIndex == ELF::SHN_UNDEF)
4851	return OldIndex;
4852	return NewIndex;
4853	};
4854
4855	// Get the extra symbol name of a split fragment; used in addExtraSymbols.
4856	auto getSplitSymbolName = [&](const FunctionFragment &FF,
4857	const ELFSymTy &FunctionSymbol) {
4858	SmallString<256> SymbolName;
4859	if (BC->HasWarmSection)
4860	SymbolName =
4861	formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
4862	FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
4863	else
4864	SymbolName = formatv("{0}.cold.{1}",
4865	cantFail(FunctionSymbol.getName(StringSection)),
4866	FF.getFragmentNum().get() - 1);
4867	return SymbolName;
4868	};
4869
4870	// Add extra symbols for the function.
4871	//
4872	// Note that addExtraSymbols() could be called multiple times for the same
4873	// function with different FunctionSymbol matching the main function entry
4874	// point.
4875	auto addExtraSymbols = [&](const BinaryFunction &Function,
4876	const ELFSymTy &FunctionSymbol) {
4877	if (Function.isFolded()) {
4878	BinaryFunction *ICFParent = Function.getFoldedIntoFunction();
4879	while (ICFParent->isFolded())
4880	ICFParent = ICFParent->getFoldedIntoFunction();
4881	ELFSymTy ICFSymbol = FunctionSymbol;
4882	SmallVector<char, 256> Buf;
4883	ICFSymbol.st_name =
4884	AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection)))
4885	.concat(Suffix: ".icf.0")
4886	.toStringRef(Out&: Buf));
4887	ICFSymbol.st_value = ICFParent->getOutputAddress();
4888	ICFSymbol.st_size = ICFParent->getOutputSize();
4889	ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex();
4890	Symbols.emplace_back(ICFSymbol);
4891	}
4892	if (Function.isSplit()) {
4893	// Prepend synthetic FILE symbol to prevent local cold fragments from
4894	// colliding with existing symbols with the same name.
4895	if (!EmittedColdFileSymbol &&
4896	FunctionSymbol.getBinding() == ELF::STB_GLOBAL) {
4897	ELFSymTy FileSymbol;
4898	FileSymbol.st_shndx = ELF::SHN_ABS;
4899	FileSymbol.st_name = AddToStrTab(getBOLTFileSymbolName());
4900	FileSymbol.st_value = 0;
4901	FileSymbol.st_size = 0;
4902	FileSymbol.st_other = 0;
4903	FileSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FILE);
4904	Symbols.emplace_back(FileSymbol);
4905	EmittedColdFileSymbol = true;
4906	}
4907	for (const FunctionFragment &FF :
4908	Function.getLayout().getSplitFragments()) {
4909	if (FF.getAddress()) {
4910	ELFSymTy NewColdSym = FunctionSymbol;
4911	const SmallString<256> SymbolName =
4912	getSplitSymbolName(FF, FunctionSymbol);
4913	NewColdSym.st_name = AddToStrTab(SymbolName);
4914	NewColdSym.st_shndx =
4915	Function.getCodeSection(Fragment: FF.getFragmentNum())->getIndex();
4916	NewColdSym.st_value = FF.getAddress();
4917	NewColdSym.st_size = FF.getImageSize();
4918	NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
4919	Symbols.emplace_back(NewColdSym);
4920	}
4921	}
4922	}
4923	if (Function.hasConstantIsland()) {
4924	uint64_t DataMark = Function.getOutputDataAddress();
4925	uint64_t CISize = getConstantIslandSize(Function);
4926	uint64_t CodeMark = DataMark + CISize;
4927	ELFSymTy DataMarkSym = FunctionSymbol;
4928	DataMarkSym.st_name = AddToStrTab("$d");
4929	DataMarkSym.st_value = DataMark;
4930	DataMarkSym.st_size = 0;
4931	DataMarkSym.setType(ELF::STT_NOTYPE);
4932	DataMarkSym.setBinding(ELF::STB_LOCAL);
4933	ELFSymTy CodeMarkSym = DataMarkSym;
4934	CodeMarkSym.st_name = AddToStrTab("$x");
4935	CodeMarkSym.st_value = CodeMark;
4936	Symbols.emplace_back(DataMarkSym);
4937	Symbols.emplace_back(CodeMarkSym);
4938	}
4939	if (Function.hasConstantIsland() && Function.isSplit()) {
4940	uint64_t DataMark = Function.getOutputColdDataAddress();
4941	uint64_t CISize = getConstantIslandSize(Function);
4942	uint64_t CodeMark = DataMark + CISize;
4943	ELFSymTy DataMarkSym = FunctionSymbol;
4944	DataMarkSym.st_name = AddToStrTab("$d");
4945	DataMarkSym.st_value = DataMark;
4946	DataMarkSym.st_size = 0;
4947	DataMarkSym.setType(ELF::STT_NOTYPE);
4948	DataMarkSym.setBinding(ELF::STB_LOCAL);
4949	ELFSymTy CodeMarkSym = DataMarkSym;
4950	CodeMarkSym.st_name = AddToStrTab("$x");
4951	CodeMarkSym.st_value = CodeMark;
4952	Symbols.emplace_back(DataMarkSym);
4953	Symbols.emplace_back(CodeMarkSym);
4954	}
4955	};
4956
4957	// For regular (non-dynamic) symbol table, exclude symbols referring
4958	// to non-allocatable sections.
4959	auto shouldStrip = [&](const ELFSymTy &Symbol) {
4960	if (Symbol.isAbsolute() || !Symbol.isDefined())
4961	return false;
4962
4963	// If we cannot link the symbol to a section, leave it as is.
4964	Expected<const typename ELFT::Shdr *> Section =
4965	Obj.getSection(Symbol.st_shndx);
4966	if (!Section)
4967	return false;
4968
4969	// Remove the section symbol iif the corresponding section was stripped.
4970	if (Symbol.getType() == ELF::STT_SECTION) {
4971	if (!getNewSectionIndex(Symbol.st_shndx))
4972	return true;
4973	return false;
4974	}
4975
4976	// Symbols in non-allocatable sections are typically remnants of relocations
4977	// emitted under "-emit-relocs" linker option. Delete those as we delete
4978	// relocations against non-allocatable sections.
4979	if (!((*Section)->sh_flags & ELF::SHF_ALLOC))
4980	return true;
4981
4982	return false;
4983	};
4984
4985	for (const ELFSymTy &Symbol : cantFail(Obj.symbols(&SymTabSection))) {
4986	// For regular (non-dynamic) symbol table strip unneeded symbols.
4987	if (!IsDynSym && shouldStrip(Symbol))
4988	continue;
4989
4990	const BinaryFunction *Function =
4991	BC->getBinaryFunctionAtAddress(Symbol.st_value);
4992	// Ignore false function references, e.g. when the section address matches
4993	// the address of the function.
4994	if (Function && Symbol.getType() == ELF::STT_SECTION)
4995	Function = nullptr;
4996
4997	// For non-dynamic symtab, make sure the symbol section matches that of
4998	// the function. It can mismatch e.g. if the symbol is a section marker
4999	// in which case we treat the symbol separately from the function.
5000	// For dynamic symbol table, the section index could be wrong on the input,
5001	// and its value is ignored by the runtime if it's different from
5002	// SHN_UNDEF and SHN_ABS.
5003	if (!IsDynSym && Function &&
5004	Symbol.st_shndx !=
5005	Function->getOriginSection()->getSectionRef().getIndex())
5006	Function = nullptr;
5007
5008	// Create a new symbol based on the existing symbol.
5009	ELFSymTy NewSymbol = Symbol;
5010
5011	// Handle special symbols based on their name.
5012	Expected<StringRef> SymbolName = Symbol.getName(StringSection);
5013	assert(SymbolName && "cannot get symbol name");
5014
5015	auto updateSymbolValue = [&](const StringRef Name,
5016	std::optional<uint64_t> Value = std::nullopt) {
5017	NewSymbol.st_value = Value ? *Value : getNewValueForSymbol(Name);
5018	NewSymbol.st_shndx = ELF::SHN_ABS;
5019	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
5020	<< Twine::utohexstr(Val: NewSymbol.st_value) << '\n';
5021	};
5022
5023	if (*SymbolName == "__hot_start" || *SymbolName == "__hot_end") {
5024	if (opts::HotText) {
5025	updateSymbolValue(*SymbolName);
5026	++NumHotTextSymsUpdated;
5027	}
5028	goto registerSymbol;
5029	}
5030
5031	if (*SymbolName == "__hot_data_start" || *SymbolName == "__hot_data_end") {
5032	if (opts::HotData) {
5033	updateSymbolValue(*SymbolName);
5034	++NumHotDataSymsUpdated;
5035	}
5036	goto registerSymbol;
5037	}
5038
5039	if (*SymbolName == "_end") {
5040	if (NextAvailableAddress > Symbol.st_value)
5041	updateSymbolValue(*SymbolName, NextAvailableAddress);
5042	goto registerSymbol;
5043	}
5044
5045	if (Function) {
5046	// If the symbol matched a function that was not emitted, update the
5047	// corresponding section index but otherwise leave it unchanged.
5048	if (Function->isEmitted()) {
5049	NewSymbol.st_value = Function->getOutputAddress();
5050	NewSymbol.st_size = Function->getOutputSize();
5051	NewSymbol.st_shndx = Function->getCodeSection()->getIndex();
5052	} else if (Symbol.st_shndx < ELF::SHN_LORESERVE) {
5053	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
5054	}
5055
5056	// Add new symbols to the symbol table if necessary.
5057	if (!IsDynSym)
5058	addExtraSymbols(*Function, NewSymbol);
5059	} else {
5060	// Check if the function symbol matches address inside a function, i.e.
5061	// it marks a secondary entry point.
5062	Function =
5063	(Symbol.getType() == ELF::STT_FUNC)
5064	? BC->getBinaryFunctionContainingAddress(Symbol.st_value,
5065	/*CheckPastEnd=*/false,
5066	/*UseMaxSize=*/true)
5067	: nullptr;
5068
5069	if (Function && Function->isEmitted()) {
5070	assert(Function->getLayout().isHotColdSplit() &&
5071	"Adding symbols based on cold fragment when there are more than "
5072	"2 fragments");
5073	const uint64_t OutputAddress =
5074	Function->translateInputToOutputAddress(Address: Symbol.st_value);
5075
5076	NewSymbol.st_value = OutputAddress;
5077	// Force secondary entry points to have zero size.
5078	NewSymbol.st_size = 0;
5079
5080	// Find fragment containing entrypoint
5081	FunctionLayout::fragment_const_iterator FF = llvm::find_if(
5082	Function->getLayout().fragments(), [&](const FunctionFragment &FF) {
5083	uint64_t Lo = FF.getAddress();
5084	uint64_t Hi = Lo + FF.getImageSize();
5085	return Lo <= OutputAddress && OutputAddress < Hi;
5086	});
5087
5088	if (FF == Function->getLayout().fragment_end()) {
5089	assert(
5090	OutputAddress >= Function->getCodeSection()->getOutputAddress() &&
5091	OutputAddress < (Function->getCodeSection()->getOutputAddress() +
5092	Function->getCodeSection()->getOutputSize()) &&
5093	"Cannot locate fragment containing secondary entrypoint");
5094	FF = Function->getLayout().fragment_begin();
5095	}
5096
5097	NewSymbol.st_shndx =
5098	Function->getCodeSection(Fragment: FF->getFragmentNum())->getIndex();
5099	} else {
5100	// Check if the symbol belongs to moved data object and update it.
5101	BinaryData *BD = opts::ReorderData.empty()
5102	? nullptr
5103	: BC->getBinaryDataAtAddress(Symbol.st_value);
5104	if (BD && BD->isMoved() && !BD->isJumpTable()) {
5105	assert((!BD->getSize() || !Symbol.st_size ||
5106	Symbol.st_size == BD->getSize()) &&
5107	"sizes must match");
5108
5109	BinarySection &OutputSection = BD->getOutputSection();
5110	assert(OutputSection.getIndex());
5111	LLVM_DEBUG(dbgs()
5112	<< "BOLT-DEBUG: moving " << BD->getName() << " from "
5113	<< *BC->getSectionNameForAddress(Symbol.st_value) << " ("
5114	<< Symbol.st_shndx << ") to " << OutputSection.getName()
5115	<< " (" << OutputSection.getIndex() << ")\n");
5116	NewSymbol.st_shndx = OutputSection.getIndex();
5117	NewSymbol.st_value = BD->getOutputAddress();
5118	} else {
5119	// Otherwise just update the section for the symbol.
5120	if (Symbol.st_shndx < ELF::SHN_LORESERVE)
5121	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
5122	}
5123
5124	// Detect local syms in the text section that we didn't update
5125	// and that were preserved by the linker to support relocations against
5126	// .text. Remove them from the symtab.
5127	if (Symbol.getType() == ELF::STT_NOTYPE &&
5128	Symbol.getBinding() == ELF::STB_LOCAL && Symbol.st_size == 0) {
5129	if (BC->getBinaryFunctionContainingAddress(Symbol.st_value,
5130	/*CheckPastEnd=*/false,
5131	/*UseMaxSize=*/true)) {
5132	// Can only delete the symbol if not patching. Such symbols should
5133	// not exist in the dynamic symbol table.
5134	assert(!IsDynSym && "cannot delete symbol");
5135	continue;
5136	}
5137	}
5138	}
5139	}
5140
5141	registerSymbol:
5142	if (IsDynSym)
5143	Write((&Symbol - cantFail(Obj.symbols(&SymTabSection)).begin()) *
5144	sizeof(ELFSymTy),
5145	NewSymbol);
5146	else
5147	Symbols.emplace_back(NewSymbol);
5148	}
5149
5150	if (IsDynSym) {
5151	assert(Symbols.empty());
5152	return;
5153	}
5154
5155	// Add symbols of injected functions
5156	for (BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {
5157	if (Function->isAnonymous())
5158	continue;
5159	ELFSymTy NewSymbol;
5160	BinarySection *OriginSection = Function->getOriginSection();
5161	NewSymbol.st_shndx =
5162	OriginSection
5163	? getNewSectionIndex(OriginSection->getSectionRef().getIndex())
5164	: Function->getCodeSection()->getIndex();
5165	NewSymbol.st_value = Function->getOutputAddress();
5166	NewSymbol.st_name = AddToStrTab(Function->getOneName());
5167	NewSymbol.st_size = Function->getOutputSize();
5168	NewSymbol.st_other = 0;
5169	NewSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
5170	Symbols.emplace_back(NewSymbol);
5171
5172	if (Function->isSplit()) {
5173	assert(Function->getLayout().isHotColdSplit() &&
5174	"Adding symbols based on cold fragment when there are more than "
5175	"2 fragments");
5176	ELFSymTy NewColdSym = NewSymbol;
5177	NewColdSym.setType(ELF::STT_NOTYPE);
5178	SmallVector<char, 256> Buf;
5179	NewColdSym.st_name = AddToStrTab(
5180	Twine(Function->getPrintName()).concat(Suffix: ".cold.0").toStringRef(Out&: Buf));
5181	const FunctionFragment &ColdFF =
5182	Function->getLayout().getFragment(Num: FragmentNum::cold());
5183	NewColdSym.st_value = ColdFF.getAddress();
5184	NewColdSym.st_size = ColdFF.getImageSize();
5185	Symbols.emplace_back(NewColdSym);
5186	}
5187	}
5188
5189	auto AddSymbol = [&](const StringRef &Name, uint64_t Address) {
5190	if (!Address)
5191	return;
5192
5193	ELFSymTy Symbol;
5194	Symbol.st_value = Address;
5195	Symbol.st_shndx = ELF::SHN_ABS;
5196	Symbol.st_name = AddToStrTab(Name);
5197	Symbol.st_size = 0;
5198	Symbol.st_other = 0;
5199	Symbol.setBindingAndType(ELF::STB_WEAK, ELF::STT_NOTYPE);
5200
5201	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
5202	<< Twine::utohexstr(Val: Symbol.st_value) << '\n';
5203
5204	Symbols.emplace_back(Symbol);
5205	};
5206
5207	// Add runtime library start and fini address symbols
5208	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {
5209	AddSymbol("__bolt_runtime_start", RtLibrary->getRuntimeStartAddress());
5210	AddSymbol("__bolt_runtime_fini", RtLibrary->getRuntimeFiniAddress());
5211	}
5212
5213	assert((!NumHotTextSymsUpdated || NumHotTextSymsUpdated == 2) &&
5214	"either none or both __hot_start/__hot_end symbols were expected");
5215	assert((!NumHotDataSymsUpdated || NumHotDataSymsUpdated == 2) &&
5216	"either none or both __hot_data_start/__hot_data_end symbols were "
5217	"expected");
5218
5219	auto AddEmittedSymbol = [&](const StringRef &Name) {
5220	AddSymbol(Name, getNewValueForSymbol(Name));
5221	};
5222
5223	if (opts::HotText && !NumHotTextSymsUpdated) {
5224	AddEmittedSymbol("__hot_start");
5225	AddEmittedSymbol("__hot_end");
5226	}
5227
5228	if (opts::HotData && !NumHotDataSymsUpdated) {
5229	AddEmittedSymbol("__hot_data_start");
5230	AddEmittedSymbol("__hot_data_end");
5231	}
5232
5233	// Put local symbols at the beginning.
5234	llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) {
5235	if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL)
5236	return true;
5237	return false;
5238	});
5239
5240	for (const ELFSymTy &Symbol : Symbols)
5241	Write(0, Symbol);
5242	}
5243
5244	template <typename ELFT>
5245	void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
5246	const ELFFile<ELFT> &Obj = File->getELFFile();
5247	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
5248	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
5249
5250	// Compute a preview of how section indices will change after rewriting, so
5251	// we can properly update the symbol table based on new section indices.
5252	std::vector<uint32_t> NewSectionIndex;
5253	getOutputSections(File, NewSectionIndex);
5254
5255	// Update dynamic symbol table.
5256	const ELFShdrTy *DynSymSection = nullptr;
5257	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
5258	if (Section.sh_type == ELF::SHT_DYNSYM) {
5259	DynSymSection = &Section;
5260	break;
5261	}
5262	}
5263	assert((DynSymSection || BC->IsStaticExecutable) &&
5264	"dynamic symbol table expected");
5265	if (DynSymSection) {
5266	updateELFSymbolTable(
5267	File,
5268	/*IsDynSym=*/true,
5269	*DynSymSection,
5270	NewSectionIndex,
5271	[&](size_t Offset, const ELFSymTy &Sym) {
5272	Out->os().pwrite(Ptr: reinterpret_cast<const char *>(&Sym),
5273	Size: sizeof(ELFSymTy),
5274	Offset: DynSymSection->sh_offset + Offset);
5275	},
5276	[](StringRef) -> size_t { return 0; });
5277	}
5278
5279	if (opts::RemoveSymtab)
5280	return;
5281
5282	// (re)create regular symbol table.
5283	const ELFShdrTy *SymTabSection = nullptr;
5284	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
5285	if (Section.sh_type == ELF::SHT_SYMTAB) {
5286	SymTabSection = &Section;
5287	break;
5288	}
5289	}
5290	if (!SymTabSection) {
5291	BC->errs() << "BOLT-WARNING: no symbol table found\n";
5292	return;
5293	}
5294
5295	const ELFShdrTy *StrTabSection =
5296	cantFail(Obj.getSection(SymTabSection->sh_link));
5297	std::string NewContents;
5298	std::string NewStrTab = std::string(
5299	File->getData().substr(StrTabSection->sh_offset, StrTabSection->sh_size));
5300	StringRef SecName = cantFail(Obj.getSectionName(*SymTabSection));
5301	StringRef StrSecName = cantFail(Obj.getSectionName(*StrTabSection));
5302
5303	NumLocalSymbols = 0;
5304	updateELFSymbolTable(
5305	File,
5306	/*IsDynSym=*/false,
5307	*SymTabSection,
5308	NewSectionIndex,
5309	[&](size_t Offset, const ELFSymTy &Sym) {
5310	if (Sym.getBinding() == ELF::STB_LOCAL)
5311	++NumLocalSymbols;
5312	NewContents.append(s: reinterpret_cast<const char *>(&Sym),
5313	n: sizeof(ELFSymTy));
5314	},
5315	[&](StringRef Str) {
5316	size_t Idx = NewStrTab.size();
5317	NewStrTab.append(str: NameResolver::restore(Name: Str).str());
5318	NewStrTab.append(n: 1, c: '\0');
5319	return Idx;
5320	});
5321
5322	BC->registerOrUpdateNoteSection(Name: SecName,
5323	Data: copyByteArray(Buffer: NewContents),
5324	Size: NewContents.size(),
5325	/*Alignment=*/1,
5326	/*IsReadOnly=*/true,
5327	ELFType: ELF::SHT_SYMTAB);
5328
5329	BC->registerOrUpdateNoteSection(Name: StrSecName,
5330	Data: copyByteArray(Buffer: NewStrTab),
5331	Size: NewStrTab.size(),
5332	/*Alignment=*/1,
5333	/*IsReadOnly=*/true,
5334	ELFType: ELF::SHT_STRTAB);
5335	}
5336
5337	template <typename ELFT>
5338	void RewriteInstance::patchELFAllocatableRelrSection(
5339	ELFObjectFile<ELFT> *File) {
5340	if (!DynamicRelrAddress)
5341	return;
5342
5343	raw_fd_ostream &OS = Out->os();
5344	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
5345	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
5346
5347	auto FixAddend = [&](const BinarySection &Section, const Relocation &Rel,
5348	uint64_t FileOffset) {
5349	// Fix relocation symbol value in place if no static relocation found
5350	// on the same address. We won't check the BF relocations here since it
5351	// is rare case and no optimization is required.
5352	if (Section.getRelocationAt(Offset: Rel.Offset))
5353	return;
5354
5355	// No fixup needed if symbol address was not changed
5356	const uint64_t Addend = getNewFunctionOrDataAddress(OldAddress: Rel.Addend);
5357	if (!Addend)
5358	return;
5359
5360	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Addend), Size: PSize, Offset: FileOffset);
5361	};
5362
5363	// Fill new relative relocation offsets set
5364	std::set<uint64_t> RelOffsets;
5365	for (const BinarySection &Section : BC->allocatableSections()) {
5366	const uint64_t SectionInputAddress = Section.getAddress();
5367	uint64_t SectionAddress = Section.getOutputAddress();
5368	if (!SectionAddress)
5369	SectionAddress = SectionInputAddress;
5370
5371	for (const Relocation &Rel : Section.dynamicRelocations()) {
5372	if (!Rel.isRelative())
5373	continue;
5374
5375	uint64_t RelOffset =
5376	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5377
5378	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5379	assert((RelOffset & 1) == 0 && "Wrong relocation offset");
5380	RelOffsets.emplace(args&: RelOffset);
5381	FixAddend(Section, Rel, RelOffset);
5382	}
5383	}
5384
5385	ErrorOr<BinarySection &> Section =
5386	BC->getSectionForAddress(Address: *DynamicRelrAddress);
5387	assert(Section && "cannot get .relr.dyn section");
5388	assert(Section->isRelr() && "Expected section to be SHT_RELR type");
5389	uint64_t RelrDynOffset = Section->getInputFileOffset();
5390	const uint64_t RelrDynEndOffset = RelrDynOffset + Section->getSize();
5391
5392	auto WriteRelr = [&](uint64_t Value) {
5393	if (RelrDynOffset + DynamicRelrEntrySize > RelrDynEndOffset) {
5394	BC->errs() << "BOLT-ERROR: Offset overflow for relr.dyn section\n";
5395	exit(status: 1);
5396	}
5397
5398	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Value), Size: DynamicRelrEntrySize,
5399	Offset: RelrDynOffset);
5400	RelrDynOffset += DynamicRelrEntrySize;
5401	};
5402
5403	for (auto RelIt = RelOffsets.begin(); RelIt != RelOffsets.end();) {
5404	WriteRelr(*RelIt);
5405	uint64_t Base = *RelIt++ + PSize;
5406	while (1) {
5407	uint64_t Bitmap = 0;
5408	for (; RelIt != RelOffsets.end(); ++RelIt) {
5409	const uint64_t Delta = *RelIt - Base;
5410	if (Delta >= MaxDelta || Delta % PSize)
5411	break;
5412
5413	Bitmap |= (1ULL << (Delta / PSize));
5414	}
5415
5416	if (!Bitmap)
5417	break;
5418
5419	WriteRelr((Bitmap << 1) | 1);
5420	Base += MaxDelta;
5421	}
5422	}
5423
5424	// Fill the rest of the section with empty bitmap value
5425	while (RelrDynOffset != RelrDynEndOffset)
5426	WriteRelr(1);
5427	}
5428
5429	template <typename ELFT>
5430	void
5431	RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile<ELFT> *File) {
5432	using Elf_Rela = typename ELFT::Rela;
5433	raw_fd_ostream &OS = Out->os();
5434	const ELFFile<ELFT> &EF = File->getELFFile();
5435
5436	uint64_t RelDynOffset = 0, RelDynEndOffset = 0;
5437	uint64_t RelPltOffset = 0, RelPltEndOffset = 0;
5438
5439	auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start,
5440	uint64_t &End) {
5441	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
5442	assert(Section && "cannot get relocation section");
5443	Start = Section->getInputFileOffset();
5444	End = Start + Section->getSize();
5445	};
5446
5447	if (!DynamicRelocationsAddress && !PLTRelocationsAddress)
5448	return;
5449
5450	if (DynamicRelocationsAddress)
5451	setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset,
5452	RelDynEndOffset);
5453
5454	if (PLTRelocationsAddress)
5455	setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset,
5456	RelPltEndOffset);
5457
5458	DynamicRelativeRelocationsCount = 0;
5459
5460	auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) {
5461	OS.pwrite(Ptr: reinterpret_cast<const char *>(RelA), Size: sizeof(*RelA), Offset);
5462	Offset += sizeof(*RelA);
5463	};
5464
5465	auto writeRelocations = [&](bool PatchRelative) {
5466	for (BinarySection &Section : BC->allocatableSections()) {
5467	const uint64_t SectionInputAddress = Section.getAddress();
5468	uint64_t SectionAddress = Section.getOutputAddress();
5469	if (!SectionAddress)
5470	SectionAddress = SectionInputAddress;
5471
5472	for (const Relocation &Rel : Section.dynamicRelocations()) {
5473	const bool IsRelative = Rel.isRelative();
5474	if (PatchRelative != IsRelative)
5475	continue;
5476
5477	if (IsRelative)
5478	++DynamicRelativeRelocationsCount;
5479
5480	Elf_Rela NewRelA;
5481	MCSymbol *Symbol = Rel.Symbol;
5482	uint32_t SymbolIdx = 0;
5483	uint64_t Addend = Rel.Addend;
5484	uint64_t RelOffset =
5485	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5486
5487	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5488	if (Rel.Symbol) {
5489	SymbolIdx = getOutputDynamicSymbolIndex(Symbol);
5490	} else {
5491	// Usually this case is used for R_*_(I)RELATIVE relocations
5492	const uint64_t Address = getNewFunctionOrDataAddress(OldAddress: Addend);
5493	if (Address)
5494	Addend = Address;
5495	}
5496
5497	NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL());
5498	NewRelA.r_offset = RelOffset;
5499	NewRelA.r_addend = Addend;
5500
5501	const bool IsJmpRel = IsJmpRelocation.contains(Val: Rel.Type);
5502	uint64_t &Offset = IsJmpRel ? RelPltOffset : RelDynOffset;
5503	const uint64_t &EndOffset =
5504	IsJmpRel ? RelPltEndOffset : RelDynEndOffset;
5505	if (!Offset || !EndOffset) {
5506	BC->errs() << "BOLT-ERROR: Invalid offsets for dynamic relocation\n";
5507	exit(status: 1);
5508	}
5509
5510	if (Offset + sizeof(NewRelA) > EndOffset) {
5511	BC->errs() << "BOLT-ERROR: Offset overflow for dynamic relocation\n";
5512	exit(status: 1);
5513	}
5514
5515	writeRela(&NewRelA, Offset);
5516	}
5517	}
5518	};
5519
5520	// Place R_*_RELATIVE relocations in RELA section if RELR is not presented.
5521	// The dynamic linker expects all R_*_RELATIVE relocations in RELA
5522	// to be emitted first.
5523	if (!DynamicRelrAddress)
5524	writeRelocations(/* PatchRelative */ true);
5525	writeRelocations(/* PatchRelative */ false);
5526
5527	auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) {
5528	if (!Offset)
5529	return;
5530
5531	typename ELFObjectFile<ELFT>::Elf_Rela RelA;
5532	RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL());
5533	RelA.r_offset = 0;
5534	RelA.r_addend = 0;
5535	while (Offset < EndOffset)
5536	writeRela(&RelA, Offset);
5537
5538	assert(Offset == EndOffset && "Unexpected section overflow");
5539	};
5540
5541	// Fill the rest of the sections with R_*_NONE relocations
5542	fillNone(RelDynOffset, RelDynEndOffset);
5543	fillNone(RelPltOffset, RelPltEndOffset);
5544	}
5545
5546	template <typename ELFT>
5547	void RewriteInstance::patchELFGOT(ELFObjectFile<ELFT> *File) {
5548	raw_fd_ostream &OS = Out->os();
5549
5550	SectionRef GOTSection;
5551	for (const SectionRef &Section : File->sections()) {
5552	StringRef SectionName = cantFail(ValOrErr: Section.getName());
5553	if (SectionName == ".got") {
5554	GOTSection = Section;
5555	break;
5556	}
5557	}
5558	if (!GOTSection.getObject()) {
5559	if (!BC->IsStaticExecutable)
5560	BC->errs() << "BOLT-INFO: no .got section found\n";
5561	return;
5562	}
5563
5564	StringRef GOTContents = cantFail(ValOrErr: GOTSection.getContents());
5565	for (const uint64_t *GOTEntry =
5566	reinterpret_cast<const uint64_t *>(GOTContents.data());
5567	GOTEntry < reinterpret_cast<const uint64_t *>(GOTContents.data() +
5568	GOTContents.size());
5569	++GOTEntry) {
5570	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: *GOTEntry)) {
5571	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x"
5572	<< Twine::utohexstr(*GOTEntry) << " with 0x"
5573	<< Twine::utohexstr(NewAddress) << '\n');
5574	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewAddress), Size: sizeof(NewAddress),
5575	Offset: reinterpret_cast<const char *>(GOTEntry) -
5576	File->getData().data());
5577	}
5578	}
5579	}
5580
5581	template <typename ELFT>
5582	void RewriteInstance::patchELFDynamic(ELFObjectFile<ELFT> *File) {
5583	if (BC->IsStaticExecutable)
5584	return;
5585
5586	const ELFFile<ELFT> &Obj = File->getELFFile();
5587	raw_fd_ostream &OS = Out->os();
5588
5589	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5590	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5591
5592	// Locate DYNAMIC by looking through program headers.
5593	uint64_t DynamicOffset = 0;
5594	const Elf_Phdr *DynamicPhdr = nullptr;
5595	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5596	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5597	DynamicOffset = Phdr.p_offset;
5598	DynamicPhdr = &Phdr;
5599	assert(Phdr.p_memsz == Phdr.p_filesz && "dynamic sizes should match");
5600	break;
5601	}
5602	}
5603	assert(DynamicPhdr && "missing dynamic in ELF binary");
5604
5605	bool ZNowSet = false;
5606
5607	// Go through all dynamic entries and patch functions addresses with
5608	// new ones.
5609	typename ELFT::DynRange DynamicEntries =
5610	cantFail(Obj.dynamicEntries(), "error accessing dynamic table");
5611	auto DTB = DynamicEntries.begin();
5612	for (const Elf_Dyn &Dyn : DynamicEntries) {
5613	Elf_Dyn NewDE = Dyn;
5614	bool ShouldPatch = true;
5615	switch (Dyn.d_tag) {
5616	default:
5617	ShouldPatch = false;
5618	break;
5619	case ELF::DT_RELACOUNT:
5620	NewDE.d_un.d_val = DynamicRelativeRelocationsCount;
5621	break;
5622	case ELF::DT_INIT:
5623	case ELF::DT_FINI: {
5624	if (BC->HasRelocations) {
5625	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: Dyn.getPtr())) {
5626	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching dynamic entry of type "
5627	<< Dyn.getTag() << '\n');
5628	NewDE.d_un.d_ptr = NewAddress;
5629	}
5630	}
5631	RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary();
5632	if (RtLibrary && Dyn.getTag() == ELF::DT_FINI) {
5633	if (uint64_t Addr = RtLibrary->getRuntimeFiniAddress())
5634	NewDE.d_un.d_ptr = Addr;
5635	}
5636	if (RtLibrary && Dyn.getTag() == ELF::DT_INIT && !BC->HasInterpHeader) {
5637	if (auto Addr = RtLibrary->getRuntimeStartAddress()) {
5638	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set DT_INIT to 0x"
5639	<< Twine::utohexstr(Addr) << '\n');
5640	NewDE.d_un.d_ptr = Addr;
5641	}
5642	}
5643	break;
5644	}
5645	case ELF::DT_FLAGS:
5646	if (BC->RequiresZNow) {
5647	NewDE.d_un.d_val |= ELF::DF_BIND_NOW;
5648	ZNowSet = true;
5649	}
5650	break;
5651	case ELF::DT_FLAGS_1:
5652	if (BC->RequiresZNow) {
5653	NewDE.d_un.d_val |= ELF::DF_1_NOW;
5654	ZNowSet = true;
5655	}
5656	break;
5657	}
5658	if (ShouldPatch)
5659	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewDE), Size: sizeof(NewDE),
5660	Offset: DynamicOffset + (&Dyn - DTB) * sizeof(Dyn));
5661	}
5662
5663	if (BC->RequiresZNow && !ZNowSet) {
5664	BC->errs()
5665	<< "BOLT-ERROR: output binary requires immediate relocation "
5666	"processing which depends on DT_FLAGS or DT_FLAGS_1 presence in "
5667	".dynamic. Please re-link the binary with -znow.\n";
5668	exit(status: 1);
5669	}
5670	}
5671
5672	template <typename ELFT>
5673	Error RewriteInstance::readELFDynamic(ELFObjectFile<ELFT> *File) {
5674	const ELFFile<ELFT> &Obj = File->getELFFile();
5675
5676	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5677	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5678
5679	// Locate DYNAMIC by looking through program headers.
5680	const Elf_Phdr *DynamicPhdr = nullptr;
5681	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5682	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5683	DynamicPhdr = &Phdr;
5684	break;
5685	}
5686	}
5687
5688	if (!DynamicPhdr) {
5689	BC->outs() << "BOLT-INFO: static input executable detected\n";
5690	// TODO: static PIE executable might have dynamic header
5691	BC->IsStaticExecutable = true;
5692	return Error::success();
5693	}
5694
5695	if (DynamicPhdr->p_memsz != DynamicPhdr->p_filesz)
5696	return createStringError(EC: errc::executable_format_error,
5697	S: "dynamic section sizes should match");
5698
5699	// Go through all dynamic entries to locate entries of interest.
5700	auto DynamicEntriesOrErr = Obj.dynamicEntries();
5701	if (!DynamicEntriesOrErr)
5702	return DynamicEntriesOrErr.takeError();
5703	typename ELFT::DynRange DynamicEntries = DynamicEntriesOrErr.get();
5704
5705	for (const Elf_Dyn &Dyn : DynamicEntries) {
5706	switch (Dyn.d_tag) {
5707	case ELF::DT_INIT:
5708	if (!BC->HasInterpHeader) {
5709	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set start function address\n");
5710	BC->StartFunctionAddress = Dyn.getPtr();
5711	}
5712	break;
5713	case ELF::DT_FINI:
5714	BC->FiniAddress = Dyn.getPtr();
5715	break;
5716	case ELF::DT_FINI_ARRAY:
5717	BC->FiniArrayAddress = Dyn.getPtr();
5718	break;
5719	case ELF::DT_FINI_ARRAYSZ:
5720	BC->FiniArraySize = Dyn.getPtr();
5721	break;
5722	case ELF::DT_RELA:
5723	DynamicRelocationsAddress = Dyn.getPtr();
5724	break;
5725	case ELF::DT_RELASZ:
5726	DynamicRelocationsSize = Dyn.getVal();
5727	break;
5728	case ELF::DT_JMPREL:
5729	PLTRelocationsAddress = Dyn.getPtr();
5730	break;
5731	case ELF::DT_PLTRELSZ:
5732	PLTRelocationsSize = Dyn.getVal();
5733	break;
5734	case ELF::DT_RELACOUNT:
5735	DynamicRelativeRelocationsCount = Dyn.getVal();
5736	break;
5737	case ELF::DT_RELR:
5738	DynamicRelrAddress = Dyn.getPtr();
5739	break;
5740	case ELF::DT_RELRSZ:
5741	DynamicRelrSize = Dyn.getVal();
5742	break;
5743	case ELF::DT_RELRENT:
5744	DynamicRelrEntrySize = Dyn.getVal();
5745	break;
5746	}
5747	}
5748
5749	if (!DynamicRelocationsAddress || !DynamicRelocationsSize) {
5750	DynamicRelocationsAddress.reset();
5751	DynamicRelocationsSize = 0;
5752	}
5753
5754	if (!PLTRelocationsAddress || !PLTRelocationsSize) {
5755	PLTRelocationsAddress.reset();
5756	PLTRelocationsSize = 0;
5757	}
5758
5759	if (!DynamicRelrAddress || !DynamicRelrSize) {
5760	DynamicRelrAddress.reset();
5761	DynamicRelrSize = 0;
5762	} else if (!DynamicRelrEntrySize) {
5763	BC->errs() << "BOLT-ERROR: expected DT_RELRENT to be presented "
5764	<< "in DYNAMIC section\n";
5765	exit(status: 1);
5766	} else if (DynamicRelrSize % DynamicRelrEntrySize) {
5767	BC->errs() << "BOLT-ERROR: expected RELR table size to be divisible "
5768	<< "by RELR entry size\n";
5769	exit(status: 1);
5770	}
5771
5772	return Error::success();
5773	}
5774
5775	uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {
5776	const BinaryFunction *Function = BC->getBinaryFunctionAtAddress(Address: OldAddress);
5777	if (!Function)
5778	return 0;
5779
5780	return Function->getOutputAddress();
5781	}
5782
5783	uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) {
5784	if (uint64_t Function = getNewFunctionAddress(OldAddress))
5785	return Function;
5786
5787	const BinaryData *BD = BC->getBinaryDataAtAddress(Address: OldAddress);
5788	if (BD && BD->isMoved())
5789	return BD->getOutputAddress();
5790
5791	if (const BinaryFunction *BF =
5792	BC->getBinaryFunctionContainingAddress(Address: OldAddress)) {
5793	if (BF->isEmitted()) {
5794	// If OldAddress is the another entry point of
5795	// the function, then BOLT could get the new address.
5796	if (BF->isMultiEntry()) {
5797	for (const BinaryBasicBlock &BB : *BF)
5798	if (BB.isEntryPoint() &&
5799	(BF->getAddress() + BB.getOffset()) == OldAddress)
5800	return BB.getOutputStartAddress();
5801	}
5802	BC->errs() << "BOLT-ERROR: unable to get new address corresponding to "
5803	"input address 0x"
5804	<< Twine::utohexstr(Val: OldAddress) << " in function " << *BF
5805	<< ". Consider adding this function to --skip-funcs=...\n";
5806	exit(status: 1);
5807	}
5808	}
5809
5810	return 0;
5811	}
5812
5813	void RewriteInstance::rewriteFile() {
5814	std::error_code EC;
5815	Out = std::make_unique<ToolOutputFile>(args&: opts::OutputFilename, args&: EC,
5816	args: sys::fs::OF_None);
5817	check_error(EC, Message: "cannot create output executable file");
5818
5819	raw_fd_ostream &OS = Out->os();
5820
5821	// Copy allocatable part of the input.
5822	OS << InputFile->getData().substr(Start: 0, N: FirstNonAllocatableOffset);
5823
5824	auto Streamer = BC->createStreamer(OS);
5825	// Make sure output stream has enough reserved space, otherwise
5826	// pwrite() will fail.
5827	uint64_t Offset = std::max(a: getFileOffsetForAddress(Address: NextAvailableAddress),
5828	b: FirstNonAllocatableOffset);
5829	Offset = OS.seek(off: Offset);
5830	assert((Offset != (uint64_t)-1) && "Error resizing output file");
5831
5832	// Overwrite functions with fixed output address. This is mostly used by
5833	// non-relocation mode, with one exception: injected functions are covered
5834	// here in both modes.
5835	uint64_t CountOverwrittenFunctions = 0;
5836	uint64_t OverwrittenScore = 0;
5837	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
5838	if (Function->getImageAddress() == 0 || Function->getImageSize() == 0)
5839	continue;
5840
5841	assert(Function->getImageSize() <= Function->getMaxSize() &&
5842	"Unexpected large function");
5843
5844	const auto HasAddress = [](const FunctionFragment &FF) {
5845	return FF.empty() ||
5846	(FF.getImageAddress() != 0 && FF.getImageSize() != 0);
5847	};
5848	const bool SplitFragmentsHaveAddress =
5849	llvm::all_of(Range: Function->getLayout().getSplitFragments(), P: HasAddress);
5850	if (Function->isSplit() && !SplitFragmentsHaveAddress) {
5851	const auto HasNoAddress = [](const FunctionFragment &FF) {
5852	return FF.getImageAddress() == 0 && FF.getImageSize() == 0;
5853	};
5854	assert(llvm::all_of(Function->getLayout().getSplitFragments(),
5855	HasNoAddress) &&
5856	"Some split fragments have an address while others do not");
5857	(void)HasNoAddress;
5858	continue;
5859	}
5860
5861	OverwrittenScore += Function->getFunctionScore();
5862	++CountOverwrittenFunctions;
5863
5864	// Overwrite function in the output file.
5865	if (opts::Verbosity >= 2)
5866	BC->outs() << "BOLT: rewriting function \"" << *Function << "\"\n";
5867
5868	OS.pwrite(Ptr: reinterpret_cast<char *>(Function->getImageAddress()),
5869	Size: Function->getImageSize(), Offset: Function->getFileOffset());
5870
5871	// Write nops at the end of the function.
5872	if (Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {
5873	uint64_t Pos = OS.tell();
5874	OS.seek(off: Function->getFileOffset() + Function->getImageSize());
5875	BC->MAB->writeNopData(
5876	OS, Count: Function->getMaxSize() - Function->getImageSize(), STI: &*BC->STI);
5877
5878	OS.seek(off: Pos);
5879	}
5880
5881	if (!Function->isSplit())
5882	continue;
5883
5884	// Write cold part
5885	if (opts::Verbosity >= 2) {
5886	BC->outs() << formatv(Fmt: "BOLT: rewriting function \"{0}\" (split parts)\n",
5887	Vals&: *Function);
5888	}
5889
5890	for (const FunctionFragment &FF :
5891	Function->getLayout().getSplitFragments()) {
5892	OS.pwrite(Ptr: reinterpret_cast<char *>(FF.getImageAddress()),
5893	Size: FF.getImageSize(), Offset: FF.getFileOffset());
5894	}
5895	}
5896
5897	// Print function statistics for non-relocation mode.
5898	if (!BC->HasRelocations) {
5899	BC->outs() << "BOLT: " << CountOverwrittenFunctions << " out of "
5900	<< BC->getBinaryFunctions().size()
5901	<< " functions were overwritten.\n";
5902	if (BC->TotalScore != 0) {
5903	double Coverage = OverwrittenScore / (double)BC->TotalScore * 100.0;
5904	BC->outs() << format(Fmt: "BOLT-INFO: rewritten functions cover %.2lf",
5905	Vals: Coverage)
5906	<< "% of the execution count of simple functions of "
5907	"this binary\n";
5908	}
5909	}
5910
5911	if (BC->HasRelocations && opts::TrapOldCode) {
5912	uint64_t SavedPos = OS.tell();
5913	// Overwrite function body to make sure we never execute these instructions.
5914	for (auto &BFI : BC->getBinaryFunctions()) {
5915	BinaryFunction &BF = BFI.second;
5916	if (!BF.getFileOffset() || !BF.isEmitted())
5917	continue;
5918	OS.seek(off: BF.getFileOffset());
5919	StringRef TrapInstr = BC->MIB->getTrapFillValue();
5920	unsigned NInstr = BF.getMaxSize() / TrapInstr.size();
5921	for (unsigned I = 0; I < NInstr; ++I)
5922	OS.write(Ptr: TrapInstr.data(), Size: TrapInstr.size());
5923	}
5924	OS.seek(off: SavedPos);
5925	}
5926
5927	// Write all allocatable sections - reloc-mode text is written here as well
5928	for (BinarySection &Section : BC->allocatableSections()) {
5929	if (!Section.isFinalized() || !Section.getOutputData()) {
5930	LLVM_DEBUG(if (opts::Verbosity > 1) {
5931	dbgs() << "BOLT-INFO: new section is finalized or !getOutputData, skip "
5932	<< Section.getName() << '\n';
5933	});
5934	continue;
5935	}
5936	if (Section.isLinkOnly()) {
5937	LLVM_DEBUG(if (opts::Verbosity > 1) {
5938	dbgs() << "BOLT-INFO: new section is link only, skip "
5939	<< Section.getName() << '\n';
5940	});
5941	continue;
5942	}
5943
5944	if (opts::Verbosity >= 1)
5945	BC->outs() << "BOLT: writing new section " << Section.getName()
5946	<< "\n data at 0x"
5947	<< Twine::utohexstr(Val: Section.getAllocAddress()) << "\n of size "
5948	<< Section.getOutputSize() << "\n at offset "
5949	<< Section.getOutputFileOffset() << " with content size "
5950	<< Section.getOutputContents().size() << '\n';
5951	OS.seek(off: Section.getOutputFileOffset());
5952	Section.write(OS);
5953	}
5954
5955	for (BinarySection &Section : BC->allocatableSections())
5956	Section.flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
5957	return getNewValueForSymbol(Name: S->getName());
5958	});
5959
5960	// If .eh_frame is present create .eh_frame_hdr.
5961	if (EHFrameSection)
5962	writeEHFrameHeader();
5963
5964	// Add BOLT Addresses Translation maps to allow profile collection to
5965	// happen in the output binary
5966	if (opts::EnableBAT)
5967	addBATSection();
5968
5969	// Patch program header table.
5970	if (!BC->IsLinuxKernel)
5971	patchELFPHDRTable();
5972
5973	// Finalize memory image of section string table.
5974	finalizeSectionStringTable();
5975
5976	// Update symbol tables.
5977	patchELFSymTabs();
5978
5979	if (opts::EnableBAT)
5980	encodeBATSection();
5981
5982	// Copy non-allocatable sections once allocatable part is finished.
5983	rewriteNoteSections();
5984
5985	if (BC->HasRelocations) {
5986	patchELFAllocatableRelaSections();
5987	patchELFAllocatableRelrSection();
5988	patchELFGOT();
5989	}
5990
5991	// Patch dynamic section/segment.
5992	patchELFDynamic();
5993
5994	// Update ELF book-keeping info.
5995	patchELFSectionHeaderTable();
5996
5997	if (opts::PrintSections) {
5998	BC->outs() << "BOLT-INFO: Sections after processing:\n";
5999	BC->printSections(OS&: BC->outs());
6000	}
6001
6002	Out->keep();
6003	EC = sys::fs::setPermissions(
6004	Path: opts::OutputFilename,
6005	Permissions: static_cast<sys::fs::perms>(sys::fs::perms::all_all &
6006	~sys::fs::getUmask()));
6007	check_error(EC, Message: "cannot set permissions of output file");
6008	}
6009
6010	void RewriteInstance::writeEHFrameHeader() {
6011	BinarySection *NewEHFrameSection =
6012	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
6013
6014	// No need to update the header if no new .eh_frame was created.
6015	if (!NewEHFrameSection)
6016	return;
6017
6018	DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true,
6019	NewEHFrameSection->getOutputAddress());
6020	Error E = NewEHFrame.parse(Data: DWARFDataExtractor(
6021	NewEHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(),
6022	BC->AsmInfo->getCodePointerSize()));
6023	check_error(E: std::move(E), Message: "failed to parse EH frame");
6024
6025	uint64_t RelocatedEHFrameAddress = 0;
6026	StringRef RelocatedEHFrameContents;
6027	BinarySection *RelocatedEHFrameSection =
6028	getSection(Name: ".relocated" + getEHFrameSectionName());
6029	if (RelocatedEHFrameSection) {
6030	RelocatedEHFrameAddress = RelocatedEHFrameSection->getOutputAddress();
6031	RelocatedEHFrameContents = RelocatedEHFrameSection->getOutputContents();
6032	}
6033	DWARFDebugFrame RelocatedEHFrame(BC->TheTriple->getArch(), true,
6034	RelocatedEHFrameAddress);
6035	Error Er = RelocatedEHFrame.parse(Data: DWARFDataExtractor(
6036	RelocatedEHFrameContents, BC->AsmInfo->isLittleEndian(),
6037	BC->AsmInfo->getCodePointerSize()));
6038	check_error(E: std::move(Er), Message: "failed to parse EH frame");
6039
6040	LLVM_DEBUG(dbgs() << "BOLT: writing a new " << getEHFrameHdrSectionName()
6041	<< '\n');
6042
6043	// Try to overwrite the original .eh_frame_hdr if the size permits.
6044	uint64_t EHFrameHdrOutputAddress = 0;
6045	uint64_t EHFrameHdrFileOffset = 0;
6046	std::vector<char> NewEHFrameHdr;
6047	BinarySection *OldEHFrameHdrSection = getSection(Name: getEHFrameHdrSectionName());
6048	if (OldEHFrameHdrSection) {
6049	NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
6050	OldEHFrame: RelocatedEHFrame, NewEHFrame, EHFrameHeaderAddress: OldEHFrameHdrSection->getAddress());
6051	if (NewEHFrameHdr.size() <= OldEHFrameHdrSection->getSize()) {
6052	BC->outs() << "BOLT-INFO: rewriting " << getEHFrameHdrSectionName()
6053	<< " in-place\n";
6054	EHFrameHdrOutputAddress = OldEHFrameHdrSection->getAddress();
6055	EHFrameHdrFileOffset = OldEHFrameHdrSection->getInputFileOffset();
6056	} else {
6057	OldEHFrameHdrSection->setOutputName(getOrgSecPrefix() +
6058	getEHFrameHdrSectionName());
6059	OldEHFrameHdrSection = nullptr;
6060	}
6061	}
6062
6063	// If there was not enough space, allocate more memory for .eh_frame_hdr.
6064	if (!OldEHFrameHdrSection) {
6065	NextAvailableAddress =
6066	appendPadding(OS&: Out->os(), Offset: NextAvailableAddress, Alignment: EHFrameHdrAlign);
6067
6068	EHFrameHdrOutputAddress = NextAvailableAddress;
6069	EHFrameHdrFileOffset = getFileOffsetForAddress(Address: NextAvailableAddress);
6070
6071	NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
6072	OldEHFrame: RelocatedEHFrame, NewEHFrame, EHFrameHeaderAddress: EHFrameHdrOutputAddress);
6073
6074	NextAvailableAddress += NewEHFrameHdr.size();
6075	if (!BC->BOLTReserved.empty() &&
6076	(NextAvailableAddress > BC->BOLTReserved.end())) {
6077	BC->errs() << "BOLT-ERROR: unable to fit " << getEHFrameHdrSectionName()
6078	<< " into reserved space\n";
6079	exit(status: 1);
6080	}
6081
6082	// Create a new entry in the section header table.
6083	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
6084	/*IsText=*/false,
6085	/*IsAllocatable=*/true);
6086	BinarySection &EHFrameHdrSec = BC->registerOrUpdateSection(
6087	Name: getNewSecPrefix() + getEHFrameHdrSectionName(), ELFType: ELF::SHT_PROGBITS,
6088	ELFFlags: Flags, Data: nullptr, Size: NewEHFrameHdr.size(), /*Alignment=*/1);
6089	EHFrameHdrSec.setOutputFileOffset(EHFrameHdrFileOffset);
6090	EHFrameHdrSec.setOutputAddress(EHFrameHdrOutputAddress);
6091	EHFrameHdrSec.setOutputName(getEHFrameHdrSectionName());
6092	}
6093
6094	Out->os().seek(off: EHFrameHdrFileOffset);
6095	Out->os().write(Ptr: NewEHFrameHdr.data(), Size: NewEHFrameHdr.size());
6096
6097	// Pad the contents if overwriting in-place.
6098	if (OldEHFrameHdrSection)
6099	Out->os().write_zeros(NumZeros: OldEHFrameHdrSection->getSize() -
6100	NewEHFrameHdr.size());
6101
6102	// Merge new .eh_frame with the relocated original so that gdb can locate all
6103	// FDEs.
6104	if (RelocatedEHFrameSection) {
6105	const uint64_t NewEHFrameSectionSize =
6106	RelocatedEHFrameSection->getOutputAddress() +
6107	RelocatedEHFrameSection->getOutputSize() -
6108	NewEHFrameSection->getOutputAddress();
6109	NewEHFrameSection->updateContents(NewData: NewEHFrameSection->getOutputData(),
6110	NewSize: NewEHFrameSectionSize);
6111	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
6112	}
6113
6114	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of .eh_frame after merge is "
6115	<< NewEHFrameSection->getOutputSize() << '\n');
6116	}
6117
6118	uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) {
6119	auto Value = Linker->lookupSymbolInfo(Name);
6120	if (Value)
6121	return Value->Address;
6122
6123	// Return the original value if we haven't emitted the symbol.
6124	BinaryData *BD = BC->getBinaryDataByName(Name);
6125	if (!BD)
6126	return 0;
6127
6128	return BD->getAddress();
6129	}
6130
6131	uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const {
6132	// Check if it's possibly part of the new segment.
6133	if (NewTextSegmentAddress && Address >= NewTextSegmentAddress)
6134	return Address - NewTextSegmentAddress + NewTextSegmentOffset;
6135
6136	// Find an existing segment that matches the address.
6137	const auto SegmentInfoI = BC->SegmentMapInfo.upper_bound(x: Address);
6138	if (SegmentInfoI == BC->SegmentMapInfo.begin())
6139	return 0;
6140
6141	const SegmentInfo &SegmentInfo = std::prev(x: SegmentInfoI)->second;
6142	if (Address < SegmentInfo.Address ||
6143	Address >= SegmentInfo.Address + SegmentInfo.FileSize)
6144	return 0;
6145
6146	return SegmentInfo.FileOffset + Address - SegmentInfo.Address;
6147	}
6148
6149	bool RewriteInstance::willOverwriteSection(StringRef SectionName) {
6150	if (llvm::is_contained(Range: SectionsToOverwrite, Element: SectionName))
6151	return true;
6152	if (llvm::is_contained(Range&: DebugSectionsToOverwrite, Element: SectionName))
6153	return true;
6154
6155	ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
6156	return Section && Section->isAllocatable() && Section->isFinalized();
6157	}
6158
6159	bool RewriteInstance::isDebugSection(StringRef SectionName) {
6160	if (SectionName.starts_with(Prefix: ".debug_") ||
6161	SectionName.starts_with(Prefix: ".zdebug_") || SectionName == ".gdb_index" ||
6162	SectionName == ".stab" || SectionName == ".stabstr")
6163	return true;
6164
6165	return false;
6166	}
6167