1	//===- bolt/Rewrite/RewriteInstance.cpp - ELF rewriter --------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "bolt/Rewrite/RewriteInstance.h"
10	#include "bolt/Core/AddressMap.h"
11	#include "bolt/Core/BinaryContext.h"
12	#include "bolt/Core/BinaryEmitter.h"
13	#include "bolt/Core/BinaryFunction.h"
14	#include "bolt/Core/DebugData.h"
15	#include "bolt/Core/Exceptions.h"
16	#include "bolt/Core/FunctionLayout.h"
17	#include "bolt/Core/MCPlusBuilder.h"
18	#include "bolt/Core/ParallelUtilities.h"
19	#include "bolt/Core/Relocation.h"
20	#include "bolt/Passes/CacheMetrics.h"
21	#include "bolt/Passes/ReorderFunctions.h"
22	#include "bolt/Profile/BoltAddressTranslation.h"
23	#include "bolt/Profile/DataAggregator.h"
24	#include "bolt/Profile/DataReader.h"
25	#include "bolt/Profile/YAMLProfileReader.h"
26	#include "bolt/Profile/YAMLProfileWriter.h"
27	#include "bolt/Rewrite/BinaryPassManager.h"
28	#include "bolt/Rewrite/DWARFRewriter.h"
29	#include "bolt/Rewrite/ExecutableFileMemoryManager.h"
30	#include "bolt/Rewrite/JITLinkLinker.h"
31	#include "bolt/Rewrite/MetadataRewriters.h"
32	#include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h"
33	#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
34	#include "bolt/Utils/CommandLineOpts.h"
35	#include "bolt/Utils/Utils.h"
36	#include "llvm/ADT/AddressRanges.h"
37	#include "llvm/ADT/STLExtras.h"
38	#include "llvm/DebugInfo/DWARF/DWARFContext.h"
39	#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"
40	#include "llvm/MC/MCAsmBackend.h"
41	#include "llvm/MC/MCAsmInfo.h"
42	#include "llvm/MC/MCAsmLayout.h"
43	#include "llvm/MC/MCDisassembler/MCDisassembler.h"
44	#include "llvm/MC/MCObjectStreamer.h"
45	#include "llvm/MC/MCStreamer.h"
46	#include "llvm/MC/MCSymbol.h"
47	#include "llvm/MC/TargetRegistry.h"
48	#include "llvm/Object/ObjectFile.h"
49	#include "llvm/Support/Alignment.h"
50	#include "llvm/Support/Casting.h"
51	#include "llvm/Support/CommandLine.h"
52	#include "llvm/Support/DataExtractor.h"
53	#include "llvm/Support/Errc.h"
54	#include "llvm/Support/Error.h"
55	#include "llvm/Support/FileSystem.h"
56	#include "llvm/Support/ManagedStatic.h"
57	#include "llvm/Support/Regex.h"
58	#include "llvm/Support/Timer.h"
59	#include "llvm/Support/ToolOutputFile.h"
60	#include "llvm/Support/raw_ostream.h"
61	#include <algorithm>
62	#include <fstream>
63	#include <memory>
64	#include <optional>
65	#include <system_error>
66
67	#undef DEBUG_TYPE
68	#define DEBUG_TYPE "bolt"
69
70	using namespace llvm;
71	using namespace object;
72	using namespace bolt;
73
74	extern cl::opt<uint32_t> X86AlignBranchBoundary;
75	extern cl::opt<bool> X86AlignBranchWithin32BBoundaries;
76
77	namespace opts {
78
79	extern cl::opt<MacroFusionType> AlignMacroOpFusion;
80	extern cl::list<std::string> HotTextMoveSections;
81	extern cl::opt<bool> Hugify;
82	extern cl::opt<bool> Instrument;
83	extern cl::opt<JumpTableSupportLevel> JumpTables;
84	extern cl::opt<bool> KeepNops;
85	extern cl::list<std::string> ReorderData;
86	extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
87	extern cl::opt<bool> TerminalTrap;
88	extern cl::opt<bool> TimeBuild;
89
90	cl::opt<bool> AllowStripped("allow-stripped",
91	cl::desc("allow processing of stripped binaries"),
92	cl::Hidden, cl::cat(BoltCategory));
93
94	static cl::opt<bool> ForceToDataRelocations(
95	"force-data-relocations",
96	cl::desc("force relocations to data sections to always be processed"),
97
98	cl::Hidden, cl::cat(BoltCategory));
99
100	cl::opt<std::string>
101	BoltID("bolt-id",
102	cl::desc("add any string to tag this execution in the "
103	"output binary via bolt info section"),
104	cl::cat(BoltCategory));
105
106	cl::opt<bool> DumpDotAll(
107	"dump-dot-all",
108	cl::desc("dump function CFGs to graphviz format after each stage;"
109	"enable '-print-loops' for color-coded blocks"),
110	cl::Hidden, cl::cat(BoltCategory));
111
112	static cl::list<std::string>
113	ForceFunctionNames("funcs",
114	cl::CommaSeparated,
115	cl::desc("limit optimizations to functions from the list"),
116	cl::value_desc("func1,func2,func3,..."),
117	cl::Hidden,
118	cl::cat(BoltCategory));
119
120	static cl::opt<std::string>
121	FunctionNamesFile("funcs-file",
122	cl::desc("file with list of functions to optimize"),
123	cl::Hidden,
124	cl::cat(BoltCategory));
125
126	static cl::list<std::string> ForceFunctionNamesNR(
127	"funcs-no-regex", cl::CommaSeparated,
128	cl::desc("limit optimizations to functions from the list (non-regex)"),
129	cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));
130
131	static cl::opt<std::string> FunctionNamesFileNR(
132	"funcs-file-no-regex",
133	cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,
134	cl::cat(BoltCategory));
135
136	cl::opt<bool>
137	KeepTmp("keep-tmp",
138	cl::desc("preserve intermediate .o file"),
139	cl::Hidden,
140	cl::cat(BoltCategory));
141
142	cl::opt<bool> Lite("lite", cl::desc("skip processing of cold functions"),
143	cl::cat(BoltCategory));
144
145	static cl::opt<unsigned>
146	LiteThresholdPct("lite-threshold-pct",
147	cl::desc("threshold (in percent) for selecting functions to process in lite "
148	"mode. Higher threshold means fewer functions to process. E.g "
149	"threshold of 90 means only top 10 percent of functions with "
150	"profile will be processed."),
151	cl::init(Val: 0),
152	cl::ZeroOrMore,
153	cl::Hidden,
154	cl::cat(BoltOptCategory));
155
156	static cl::opt<unsigned> LiteThresholdCount(
157	"lite-threshold-count",
158	cl::desc("similar to '-lite-threshold-pct' but specify threshold using "
159	"absolute function call count. I.e. limit processing to functions "
160	"executed at least the specified number of times."),
161	cl::init(Val: 0), cl::Hidden, cl::cat(BoltOptCategory));
162
163	static cl::opt<unsigned>
164	MaxFunctions("max-funcs",
165	cl::desc("maximum number of functions to process"), cl::Hidden,
166	cl::cat(BoltCategory));
167
168	static cl::opt<unsigned> MaxDataRelocations(
169	"max-data-relocations",
170	cl::desc("maximum number of data relocations to process"), cl::Hidden,
171	cl::cat(BoltCategory));
172
173	cl::opt<bool> PrintAll("print-all",
174	cl::desc("print functions after each stage"), cl::Hidden,
175	cl::cat(BoltCategory));
176
177	cl::opt<bool> PrintProfile("print-profile",
178	cl::desc("print functions after attaching profile"),
179	cl::Hidden, cl::cat(BoltCategory));
180
181	cl::opt<bool> PrintCFG("print-cfg",
182	cl::desc("print functions after CFG construction"),
183	cl::Hidden, cl::cat(BoltCategory));
184
185	cl::opt<bool> PrintDisasm("print-disasm",
186	cl::desc("print function after disassembly"),
187	cl::Hidden, cl::cat(BoltCategory));
188
189	static cl::opt<bool>
190	PrintGlobals("print-globals",
191	cl::desc("print global symbols after disassembly"), cl::Hidden,
192	cl::cat(BoltCategory));
193
194	extern cl::opt<bool> PrintSections;
195
196	static cl::opt<bool> PrintLoopInfo("print-loops",
197	cl::desc("print loop related information"),
198	cl::Hidden, cl::cat(BoltCategory));
199
200	static cl::opt<cl::boolOrDefault> RelocationMode(
201	"relocs", cl::desc("use relocations in the binary (default=autodetect)"),
202	cl::cat(BoltCategory));
203
204	extern cl::opt<std::string> SaveProfile;
205
206	static cl::list<std::string>
207	SkipFunctionNames("skip-funcs",
208	cl::CommaSeparated,
209	cl::desc("list of functions to skip"),
210	cl::value_desc("func1,func2,func3,..."),
211	cl::Hidden,
212	cl::cat(BoltCategory));
213
214	static cl::opt<std::string>
215	SkipFunctionNamesFile("skip-funcs-file",
216	cl::desc("file with list of functions to skip"),
217	cl::Hidden,
218	cl::cat(BoltCategory));
219
220	cl::opt<bool>
221	TrapOldCode("trap-old-code",
222	cl::desc("insert traps in old function bodies (relocation mode)"),
223	cl::Hidden,
224	cl::cat(BoltCategory));
225
226	static cl::opt<std::string> DWPPathName("dwp",
227	cl::desc("Path and name to DWP file."),
228	cl::Hidden, cl::init(Val: ""),
229	cl::cat(BoltCategory));
230
231	static cl::opt<bool>
232	UseGnuStack("use-gnu-stack",
233	cl::desc("use GNU_STACK program header for new segment (workaround for "
234	"issues with strip/objcopy)"),
235	cl::ZeroOrMore,
236	cl::cat(BoltCategory));
237
238	static cl::opt<bool>
239	TimeRewrite("time-rewrite",
240	cl::desc("print time spent in rewriting passes"), cl::Hidden,
241	cl::cat(BoltCategory));
242
243	static cl::opt<bool>
244	SequentialDisassembly("sequential-disassembly",
245	cl::desc("performs disassembly sequentially"),
246	cl::init(Val: false),
247	cl::cat(BoltOptCategory));
248
249	static cl::opt<bool> WriteBoltInfoSection(
250	"bolt-info", cl::desc("write bolt info section in the output binary"),
251	cl::init(Val: true), cl::Hidden, cl::cat(BoltOutputCategory));
252
253	} // namespace opts
254
255	// FIXME: implement a better way to mark sections for replacement.
256	constexpr const char *RewriteInstance::SectionsToOverwrite[];
257	std::vector<std::string> RewriteInstance::DebugSectionsToOverwrite = {
258	".debug_abbrev", ".debug_aranges", ".debug_line", ".debug_line_str",
259	".debug_loc", ".debug_loclists", ".debug_ranges", ".debug_rnglists",
260	".gdb_index", ".debug_addr", ".debug_abbrev", ".debug_info",
261	".debug_types", ".pseudo_probe"};
262
263	const char RewriteInstance::TimerGroupName[] = "rewrite";
264	const char RewriteInstance::TimerGroupDesc[] = "Rewrite passes";
265
266	namespace llvm {
267	namespace bolt {
268
269	extern const char *BoltRevision;
270
271	// Weird location for createMCPlusBuilder, but this is here to avoid a
272	// cyclic dependency of libCore (its natural place) and libTarget. libRewrite
273	// can depend on libTarget, but not libCore. Since libRewrite is the only
274	// user of this function, we define it here.
275	MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
276	const MCInstrAnalysis *Analysis,
277	const MCInstrInfo *Info,
278	const MCRegisterInfo *RegInfo,
279	const MCSubtargetInfo *STI) {
280	#ifdef X86_AVAILABLE
281	if (Arch == Triple::x86_64)
282	return createX86MCPlusBuilder(Analysis, Info, RegInfo, STI);
283	#endif
284
285	#ifdef AARCH64_AVAILABLE
286	if (Arch == Triple::aarch64)
287	return createAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);
288	#endif
289
290	#ifdef RISCV_AVAILABLE
291	if (Arch == Triple::riscv64)
292	return createRISCVMCPlusBuilder(Analysis, Info, RegInfo, STI);
293	#endif
294
295	llvm_unreachable("architecture unsupported by MCPlusBuilder");
296	}
297
298	} // namespace bolt
299	} // namespace llvm
300
301	using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;
302
303	namespace {
304
305	bool refersToReorderedSection(ErrorOr<BinarySection &> Section) {
306	return llvm::any_of(Range&: opts::ReorderData, P: [&](const std::string &SectionName) {
307	return Section && Section->getName() == SectionName;
308	});
309	}
310
311	} // anonymous namespace
312
313	Expected<std::unique_ptr<RewriteInstance>>
314	RewriteInstance::create(ELFObjectFileBase *File, const int Argc,
315	const char *const *Argv, StringRef ToolPath,
316	raw_ostream &Stdout, raw_ostream &Stderr) {
317	Error Err = Error::success();
318	auto RI = std::make_unique<RewriteInstance>(args&: File, args: Argc, args&: Argv, args&: ToolPath,
319	args&: Stdout, args&: Stderr, args&: Err);
320	if (Err)
321	return std::move(Err);
322	return std::move(RI);
323	}
324
325	RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,
326	const char *const *Argv, StringRef ToolPath,
327	raw_ostream &Stdout, raw_ostream &Stderr,
328	Error &Err)
329	: InputFile(File), Argc(Argc), Argv(Argv), ToolPath(ToolPath),
330	SHStrTab(StringTableBuilder::ELF) {
331	ErrorAsOutParameter EAO(&Err);
332	auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(Val: InputFile);
333	if (!ELF64LEFile) {
334	Err = createStringError(EC: errc::not_supported,
335	Msg: "Only 64-bit LE ELF binaries are supported");
336	return;
337	}
338
339	bool IsPIC = false;
340	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
341	if (Obj.getHeader().e_type != ELF::ET_EXEC) {
342	Stdout << "BOLT-INFO: shared object or position-independent executable "
343	"detected\n";
344	IsPIC = true;
345	}
346
347	// Make sure we don't miss any output on core dumps.
348	Stdout.SetUnbuffered();
349	Stderr.SetUnbuffered();
350	LLVM_DEBUG(dbgs().SetUnbuffered());
351
352	// Read RISCV subtarget features from input file
353	std::unique_ptr<SubtargetFeatures> Features;
354	Triple TheTriple = File->makeTriple();
355	if (TheTriple.getArch() == llvm::Triple::riscv64) {
356	Expected<SubtargetFeatures> FeaturesOrErr = File->getFeatures();
357	if (auto E = FeaturesOrErr.takeError()) {
358	Err = std::move(E);
359	return;
360	} else {
361	Features.reset(p: new SubtargetFeatures(*FeaturesOrErr));
362	}
363	}
364
365	auto BCOrErr = BinaryContext::createBinaryContext(
366	TheTriple, InputFileName: File->getFileName(), Features: Features.get(), IsPIC,
367	DwCtx: DWARFContext::create(Obj: *File, RelocAction: DWARFContext::ProcessDebugRelocations::Ignore,
368	L: nullptr, DWPName: opts::DWPPathName,
369	RecoverableErrorHandler: WithColor::defaultErrorHandler,
370	WarningHandler: WithColor::defaultWarningHandler),
371	Logger: JournalingStreams{.Out: Stdout, .Err: Stderr});
372	if (Error E = BCOrErr.takeError()) {
373	Err = std::move(E);
374	return;
375	}
376	BC = std::move(BCOrErr.get());
377	BC->initializeTarget(TargetBuilder: std::unique_ptr<MCPlusBuilder>(
378	createMCPlusBuilder(Arch: BC->TheTriple->getArch(), Analysis: BC->MIA.get(),
379	Info: BC->MII.get(), RegInfo: BC->MRI.get(), STI: BC->STI.get())));
380
381	BAT = std::make_unique<BoltAddressTranslation>();
382
383	if (opts::UpdateDebugSections)
384	DebugInfoRewriter = std::make_unique<DWARFRewriter>(args&: *BC);
385
386	if (opts::Instrument)
387	BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
388	else if (opts::Hugify)
389	BC->setRuntimeLibrary(std::make_unique<HugifyRuntimeLibrary>());
390	}
391
392	RewriteInstance::~RewriteInstance() {}
393
394	Error RewriteInstance::setProfile(StringRef Filename) {
395	if (!sys::fs::exists(Path: Filename))
396	return errorCodeToError(EC: make_error_code(E: errc::no_such_file_or_directory));
397
398	if (ProfileReader) {
399	// Already exists
400	return make_error<StringError>(Args: Twine("multiple profiles specified: ") +
401	ProfileReader->getFilename() + " and " +
402	Filename,
403	Args: inconvertibleErrorCode());
404	}
405
406	// Spawn a profile reader based on file contents.
407	if (DataAggregator::checkPerfDataMagic(FileName: Filename))
408	ProfileReader = std::make_unique<DataAggregator>(args&: Filename);
409	else if (YAMLProfileReader::isYAML(Filename))
410	ProfileReader = std::make_unique<YAMLProfileReader>(args&: Filename);
411	else
412	ProfileReader = std::make_unique<DataReader>(args&: Filename);
413
414	return Error::success();
415	}
416
417	/// Return true if the function \p BF should be disassembled.
418	static bool shouldDisassemble(const BinaryFunction &BF) {
419	if (BF.isPseudo())
420	return false;
421
422	if (opts::processAllFunctions())
423	return true;
424
425	return !BF.isIgnored();
426	}
427
428	// Return if a section stored in the image falls into a segment address space.
429	// If not, Set \p Overlap to true if there's a partial overlap.
430	template <class ELFT>
431	static bool checkOffsets(const typename ELFT::Phdr &Phdr,
432	const typename ELFT::Shdr &Sec, bool &Overlap) {
433	// SHT_NOBITS sections don't need to have an offset inside the segment.
434	if (Sec.sh_type == ELF::SHT_NOBITS)
435	return true;
436
437	// Only non-empty sections can be at the end of a segment.
438	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
439	AddressRange SectionAddressRange((uint64_t)Sec.sh_offset,
440	Sec.sh_offset + SectionSize);
441	AddressRange SegmentAddressRange(Phdr.p_offset,
442	Phdr.p_offset + Phdr.p_filesz);
443	if (SegmentAddressRange.contains(R: SectionAddressRange))
444	return true;
445
446	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
447	return false;
448	}
449
450	// Check that an allocatable section belongs to a virtual address
451	// space of a segment.
452	template <class ELFT>
453	static bool checkVMA(const typename ELFT::Phdr &Phdr,
454	const typename ELFT::Shdr &Sec, bool &Overlap) {
455	// Only non-empty sections can be at the end of a segment.
456	uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;
457	AddressRange SectionAddressRange((uint64_t)Sec.sh_addr,
458	Sec.sh_addr + SectionSize);
459	AddressRange SegmentAddressRange(Phdr.p_vaddr, Phdr.p_vaddr + Phdr.p_memsz);
460
461	if (SegmentAddressRange.contains(R: SectionAddressRange))
462	return true;
463	Overlap = SegmentAddressRange.intersects(R: SectionAddressRange);
464	return false;
465	}
466
467	void RewriteInstance::markGnuRelroSections() {
468	using ELFT = ELF64LE;
469	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
470	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
471	const ELFFile<ELFT> &Obj = ELF64LEFile->getELFFile();
472
473	auto handleSection = [&](const ELFT::Phdr &Phdr, SectionRef SecRef) {
474	BinarySection *BinarySection = BC->getSectionForSectionRef(Section: SecRef);
475	// If the section is non-allocatable, ignore it for GNU_RELRO purposes:
476	// it can't be made read-only after runtime relocations processing.
477	if (!BinarySection || !BinarySection->isAllocatable())
478	return;
479	const ELFShdrTy *Sec = cantFail(ValOrErr: Obj.getSection(Index: SecRef.getIndex()));
480	bool ImageOverlap{false}, VMAOverlap{false};
481	bool ImageContains = checkOffsets<ELFT>(Phdr, Sec: *Sec, Overlap&: ImageOverlap);
482	bool VMAContains = checkVMA<ELFT>(Phdr, Sec: *Sec, Overlap&: VMAOverlap);
483	if (ImageOverlap) {
484	if (opts::Verbosity >= 1)
485	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial file offset "
486	<< "overlap with section " << BinarySection->getName()
487	<< '\n';
488	return;
489	}
490	if (VMAOverlap) {
491	if (opts::Verbosity >= 1)
492	BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial VMA overlap "
493	<< "with section " << BinarySection->getName() << '\n';
494	return;
495	}
496	if (!ImageContains || !VMAContains)
497	return;
498	BinarySection->setRelro();
499	if (opts::Verbosity >= 1)
500	BC->outs() << "BOLT-INFO: marking " << BinarySection->getName()
501	<< " as GNU_RELRO\n";
502	};
503
504	for (const ELFT::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers()))
505	if (Phdr.p_type == ELF::PT_GNU_RELRO)
506	for (SectionRef SecRef : InputFile->sections())
507	handleSection(Phdr, SecRef);
508	}
509
510	Error RewriteInstance::discoverStorage() {
511	NamedRegionTimer T("discoverStorage", "discover storage", TimerGroupName,
512	TimerGroupDesc, opts::TimeRewrite);
513
514	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
515	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
516
517	BC->StartFunctionAddress = Obj.getHeader().e_entry;
518
519	NextAvailableAddress = 0;
520	uint64_t NextAvailableOffset = 0;
521	Expected<ELF64LE::PhdrRange> PHsOrErr = Obj.program_headers();
522	if (Error E = PHsOrErr.takeError())
523	return E;
524
525	ELF64LE::PhdrRange PHs = PHsOrErr.get();
526	for (const ELF64LE::Phdr &Phdr : PHs) {
527	switch (Phdr.p_type) {
528	case ELF::PT_LOAD:
529	BC->FirstAllocAddress = std::min(a: BC->FirstAllocAddress,
530	b: static_cast<uint64_t>(Phdr.p_vaddr));
531	NextAvailableAddress = std::max(a: NextAvailableAddress,
532	b: Phdr.p_vaddr + Phdr.p_memsz);
533	NextAvailableOffset = std::max(a: NextAvailableOffset,
534	b: Phdr.p_offset + Phdr.p_filesz);
535
536	BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{.Address: Phdr.p_vaddr,
537	.Size: Phdr.p_memsz,
538	.FileOffset: Phdr.p_offset,
539	.FileSize: Phdr.p_filesz,
540	.Alignment: Phdr.p_align};
541	if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
542	Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
543	BC->IsLinuxKernel = true;
544	break;
545	case ELF::PT_INTERP:
546	BC->HasInterpHeader = true;
547	break;
548	}
549	}
550
551	if (BC->IsLinuxKernel)
552	BC->outs() << "BOLT-INFO: Linux kernel binary detected\n";
553
554	for (const SectionRef &Section : InputFile->sections()) {
555	Expected<StringRef> SectionNameOrErr = Section.getName();
556	if (Error E = SectionNameOrErr.takeError())
557	return E;
558	StringRef SectionName = SectionNameOrErr.get();
559	if (SectionName == BC->getMainCodeSectionName()) {
560	BC->OldTextSectionAddress = Section.getAddress();
561	BC->OldTextSectionSize = Section.getSize();
562
563	Expected<StringRef> SectionContentsOrErr = Section.getContents();
564	if (Error E = SectionContentsOrErr.takeError())
565	return E;
566	StringRef SectionContents = SectionContentsOrErr.get();
567	BC->OldTextSectionOffset =
568	SectionContents.data() - InputFile->getData().data();
569	}
570
571	if (!opts::HeatmapMode &&
572	!(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&
573	(SectionName.starts_with(Prefix: getOrgSecPrefix()) ||
574	SectionName == getBOLTTextSectionName()))
575	return createStringError(
576	EC: errc::function_not_supported,
577	Msg: "BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize");
578	}
579
580	if (!NextAvailableAddress || !NextAvailableOffset)
581	return createStringError(EC: errc::executable_format_error,
582	Msg: "no PT_LOAD pheader seen");
583
584	BC->outs() << "BOLT-INFO: first alloc address is 0x"
585	<< Twine::utohexstr(Val: BC->FirstAllocAddress) << '\n';
586
587	FirstNonAllocatableOffset = NextAvailableOffset;
588
589	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->PageAlign);
590	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: BC->PageAlign);
591
592	// Hugify: Additional huge page from left side due to
593	// weird ASLR mapping addresses (4KB aligned)
594	if (opts::Hugify && !BC->HasFixedLoadAddress)
595	NextAvailableAddress += BC->PageAlign;
596
597	if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
598	// This is where the black magic happens. Creating PHDR table in a segment
599	// other than that containing ELF header is tricky. Some loaders and/or
600	// parts of loaders will apply e_phoff from ELF header assuming both are in
601	// the same segment, while others will do the proper calculation.
602	// We create the new PHDR table in such a way that both of the methods
603	// of loading and locating the table work. There's a slight file size
604	// overhead because of that.
605	//
606	// NB: bfd's strip command cannot do the above and will corrupt the
607	// binary during the process of stripping non-allocatable sections.
608	if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress)
609	NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress;
610	else
611	NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress;
612
613	assert(NextAvailableOffset ==
614	NextAvailableAddress - BC->FirstAllocAddress &&
615	"PHDR table address calculation error");
616
617	BC->outs() << "BOLT-INFO: creating new program header table at address 0x"
618	<< Twine::utohexstr(Val: NextAvailableAddress) << ", offset 0x"
619	<< Twine::utohexstr(Val: NextAvailableOffset) << '\n';
620
621	PHDRTableAddress = NextAvailableAddress;
622	PHDRTableOffset = NextAvailableOffset;
623
624	// Reserve space for 3 extra pheaders.
625	unsigned Phnum = Obj.getHeader().e_phnum;
626	Phnum += 3;
627
628	NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);
629	NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);
630	}
631
632	// Align at cache line.
633	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 64);
634	NextAvailableOffset = alignTo(Value: NextAvailableOffset, Align: 64);
635
636	NewTextSegmentAddress = NextAvailableAddress;
637	NewTextSegmentOffset = NextAvailableOffset;
638	BC->LayoutStartAddress = NextAvailableAddress;
639
640	// Tools such as objcopy can strip section contents but leave header
641	// entries. Check that at least .text is mapped in the file.
642	if (!getFileOffsetForAddress(Address: BC->OldTextSectionAddress))
643	return createStringError(EC: errc::executable_format_error,
644	Msg: "BOLT-ERROR: input binary is not a valid ELF "
645	"executable as its text section is not "
646	"mapped to a valid segment");
647	return Error::success();
648	}
649
650	void RewriteInstance::parseBuildID() {
651	if (!BuildIDSection)
652	return;
653
654	StringRef Buf = BuildIDSection->getContents();
655
656	// Reading notes section (see Portable Formats Specification, Version 1.1,
657	// pg 2-5, section "Note Section").
658	DataExtractor DE =
659	DataExtractor(Buf,
660	/*IsLittleEndian=*/true, InputFile->getBytesInAddress());
661	uint64_t Offset = 0;
662	if (!DE.isValidOffset(offset: Offset))
663	return;
664	uint32_t NameSz = DE.getU32(offset_ptr: &Offset);
665	if (!DE.isValidOffset(offset: Offset))
666	return;
667	uint32_t DescSz = DE.getU32(offset_ptr: &Offset);
668	if (!DE.isValidOffset(offset: Offset))
669	return;
670	uint32_t Type = DE.getU32(offset_ptr: &Offset);
671
672	LLVM_DEBUG(dbgs() << "NameSz = " << NameSz << "; DescSz = " << DescSz
673	<< "; Type = " << Type << "\n");
674
675	// Type 3 is a GNU build-id note section
676	if (Type != 3)
677	return;
678
679	StringRef Name = Buf.slice(Start: Offset, End: Offset + NameSz);
680	Offset = alignTo(Value: Offset + NameSz, Align: 4);
681	if (Name.substr(Start: 0, N: 3) != "GNU")
682	return;
683
684	BuildID = Buf.slice(Start: Offset, End: Offset + DescSz);
685	}
686
687	std::optional<std::string> RewriteInstance::getPrintableBuildID() const {
688	if (BuildID.empty())
689	return std::nullopt;
690
691	std::string Str;
692	raw_string_ostream OS(Str);
693	const unsigned char *CharIter = BuildID.bytes_begin();
694	while (CharIter != BuildID.bytes_end()) {
695	if (*CharIter < 0x10)
696	OS << "0";
697	OS << Twine::utohexstr(Val: *CharIter);
698	++CharIter;
699	}
700	return OS.str();
701	}
702
703	void RewriteInstance::patchBuildID() {
704	raw_fd_ostream &OS = Out->os();
705
706	if (BuildID.empty())
707	return;
708
709	size_t IDOffset = BuildIDSection->getContents().rfind(Str: BuildID);
710	assert(IDOffset != StringRef::npos && "failed to patch build-id");
711
712	uint64_t FileOffset = getFileOffsetForAddress(Address: BuildIDSection->getAddress());
713	if (!FileOffset) {
714	BC->errs()
715	<< "BOLT-WARNING: Non-allocatable build-id will not be updated.\n";
716	return;
717	}
718
719	char LastIDByte = BuildID[BuildID.size() - 1];
720	LastIDByte ^= 1;
721	OS.pwrite(Ptr: &LastIDByte, Size: 1, Offset: FileOffset + IDOffset + BuildID.size() - 1);
722
723	BC->outs() << "BOLT-INFO: patched build-id (flipped last bit)\n";
724	}
725
726	Error RewriteInstance::run() {
727	assert(BC && "failed to create a binary context");
728
729	BC->outs() << "BOLT-INFO: Target architecture: "
730	<< Triple::getArchTypeName(
731	Kind: (llvm::Triple::ArchType)InputFile->getArch())
732	<< "\n";
733	BC->outs() << "BOLT-INFO: BOLT version: " << BoltRevision << "\n";
734
735	if (Error E = discoverStorage())
736	return E;
737	if (Error E = readSpecialSections())
738	return E;
739	adjustCommandLineOptions();
740	discoverFileObjects();
741
742	if (opts::Instrument && !BC->IsStaticExecutable)
743	if (Error E = discoverRtFiniAddress())
744	return E;
745
746	preprocessProfileData();
747
748	// Skip disassembling if we have a translation table and we are running an
749	// aggregation job.
750	if (opts::AggregateOnly && BAT->enabledFor(InputFile)) {
751	// YAML profile in BAT mode requires CFG for .bolt.org.text functions
752	if (!opts::SaveProfile.empty() ||
753	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML) {
754	selectFunctionsToProcess();
755	disassembleFunctions();
756	buildFunctionsCFG();
757	}
758	processProfileData();
759	return Error::success();
760	}
761
762	selectFunctionsToProcess();
763
764	readDebugInfo();
765
766	disassembleFunctions();
767
768	processMetadataPreCFG();
769
770	buildFunctionsCFG();
771
772	processProfileData();
773
774	// Save input binary metadata if BAT section needs to be emitted
775	if (opts::EnableBAT)
776	BAT->saveMetadata(BC&: *BC);
777
778	postProcessFunctions();
779
780	processMetadataPostCFG();
781
782	if (opts::DiffOnly)
783	return Error::success();
784
785	preregisterSections();
786
787	runOptimizationPasses();
788
789	finalizeMetadataPreEmit();
790
791	emitAndLink();
792
793	updateMetadata();
794
795	if (opts::Instrument && !BC->IsStaticExecutable)
796	updateRtFiniReloc();
797
798	if (opts::OutputFilename == "/dev/null") {
799	BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n";
800	return Error::success();
801	} else if (BC->IsLinuxKernel) {
802	BC->errs() << "BOLT-WARNING: Linux kernel support is experimental\n";
803	}
804
805	// Rewrite allocatable contents and copy non-allocatable parts with mods.
806	rewriteFile();
807	return Error::success();
808	}
809
810	void RewriteInstance::discoverFileObjects() {
811	NamedRegionTimer T("discoverFileObjects", "discover file objects",
812	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
813
814	// For local symbols we want to keep track of associated FILE symbol name for
815	// disambiguation by combined name.
816	StringRef FileSymbolName;
817	bool SeenFileName = false;
818	struct SymbolRefHash {
819	size_t operator()(SymbolRef const &S) const {
820	return std::hash<decltype(DataRefImpl::p)>{}(S.getRawDataRefImpl().p);
821	}
822	};
823	std::unordered_map<SymbolRef, StringRef, SymbolRefHash> SymbolToFileName;
824	for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
825	Expected<StringRef> NameOrError = Symbol.getName();
826	if (NameOrError && NameOrError->starts_with(Prefix: "__asan_init")) {
827	BC->errs()
828	<< "BOLT-ERROR: input file was compiled or linked with sanitizer "
829	"support. Cannot optimize.\n";
830	exit(status: 1);
831	}
832	if (NameOrError && NameOrError->starts_with(Prefix: "__llvm_coverage_mapping")) {
833	BC->errs()
834	<< "BOLT-ERROR: input file was compiled or linked with coverage "
835	"support. Cannot optimize.\n";
836	exit(status: 1);
837	}
838
839	if (cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Undefined)
840	continue;
841
842	if (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_File) {
843	StringRef Name =
844	cantFail(ValOrErr: std::move(NameOrError), Msg: "cannot get symbol name for file");
845	// Ignore Clang LTO artificial FILE symbol as it is not always generated,
846	// and this uncertainty is causing havoc in function name matching.
847	if (Name == "ld-temp.o")
848	continue;
849	FileSymbolName = Name;
850	SeenFileName = true;
851	continue;
852	}
853	if (!FileSymbolName.empty() &&
854	!(cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Global))
855	SymbolToFileName[Symbol] = FileSymbolName;
856	}
857
858	// Sort symbols in the file by value. Ignore symbols from non-allocatable
859	// sections. We memoize getAddress(), as it has rather high overhead.
860	struct SymbolInfo {
861	uint64_t Address;
862	SymbolRef Symbol;
863	};
864	std::vector<SymbolInfo> SortedSymbols;
865	auto isSymbolInMemory = [this](const SymbolRef &Sym) {
866	if (cantFail(ValOrErr: Sym.getType()) == SymbolRef::ST_File)
867	return false;
868	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Absolute)
869	return true;
870	if (cantFail(ValOrErr: Sym.getFlags()) & SymbolRef::SF_Undefined)
871	return false;
872	BinarySection Section(*BC, *cantFail(ValOrErr: Sym.getSection()));
873	return Section.isAllocatable();
874	};
875	for (const SymbolRef &Symbol : InputFile->symbols())
876	if (isSymbolInMemory(Symbol))
877	SortedSymbols.push_back(x: {.Address: cantFail(ValOrErr: Symbol.getAddress()), .Symbol: Symbol});
878
879	auto CompareSymbols = [this](const SymbolInfo &A, const SymbolInfo &B) {
880	if (A.Address != B.Address)
881	return A.Address < B.Address;
882
883	const bool AMarker = BC->isMarker(Symbol: A.Symbol);
884	const bool BMarker = BC->isMarker(Symbol: B.Symbol);
885	if (AMarker || BMarker) {
886	return AMarker && !BMarker;
887	}
888
889	const auto AType = cantFail(ValOrErr: A.Symbol.getType());
890	const auto BType = cantFail(ValOrErr: B.Symbol.getType());
891	if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function)
892	return true;
893	if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug)
894	return true;
895
896	return false;
897	};
898	llvm::stable_sort(Range&: SortedSymbols, C: CompareSymbols);
899
900	auto LastSymbol = SortedSymbols.end();
901	if (!SortedSymbols.empty())
902	--LastSymbol;
903
904	// For aarch64, the ABI defines mapping symbols so we identify data in the
905	// code section (see IHI0056B). $d identifies data contents.
906	// Compilers usually merge multiple data objects in a single $d-$x interval,
907	// but we need every data object to be marked with $d. Because of that we
908	// create a vector of MarkerSyms with all locations of data objects.
909
910	struct MarkerSym {
911	uint64_t Address;
912	MarkerSymType Type;
913	};
914
915	std::vector<MarkerSym> SortedMarkerSymbols;
916	auto addExtraDataMarkerPerSymbol = [&]() {
917	bool IsData = false;
918	uint64_t LastAddr = 0;
919	for (const auto &SymInfo : SortedSymbols) {
920	if (LastAddr == SymInfo.Address) // don't repeat markers
921	continue;
922
923	MarkerSymType MarkerType = BC->getMarkerType(Symbol: SymInfo.Symbol);
924	if (MarkerType != MarkerSymType::NONE) {
925	SortedMarkerSymbols.push_back(x: MarkerSym{.Address: SymInfo.Address, .Type: MarkerType});
926	LastAddr = SymInfo.Address;
927	IsData = MarkerType == MarkerSymType::DATA;
928	continue;
929	}
930
931	if (IsData) {
932	SortedMarkerSymbols.push_back(x: {.Address: SymInfo.Address, .Type: MarkerSymType::DATA});
933	LastAddr = SymInfo.Address;
934	}
935	}
936	};
937
938	if (BC->isAArch64() || BC->isRISCV()) {
939	addExtraDataMarkerPerSymbol();
940	LastSymbol = std::stable_partition(
941	first: SortedSymbols.begin(), last: SortedSymbols.end(),
942	pred: [this](const SymbolInfo &S) { return !BC->isMarker(Symbol: S.Symbol); });
943	if (!SortedSymbols.empty())
944	--LastSymbol;
945	}
946
947	BinaryFunction *PreviousFunction = nullptr;
948	unsigned AnonymousId = 0;
949
950	// Regex object for matching cold fragments.
951	const Regex ColdFragment(".*\\.cold(\\.[0-9]+)?");
952
953	const auto SortedSymbolsEnd =
954	LastSymbol == SortedSymbols.end() ? LastSymbol : std::next(x: LastSymbol);
955	for (auto Iter = SortedSymbols.begin(); Iter != SortedSymbolsEnd; ++Iter) {
956	const SymbolRef &Symbol = Iter->Symbol;
957	const uint64_t SymbolAddress = Iter->Address;
958	const auto SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
959	const SymbolRef::Type SymbolType = cantFail(ValOrErr: Symbol.getType());
960
961	if (SymbolType == SymbolRef::ST_File)
962	continue;
963
964	StringRef SymName = cantFail(ValOrErr: Symbol.getName(), Msg: "cannot get symbol name");
965	if (SymbolAddress == 0) {
966	if (opts::Verbosity >= 1 && SymbolType == SymbolRef::ST_Function)
967	BC->errs() << "BOLT-WARNING: function with 0 address seen\n";
968	continue;
969	}
970
971	// Ignore input hot markers
972	if (SymName == "__hot_start" || SymName == "__hot_end")
973	continue;
974
975	FileSymRefs[SymbolAddress] = Symbol;
976
977	// Skip section symbols that will be registered by disassemblePLT().
978	if (SymbolType == SymbolRef::ST_Debug) {
979	ErrorOr<BinarySection &> BSection =
980	BC->getSectionForAddress(Address: SymbolAddress);
981	if (BSection && getPLTSectionInfo(SectionName: BSection->getName()))
982	continue;
983	}
984
985	/// It is possible we are seeing a globalized local. LLVM might treat it as
986	/// a local if it has a "private global" prefix, e.g. ".L". Thus we have to
987	/// change the prefix to enforce global scope of the symbol.
988	std::string Name =
989	SymName.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix())
990	? "PG" + std::string(SymName)
991	: std::string(SymName);
992
993	// Disambiguate all local symbols before adding to symbol table.
994	// Since we don't know if we will see a global with the same name,
995	// always modify the local name.
996	//
997	// NOTE: the naming convention for local symbols should match
998	// the one we use for profile data.
999	std::string UniqueName;
1000	std::string AlternativeName;
1001	if (Name.empty()) {
1002	UniqueName = "ANONYMOUS." + std::to_string(val: AnonymousId++);
1003	} else if (SymbolFlags & SymbolRef::SF_Global) {
1004	if (const BinaryData *BD = BC->getBinaryDataByName(Name)) {
1005	if (BD->getSize() == ELFSymbolRef(Symbol).getSize() &&
1006	BD->getAddress() == SymbolAddress) {
1007	if (opts::Verbosity > 1)
1008	BC->errs() << "BOLT-WARNING: ignoring duplicate global symbol "
1009	<< Name << "\n";
1010	// Ignore duplicate entry - possibly a bug in the linker
1011	continue;
1012	}
1013	BC->errs() << "BOLT-ERROR: bad input binary, global symbol \"" << Name
1014	<< "\" is not unique\n";
1015	exit(status: 1);
1016	}
1017	UniqueName = Name;
1018	} else {
1019	// If we have a local file name, we should create 2 variants for the
1020	// function name. The reason is that perf profile might have been
1021	// collected on a binary that did not have the local file name (e.g. as
1022	// a side effect of stripping debug info from the binary):
1023	//
1024	// primary: <function>/<id>
1025	// alternative: <function>/<file>/<id2>
1026	//
1027	// The <id> field is used for disambiguation of local symbols since there
1028	// could be identical function names coming from identical file names
1029	// (e.g. from different directories).
1030	std::string AltPrefix;
1031	auto SFI = SymbolToFileName.find(x: Symbol);
1032	if (SymbolType == SymbolRef::ST_Function && SFI != SymbolToFileName.end())
1033	AltPrefix = Name + "/" + std::string(SFI->second);
1034
1035	UniqueName = NR.uniquify(Name);
1036	if (!AltPrefix.empty())
1037	AlternativeName = NR.uniquify(Name: AltPrefix);
1038	}
1039
1040	uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1041	uint64_t SymbolAlignment = Symbol.getAlignment();
1042
1043	auto registerName = [&](uint64_t FinalSize) {
1044	// Register names even if it's not a function, e.g. for an entry point.
1045	BC->registerNameAtAddress(Name: UniqueName, Address: SymbolAddress, Size: FinalSize,
1046	Alignment: SymbolAlignment, Flags: SymbolFlags);
1047	if (!AlternativeName.empty())
1048	BC->registerNameAtAddress(Name: AlternativeName, Address: SymbolAddress, Size: FinalSize,
1049	Alignment: SymbolAlignment, Flags: SymbolFlags);
1050	};
1051
1052	section_iterator Section =
1053	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
1054	if (Section == InputFile->section_end()) {
1055	// Could be an absolute symbol. Used on RISC-V for __global_pointer$ so we
1056	// need to record it to handle relocations against it. For other instances
1057	// of absolute symbols, we record for pretty printing.
1058	LLVM_DEBUG(if (opts::Verbosity > 1) {
1059	dbgs() << "BOLT-INFO: absolute sym " << UniqueName << "\n";
1060	});
1061	registerName(SymbolSize);
1062	continue;
1063	}
1064
1065	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName
1066	<< " for function\n");
1067
1068	if (SymbolAddress == Section->getAddress() + Section->getSize()) {
1069	assert(SymbolSize == 0 &&
1070	"unexpect non-zero sized symbol at end of section");
1071	LLVM_DEBUG(
1072	dbgs()
1073	<< "BOLT-DEBUG: rejecting as symbol points to end of its section\n");
1074	registerName(SymbolSize);
1075	continue;
1076	}
1077
1078	if (!Section->isText()) {
1079	assert(SymbolType != SymbolRef::ST_Function &&
1080	"unexpected function inside non-code section");
1081	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code\n");
1082	registerName(SymbolSize);
1083	continue;
1084	}
1085
1086	// Assembly functions could be ST_NONE with 0 size. Check that the
1087	// corresponding section is a code section and they are not inside any
1088	// other known function to consider them.
1089	//
1090	// Sometimes assembly functions are not marked as functions and neither are
1091	// their local labels. The only way to tell them apart is to look at
1092	// symbol scope - global vs local.
1093	if (PreviousFunction && SymbolType != SymbolRef::ST_Function) {
1094	if (PreviousFunction->containsAddress(PC: SymbolAddress)) {
1095	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1096	LLVM_DEBUG(dbgs()
1097	<< "BOLT-DEBUG: symbol is a function local symbol\n");
1098	} else if (SymbolAddress == PreviousFunction->getAddress() &&
1099	!SymbolSize) {
1100	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring symbol as a marker\n");
1101	} else if (opts::Verbosity > 1) {
1102	BC->errs() << "BOLT-WARNING: symbol " << UniqueName
1103	<< " seen in the middle of function " << *PreviousFunction
1104	<< ". Could be a new entry.\n";
1105	}
1106	registerName(SymbolSize);
1107	continue;
1108	} else if (PreviousFunction->getSize() == 0 &&
1109	PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1110	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: symbol is a function local symbol\n");
1111	registerName(SymbolSize);
1112	continue;
1113	}
1114	}
1115
1116	if (PreviousFunction && PreviousFunction->containsAddress(PC: SymbolAddress) &&
1117	PreviousFunction->getAddress() != SymbolAddress) {
1118	if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {
1119	if (opts::Verbosity >= 1)
1120	BC->outs()
1121	<< "BOLT-INFO: skipping possibly another entry for function "
1122	<< *PreviousFunction << " : " << UniqueName << '\n';
1123	registerName(SymbolSize);
1124	} else {
1125	BC->outs() << "BOLT-INFO: using " << UniqueName
1126	<< " as another entry to "
1127	<< "function " << *PreviousFunction << '\n';
1128
1129	registerName(0);
1130
1131	PreviousFunction->addEntryPointAtOffset(Offset: SymbolAddress -
1132	PreviousFunction->getAddress());
1133
1134	// Remove the symbol from FileSymRefs so that we can skip it from
1135	// in the future.
1136	auto SI = FileSymRefs.find(x: SymbolAddress);
1137	assert(SI != FileSymRefs.end() && "symbol expected to be present");
1138	assert(SI->second == Symbol && "wrong symbol found");
1139	FileSymRefs.erase(position: SI);
1140	}
1141	continue;
1142	}
1143
1144	// Checkout for conflicts with function data from FDEs.
1145	bool IsSimple = true;
1146	auto FDEI = CFIRdWrt->getFDEs().lower_bound(x: SymbolAddress);
1147	if (FDEI != CFIRdWrt->getFDEs().end()) {
1148	const dwarf::FDE &FDE = *FDEI->second;
1149	if (FDEI->first != SymbolAddress) {
1150	// There's no matching starting address in FDE. Make sure the previous
1151	// FDE does not contain this address.
1152	if (FDEI != CFIRdWrt->getFDEs().begin()) {
1153	--FDEI;
1154	const dwarf::FDE &PrevFDE = *FDEI->second;
1155	uint64_t PrevStart = PrevFDE.getInitialLocation();
1156	uint64_t PrevLength = PrevFDE.getAddressRange();
1157	if (SymbolAddress > PrevStart &&
1158	SymbolAddress < PrevStart + PrevLength) {
1159	BC->errs() << "BOLT-ERROR: function " << UniqueName
1160	<< " is in conflict with FDE ["
1161	<< Twine::utohexstr(Val: PrevStart) << ", "
1162	<< Twine::utohexstr(Val: PrevStart + PrevLength)
1163	<< "). Skipping.\n";
1164	IsSimple = false;
1165	}
1166	}
1167	} else if (FDE.getAddressRange() != SymbolSize) {
1168	if (SymbolSize) {
1169	// Function addresses match but sizes differ.
1170	BC->errs() << "BOLT-WARNING: sizes differ for function " << UniqueName
1171	<< ". FDE : " << FDE.getAddressRange()
1172	<< "; symbol table : " << SymbolSize
1173	<< ". Using max size.\n";
1174	}
1175	SymbolSize = std::max(a: SymbolSize, b: FDE.getAddressRange());
1176	if (BC->getBinaryDataAtAddress(Address: SymbolAddress)) {
1177	BC->setBinaryDataSize(Address: SymbolAddress, Size: SymbolSize);
1178	} else {
1179	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: No BD @ 0x"
1180	<< Twine::utohexstr(SymbolAddress) << "\n");
1181	}
1182	}
1183	}
1184
1185	BinaryFunction *BF = nullptr;
1186	// Since function may not have yet obtained its real size, do a search
1187	// using the list of registered functions instead of calling
1188	// getBinaryFunctionAtAddress().
1189	auto BFI = BC->getBinaryFunctions().find(x: SymbolAddress);
1190	if (BFI != BC->getBinaryFunctions().end()) {
1191	BF = &BFI->second;
1192	// Duplicate the function name. Make sure everything matches before we add
1193	// an alternative name.
1194	if (SymbolSize != BF->getSize()) {
1195	if (opts::Verbosity >= 1) {
1196	if (SymbolSize && BF->getSize())
1197	BC->errs() << "BOLT-WARNING: size mismatch for duplicate entries "
1198	<< *BF << " and " << UniqueName << '\n';
1199	BC->outs() << "BOLT-INFO: adjusting size of function " << *BF
1200	<< " old " << BF->getSize() << " new " << SymbolSize
1201	<< "\n";
1202	}
1203	BF->setSize(std::max(a: SymbolSize, b: BF->getSize()));
1204	BC->setBinaryDataSize(Address: SymbolAddress, Size: BF->getSize());
1205	}
1206	BF->addAlternativeName(NewName: UniqueName);
1207	} else {
1208	ErrorOr<BinarySection &> Section =
1209	BC->getSectionForAddress(Address: SymbolAddress);
1210	// Skip symbols from invalid sections
1211	if (!Section) {
1212	BC->errs() << "BOLT-WARNING: " << UniqueName << " (0x"
1213	<< Twine::utohexstr(Val: SymbolAddress)
1214	<< ") does not have any section\n";
1215	continue;
1216	}
1217
1218	// Skip symbols from zero-sized sections.
1219	if (!Section->getSize())
1220	continue;
1221
1222	BF = BC->createBinaryFunction(Name: UniqueName, Section&: *Section, Address: SymbolAddress,
1223	Size: SymbolSize);
1224	if (!IsSimple)
1225	BF->setSimple(false);
1226	}
1227
1228	// Check if it's a cold function fragment.
1229	if (ColdFragment.match(String: SymName)) {
1230	static bool PrintedWarning = false;
1231	if (!PrintedWarning) {
1232	PrintedWarning = true;
1233	BC->errs() << "BOLT-WARNING: split function detected on input : "
1234	<< SymName;
1235	if (BC->HasRelocations)
1236	BC->errs() << ". The support is limited in relocation mode\n";
1237	else
1238	BC->errs() << '\n';
1239	}
1240	BC->HasSplitFunctions = true;
1241	BF->IsFragment = true;
1242	}
1243
1244	if (!AlternativeName.empty())
1245	BF->addAlternativeName(NewName: AlternativeName);
1246
1247	registerName(SymbolSize);
1248	PreviousFunction = BF;
1249	}
1250
1251	// Read dynamic relocation first as their presence affects the way we process
1252	// static relocations. E.g. we will ignore a static relocation at an address
1253	// that is a subject to dynamic relocation processing.
1254	processDynamicRelocations();
1255
1256	// Process PLT section.
1257	disassemblePLT();
1258
1259	// See if we missed any functions marked by FDE.
1260	for (const auto &FDEI : CFIRdWrt->getFDEs()) {
1261	const uint64_t Address = FDEI.first;
1262	const dwarf::FDE *FDE = FDEI.second;
1263	const BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address);
1264	if (BF)
1265	continue;
1266
1267	BF = BC->getBinaryFunctionContainingAddress(Address);
1268	if (BF) {
1269	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1270	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1271	<< ") conflicts with function " << *BF << '\n';
1272	continue;
1273	}
1274
1275	if (opts::Verbosity >= 1)
1276	BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Val: Address)
1277	<< ", 0x" << Twine::utohexstr(Val: Address + FDE->getAddressRange())
1278	<< ") has no corresponding symbol table entry\n";
1279
1280	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
1281	assert(Section && "cannot get section for address from FDE");
1282	std::string FunctionName =
1283	"__BOLT_FDE_FUNCat" + Twine::utohexstr(Val: Address).str();
1284	BC->createBinaryFunction(Name: FunctionName, Section&: *Section, Address,
1285	Size: FDE->getAddressRange());
1286	}
1287
1288	BC->setHasSymbolsWithFileName(SeenFileName);
1289
1290	// Now that all the functions were created - adjust their boundaries.
1291	adjustFunctionBoundaries();
1292
1293	// Annotate functions with code/data markers in AArch64
1294	for (auto ISym = SortedMarkerSymbols.begin();
1295	ISym != SortedMarkerSymbols.end(); ++ISym) {
1296
1297	auto *BF =
1298	BC->getBinaryFunctionContainingAddress(Address: ISym->Address, CheckPastEnd: true, UseMaxSize: true);
1299
1300	if (!BF) {
1301	// Stray marker
1302	continue;
1303	}
1304	const auto EntryOffset = ISym->Address - BF->getAddress();
1305	if (ISym->Type == MarkerSymType::CODE) {
1306	BF->markCodeAtOffset(Offset: EntryOffset);
1307	continue;
1308	}
1309	if (ISym->Type == MarkerSymType::DATA) {
1310	BF->markDataAtOffset(Offset: EntryOffset);
1311	BC->AddressToConstantIslandMap[ISym->Address] = BF;
1312	continue;
1313	}
1314	llvm_unreachable("Unknown marker");
1315	}
1316
1317	if (BC->isAArch64()) {
1318	// Check for dynamic relocations that might be contained in
1319	// constant islands.
1320	for (const BinarySection &Section : BC->allocatableSections()) {
1321	const uint64_t SectionAddress = Section.getAddress();
1322	for (const Relocation &Rel : Section.dynamicRelocations()) {
1323	const uint64_t RelAddress = SectionAddress + Rel.Offset;
1324	BinaryFunction *BF =
1325	BC->getBinaryFunctionContainingAddress(Address: RelAddress,
1326	/*CheckPastEnd*/ false,
1327	/*UseMaxSize*/ true);
1328	if (BF) {
1329	assert(Rel.isRelative() && "Expected relative relocation for island");
1330	BC->logBOLTErrorsAndQuitOnFatal(
1331	E: BF->markIslandDynamicRelocationAtAddress(Address: RelAddress));
1332	}
1333	}
1334	}
1335	}
1336
1337	if (!BC->IsLinuxKernel) {
1338	// Read all relocations now that we have binary functions mapped.
1339	processRelocations();
1340	}
1341
1342	registerFragments();
1343	}
1344
1345	Error RewriteInstance::discoverRtFiniAddress() {
1346	// Use DT_FINI if it's available.
1347	if (BC->FiniAddress) {
1348	BC->FiniFunctionAddress = BC->FiniAddress;
1349	return Error::success();
1350	}
1351
1352	if (!BC->FiniArrayAddress || !BC->FiniArraySize) {
1353	return createStringError(
1354	EC: std::errc::not_supported,
1355	Fmt: "Instrumentation needs either DT_FINI or DT_FINI_ARRAY");
1356	}
1357
1358	if (*BC->FiniArraySize < BC->AsmInfo->getCodePointerSize()) {
1359	return createStringError(EC: std::errc::not_supported,
1360	Fmt: "Need at least 1 DT_FINI_ARRAY slot");
1361	}
1362
1363	ErrorOr<BinarySection &> FiniArraySection =
1364	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1365	if (auto EC = FiniArraySection.getError())
1366	return errorCodeToError(EC);
1367
1368	if (const Relocation *Reloc = FiniArraySection->getDynamicRelocationAt(Offset: 0)) {
1369	BC->FiniFunctionAddress = Reloc->Addend;
1370	return Error::success();
1371	}
1372
1373	if (const Relocation *Reloc = FiniArraySection->getRelocationAt(Offset: 0)) {
1374	BC->FiniFunctionAddress = Reloc->Value;
1375	return Error::success();
1376	}
1377
1378	return createStringError(EC: std::errc::not_supported,
1379	Fmt: "No relocation for first DT_FINI_ARRAY slot");
1380	}
1381
1382	void RewriteInstance::updateRtFiniReloc() {
1383	// Updating DT_FINI is handled by patchELFDynamic.
1384	if (BC->FiniAddress)
1385	return;
1386
1387	const RuntimeLibrary *RT = BC->getRuntimeLibrary();
1388	if (!RT || !RT->getRuntimeFiniAddress())
1389	return;
1390
1391	assert(BC->FiniArrayAddress && BC->FiniArraySize &&
1392	"inconsistent .fini_array state");
1393
1394	ErrorOr<BinarySection &> FiniArraySection =
1395	BC->getSectionForAddress(Address: *BC->FiniArrayAddress);
1396	assert(FiniArraySection && ".fini_array removed");
1397
1398	if (std::optional<Relocation> Reloc =
1399	FiniArraySection->takeDynamicRelocationAt(Offset: 0)) {
1400	assert(Reloc->Addend == BC->FiniFunctionAddress &&
1401	"inconsistent .fini_array dynamic relocation");
1402	Reloc->Addend = RT->getRuntimeFiniAddress();
1403	FiniArraySection->addDynamicRelocation(Reloc: *Reloc);
1404	}
1405
1406	// Update the static relocation by adding a pending relocation which will get
1407	// patched when flushPendingRelocations is called in rewriteFile. Note that
1408	// flushPendingRelocations will calculate the value to patch as
1409	// "Symbol + Addend". Since we don't have a symbol, just set the addend to the
1410	// desired value.
1411	FiniArraySection->addPendingRelocation(Rel: Relocation{
1412	/*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),
1413	/*Addend*/ RT->getRuntimeFiniAddress(), /*Value*/ 0});
1414	}
1415
1416	void RewriteInstance::registerFragments() {
1417	if (!BC->HasSplitFunctions)
1418	return;
1419
1420	for (auto &BFI : BC->getBinaryFunctions()) {
1421	BinaryFunction &Function = BFI.second;
1422	if (!Function.isFragment())
1423	continue;
1424	unsigned ParentsFound = 0;
1425	for (StringRef Name : Function.getNames()) {
1426	StringRef BaseName, Suffix;
1427	std::tie(args&: BaseName, args&: Suffix) = Name.split(Separator: '/');
1428	const size_t ColdSuffixPos = BaseName.find(Str: ".cold");
1429	if (ColdSuffixPos == StringRef::npos)
1430	continue;
1431	// For cold function with local (foo.cold/1) symbol, prefer a parent with
1432	// local symbol as well (foo/1) over global symbol (foo).
1433	std::string ParentName = BaseName.substr(Start: 0, N: ColdSuffixPos).str();
1434	const BinaryData *BD = BC->getBinaryDataByName(Name: ParentName);
1435	if (Suffix != "") {
1436	ParentName.append(str: Twine("/", Suffix).str());
1437	const BinaryData *BDLocal = BC->getBinaryDataByName(Name: ParentName);
1438	if (BDLocal || !BD)
1439	BD = BDLocal;
1440	}
1441	if (!BD) {
1442	if (opts::Verbosity >= 1)
1443	BC->outs() << "BOLT-INFO: parent function not found for " << Name
1444	<< "\n";
1445	continue;
1446	}
1447	const uint64_t Address = BD->getAddress();
1448	BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address);
1449	if (!BF) {
1450	if (opts::Verbosity >= 1)
1451	BC->outs() << formatv(
1452	Fmt: "BOLT-INFO: parent function not found at {0:x}\n", Vals: Address);
1453	continue;
1454	}
1455	BC->registerFragment(TargetFunction&: Function, Function&: *BF);
1456	++ParentsFound;
1457	}
1458	if (!ParentsFound) {
1459	BC->errs() << "BOLT-ERROR: parent function not found for " << Function
1460	<< '\n';
1461	exit(status: 1);
1462	}
1463	}
1464	}
1465
1466	void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,
1467	uint64_t EntryAddress,
1468	uint64_t EntrySize) {
1469	if (!TargetAddress)
1470	return;
1471
1472	auto setPLTSymbol = [&](BinaryFunction *BF, StringRef Name) {
1473	const unsigned PtrSize = BC->AsmInfo->getCodePointerSize();
1474	MCSymbol *TargetSymbol = BC->registerNameAtAddress(
1475	Name: Name.str() + "@GOT", Address: TargetAddress, Size: PtrSize, Alignment: PtrSize);
1476	BF->setPLTSymbol(TargetSymbol);
1477	};
1478
1479	BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address: EntryAddress);
1480	if (BF && BC->isAArch64()) {
1481	// Handle IFUNC trampoline with symbol
1482	setPLTSymbol(BF, BF->getOneName());
1483	return;
1484	}
1485
1486	const Relocation *Rel = BC->getDynamicRelocationAt(Address: TargetAddress);
1487	if (!Rel)
1488	return;
1489
1490	MCSymbol *Symbol = Rel->Symbol;
1491	if (!Symbol) {
1492	if (!BC->isAArch64() || !Rel->Addend || !Rel->isIRelative())
1493	return;
1494
1495	// IFUNC trampoline without symbol
1496	BinaryFunction *TargetBF = BC->getBinaryFunctionAtAddress(Address: Rel->Addend);
1497	if (!TargetBF) {
1498	BC->errs()
1499	<< "BOLT-WARNING: Expected BF to be presented as IFUNC resolver at "
1500	<< Twine::utohexstr(Val: Rel->Addend) << ", skipping\n";
1501	return;
1502	}
1503
1504	Symbol = TargetBF->getSymbol();
1505	}
1506
1507	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: EntryAddress);
1508	assert(Section && "cannot get section for address");
1509	if (!BF)
1510	BF = BC->createBinaryFunction(Name: Symbol->getName().str() + "@PLT", Section&: *Section,
1511	Address: EntryAddress, Size: 0, SymbolSize: EntrySize,
1512	Alignment: Section->getAlignment());
1513	else
1514	BF->addAlternativeName(NewName: Symbol->getName().str() + "@PLT");
1515	setPLTSymbol(BF, Symbol->getName());
1516	}
1517
1518	void RewriteInstance::disassemblePLTInstruction(const BinarySection &Section,
1519	uint64_t InstrOffset,
1520	MCInst &Instruction,
1521	uint64_t &InstrSize) {
1522	const uint64_t SectionAddress = Section.getAddress();
1523	const uint64_t SectionSize = Section.getSize();
1524	StringRef PLTContents = Section.getContents();
1525	ArrayRef<uint8_t> PLTData(
1526	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1527
1528	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1529	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1530	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1531	CStream&: nulls())) {
1532	BC->errs()
1533	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1534	<< Section.getName() << formatv(Fmt: " at offset {0:x}\n", Vals&: InstrOffset);
1535	exit(status: 1);
1536	}
1537	}
1538
1539	void RewriteInstance::disassemblePLTSectionAArch64(BinarySection &Section) {
1540	const uint64_t SectionAddress = Section.getAddress();
1541	const uint64_t SectionSize = Section.getSize();
1542
1543	uint64_t InstrOffset = 0;
1544	// Locate new plt entry
1545	while (InstrOffset < SectionSize) {
1546	InstructionListType Instructions;
1547	MCInst Instruction;
1548	uint64_t EntryOffset = InstrOffset;
1549	uint64_t EntrySize = 0;
1550	uint64_t InstrSize;
1551	// Loop through entry instructions
1552	while (InstrOffset < SectionSize) {
1553	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1554	EntrySize += InstrSize;
1555	if (!BC->MIB->isIndirectBranch(Inst: Instruction)) {
1556	Instructions.emplace_back(args&: Instruction);
1557	InstrOffset += InstrSize;
1558	continue;
1559	}
1560
1561	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1562	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1563	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1564
1565	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1566	break;
1567	}
1568
1569	// Branch instruction
1570	InstrOffset += InstrSize;
1571
1572	// Skip nops if any
1573	while (InstrOffset < SectionSize) {
1574	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1575	if (!BC->MIB->isNoop(Inst: Instruction))
1576	break;
1577
1578	InstrOffset += InstrSize;
1579	}
1580	}
1581	}
1582
1583	void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) {
1584	const uint64_t SectionAddress = Section.getAddress();
1585	const uint64_t SectionSize = Section.getSize();
1586	StringRef PLTContents = Section.getContents();
1587	ArrayRef<uint8_t> PLTData(
1588	reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);
1589
1590	auto disassembleInstruction = [&](uint64_t InstrOffset, MCInst &Instruction,
1591	uint64_t &InstrSize) {
1592	const uint64_t InstrAddr = SectionAddress + InstrOffset;
1593	if (!BC->DisAsm->getInstruction(Instr&: Instruction, Size&: InstrSize,
1594	Bytes: PLTData.slice(N: InstrOffset), Address: InstrAddr,
1595	CStream&: nulls())) {
1596	BC->errs()
1597	<< "BOLT-ERROR: unable to disassemble instruction in PLT section "
1598	<< Section.getName() << " at offset 0x"
1599	<< Twine::utohexstr(Val: InstrOffset) << '\n';
1600	exit(status: 1);
1601	}
1602	};
1603
1604	// Skip the first special entry since no relocation points to it.
1605	uint64_t InstrOffset = 32;
1606
1607	while (InstrOffset < SectionSize) {
1608	InstructionListType Instructions;
1609	MCInst Instruction;
1610	const uint64_t EntryOffset = InstrOffset;
1611	const uint64_t EntrySize = 16;
1612	uint64_t InstrSize;
1613
1614	while (InstrOffset < EntryOffset + EntrySize) {
1615	disassembleInstruction(InstrOffset, Instruction, InstrSize);
1616	Instructions.emplace_back(args&: Instruction);
1617	InstrOffset += InstrSize;
1618	}
1619
1620	const uint64_t EntryAddress = SectionAddress + EntryOffset;
1621	const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(
1622	Instruction, Begin: Instructions.begin(), End: Instructions.end(), BeginPC: EntryAddress);
1623
1624	createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);
1625	}
1626	}
1627
1628	void RewriteInstance::disassemblePLTSectionX86(BinarySection &Section,
1629	uint64_t EntrySize) {
1630	const uint64_t SectionAddress = Section.getAddress();
1631	const uint64_t SectionSize = Section.getSize();
1632
1633	for (uint64_t EntryOffset = 0; EntryOffset + EntrySize <= SectionSize;
1634	EntryOffset += EntrySize) {
1635	MCInst Instruction;
1636	uint64_t InstrSize, InstrOffset = EntryOffset;
1637	while (InstrOffset < EntryOffset + EntrySize) {
1638	disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);
1639	// Check if the entry size needs adjustment.
1640	if (EntryOffset == 0 && BC->MIB->isTerminateBranch(Inst: Instruction) &&
1641	EntrySize == 8)
1642	EntrySize = 16;
1643
1644	if (BC->MIB->isIndirectBranch(Inst: Instruction))
1645	break;
1646
1647	InstrOffset += InstrSize;
1648	}
1649
1650	if (InstrOffset + InstrSize > EntryOffset + EntrySize)
1651	continue;
1652
1653	uint64_t TargetAddress;
1654	if (!BC->MIB->evaluateMemOperandTarget(Inst: Instruction, Target&: TargetAddress,
1655	Address: SectionAddress + InstrOffset,
1656	Size: InstrSize)) {
1657	BC->errs() << "BOLT-ERROR: error evaluating PLT instruction at offset 0x"
1658	<< Twine::utohexstr(Val: SectionAddress + InstrOffset) << '\n';
1659	exit(status: 1);
1660	}
1661
1662	createPLTBinaryFunction(TargetAddress, EntryAddress: SectionAddress + EntryOffset,
1663	EntrySize);
1664	}
1665	}
1666
1667	void RewriteInstance::disassemblePLT() {
1668	auto analyzeOnePLTSection = [&](BinarySection &Section, uint64_t EntrySize) {
1669	if (BC->isAArch64())
1670	return disassemblePLTSectionAArch64(Section);
1671	if (BC->isRISCV())
1672	return disassemblePLTSectionRISCV(Section);
1673	if (BC->isX86())
1674	return disassemblePLTSectionX86(Section, EntrySize);
1675	llvm_unreachable("Unmplemented PLT");
1676	};
1677
1678	for (BinarySection &Section : BC->allocatableSections()) {
1679	const PLTSectionInfo *PLTSI = getPLTSectionInfo(SectionName: Section.getName());
1680	if (!PLTSI)
1681	continue;
1682
1683	analyzeOnePLTSection(Section, PLTSI->EntrySize);
1684
1685	BinaryFunction *PltBF;
1686	auto BFIter = BC->getBinaryFunctions().find(x: Section.getAddress());
1687	if (BFIter != BC->getBinaryFunctions().end()) {
1688	PltBF = &BFIter->second;
1689	} else {
1690	// If we did not register any function at the start of the section,
1691	// then it must be a general PLT entry. Add a function at the location.
1692	PltBF = BC->createBinaryFunction(
1693	Name: "__BOLT_PSEUDO_" + Section.getName().str(), Section,
1694	Address: Section.getAddress(), Size: 0, SymbolSize: PLTSI->EntrySize, Alignment: Section.getAlignment());
1695	}
1696	PltBF->setPseudo(true);
1697	}
1698	}
1699
1700	void RewriteInstance::adjustFunctionBoundaries() {
1701	for (auto BFI = BC->getBinaryFunctions().begin(),
1702	BFE = BC->getBinaryFunctions().end();
1703	BFI != BFE; ++BFI) {
1704	BinaryFunction &Function = BFI->second;
1705	const BinaryFunction *NextFunction = nullptr;
1706	if (std::next(x: BFI) != BFE)
1707	NextFunction = &std::next(x: BFI)->second;
1708
1709	// Check if there's a symbol or a function with a larger address in the
1710	// same section. If there is - it determines the maximum size for the
1711	// current function. Otherwise, it is the size of a containing section
1712	// the defines it.
1713	//
1714	// NOTE: ignore some symbols that could be tolerated inside the body
1715	// of a function.
1716	auto NextSymRefI = FileSymRefs.upper_bound(x: Function.getAddress());
1717	while (NextSymRefI != FileSymRefs.end()) {
1718	SymbolRef &Symbol = NextSymRefI->second;
1719	const uint64_t SymbolAddress = NextSymRefI->first;
1720	const uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
1721
1722	if (NextFunction && SymbolAddress >= NextFunction->getAddress())
1723	break;
1724
1725	if (!Function.isSymbolValidInScope(Symbol, SymbolSize))
1726	break;
1727
1728	// Ignore unnamed symbols. Used, for example, by debugging info on RISC-V.
1729	if (BC->isRISCV() && cantFail(ValOrErr: Symbol.getName()).empty()) {
1730	++NextSymRefI;
1731	continue;
1732	}
1733
1734	// Skip basic block labels. This happens on RISC-V with linker relaxation
1735	// enabled because every branch needs a relocation and corresponding
1736	// symbol. We don't want to add such symbols as entry points.
1737	const auto PrivateLabelPrefix = BC->AsmInfo->getPrivateLabelPrefix();
1738	if (!PrivateLabelPrefix.empty() &&
1739	cantFail(ValOrErr: Symbol.getName()).starts_with(Prefix: PrivateLabelPrefix)) {
1740	++NextSymRefI;
1741	continue;
1742	}
1743
1744	// This is potentially another entry point into the function.
1745	uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();
1746	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "
1747	<< Function << " at offset 0x"
1748	<< Twine::utohexstr(EntryOffset) << '\n');
1749	Function.addEntryPointAtOffset(Offset: EntryOffset);
1750
1751	++NextSymRefI;
1752	}
1753
1754	// Function runs at most till the end of the containing section.
1755	uint64_t NextObjectAddress = Function.getOriginSection()->getEndAddress();
1756	// Or till the next object marked by a symbol.
1757	if (NextSymRefI != FileSymRefs.end())
1758	NextObjectAddress = std::min(a: NextSymRefI->first, b: NextObjectAddress);
1759
1760	// Or till the next function not marked by a symbol.
1761	if (NextFunction)
1762	NextObjectAddress =
1763	std::min(a: NextFunction->getAddress(), b: NextObjectAddress);
1764
1765	const uint64_t MaxSize = NextObjectAddress - Function.getAddress();
1766	if (MaxSize < Function.getSize()) {
1767	BC->errs() << "BOLT-ERROR: symbol seen in the middle of the function "
1768	<< Function << ". Skipping.\n";
1769	Function.setSimple(false);
1770	Function.setMaxSize(Function.getSize());
1771	continue;
1772	}
1773	Function.setMaxSize(MaxSize);
1774	if (!Function.getSize() && Function.isSimple()) {
1775	// Some assembly functions have their size set to 0, use the max
1776	// size as their real size.
1777	if (opts::Verbosity >= 1)
1778	BC->outs() << "BOLT-INFO: setting size of function " << Function
1779	<< " to " << Function.getMaxSize() << " (was 0)\n";
1780	Function.setSize(Function.getMaxSize());
1781	}
1782	}
1783	}
1784
1785	void RewriteInstance::relocateEHFrameSection() {
1786	assert(EHFrameSection && "Non-empty .eh_frame section expected.");
1787
1788	BinarySection *RelocatedEHFrameSection =
1789	getSection(Name: ".relocated" + getEHFrameSectionName());
1790	assert(RelocatedEHFrameSection &&
1791	"Relocated eh_frame section should be preregistered.");
1792	DWARFDataExtractor DE(EHFrameSection->getContents(),
1793	BC->AsmInfo->isLittleEndian(),
1794	BC->AsmInfo->getCodePointerSize());
1795	auto createReloc = [&](uint64_t Value, uint64_t Offset, uint64_t DwarfType) {
1796	if (DwarfType == dwarf::DW_EH_PE_omit)
1797	return;
1798
1799	// Only fix references that are relative to other locations.
1800	if (!(DwarfType & dwarf::DW_EH_PE_pcrel) &&
1801	!(DwarfType & dwarf::DW_EH_PE_textrel) &&
1802	!(DwarfType & dwarf::DW_EH_PE_funcrel) &&
1803	!(DwarfType & dwarf::DW_EH_PE_datarel))
1804	return;
1805
1806	if (!(DwarfType & dwarf::DW_EH_PE_sdata4))
1807	return;
1808
1809	uint64_t RelType;
1810	switch (DwarfType & 0x0f) {
1811	default:
1812	llvm_unreachable("unsupported DWARF encoding type");
1813	case dwarf::DW_EH_PE_sdata4:
1814	case dwarf::DW_EH_PE_udata4:
1815	RelType = Relocation::getPC32();
1816	Offset -= 4;
1817	break;
1818	case dwarf::DW_EH_PE_sdata8:
1819	case dwarf::DW_EH_PE_udata8:
1820	RelType = Relocation::getPC64();
1821	Offset -= 8;
1822	break;
1823	}
1824
1825	// Create a relocation against an absolute value since the goal is to
1826	// preserve the contents of the section independent of the new values
1827	// of referenced symbols.
1828	RelocatedEHFrameSection->addRelocation(Offset, Symbol: nullptr, Type: RelType, Addend: Value);
1829	};
1830
1831	Error E = EHFrameParser::parse(Data: DE, EHFrameAddress: EHFrameSection->getAddress(), PatcherCallback: createReloc);
1832	check_error(E: std::move(E), Message: "failed to patch EH frame");
1833	}
1834
1835	Error RewriteInstance::readSpecialSections() {
1836	NamedRegionTimer T("readSpecialSections", "read special sections",
1837	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
1838
1839	bool HasTextRelocations = false;
1840	bool HasSymbolTable = false;
1841	bool HasDebugInfo = false;
1842
1843	// Process special sections.
1844	for (const SectionRef &Section : InputFile->sections()) {
1845	Expected<StringRef> SectionNameOrErr = Section.getName();
1846	check_error(E: SectionNameOrErr.takeError(), Message: "cannot get section name");
1847	StringRef SectionName = *SectionNameOrErr;
1848
1849	if (Error E = Section.getContents().takeError())
1850	return E;
1851	BC->registerSection(Section);
1852	LLVM_DEBUG(
1853	dbgs() << "BOLT-DEBUG: registering section " << SectionName << " @ 0x"
1854	<< Twine::utohexstr(Section.getAddress()) << ":0x"
1855	<< Twine::utohexstr(Section.getAddress() + Section.getSize())
1856	<< "\n");
1857	if (isDebugSection(SectionName))
1858	HasDebugInfo = true;
1859	}
1860
1861	// Set IsRelro section attribute based on PT_GNU_RELRO segment.
1862	markGnuRelroSections();
1863
1864	if (HasDebugInfo && !opts::UpdateDebugSections && !opts::AggregateOnly) {
1865	BC->errs() << "BOLT-WARNING: debug info will be stripped from the binary. "
1866	"Use -update-debug-sections to keep it.\n";
1867	}
1868
1869	HasTextRelocations = (bool)BC->getUniqueSectionByName(
1870	SectionName: ".rela" + std::string(BC->getMainCodeSectionName()));
1871	HasSymbolTable = (bool)BC->getUniqueSectionByName(SectionName: ".symtab");
1872	EHFrameSection = BC->getUniqueSectionByName(SectionName: ".eh_frame");
1873	BuildIDSection = BC->getUniqueSectionByName(SectionName: ".note.gnu.build-id");
1874
1875	if (ErrorOr<BinarySection &> BATSec =
1876	BC->getUniqueSectionByName(SectionName: BoltAddressTranslation::SECTION_NAME)) {
1877	// Do not read BAT when plotting a heatmap
1878	if (!opts::HeatmapMode) {
1879	if (std::error_code EC = BAT->parse(OS&: BC->outs(), Buf: BATSec->getContents())) {
1880	BC->errs() << "BOLT-ERROR: failed to parse BOLT address translation "
1881	"table.\n";
1882	exit(status: 1);
1883	}
1884	}
1885	}
1886
1887	if (opts::PrintSections) {
1888	BC->outs() << "BOLT-INFO: Sections from original binary:\n";
1889	BC->printSections(OS&: BC->outs());
1890	}
1891
1892	if (opts::RelocationMode == cl::BOU_TRUE && !HasTextRelocations) {
1893	BC->errs()
1894	<< "BOLT-ERROR: relocations against code are missing from the input "
1895	"file. Cannot proceed in relocations mode (-relocs).\n";
1896	exit(status: 1);
1897	}
1898
1899	BC->HasRelocations =
1900	HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE);
1901
1902	if (BC->IsLinuxKernel && BC->HasRelocations) {
1903	BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";
1904	BC->HasRelocations = false;
1905	}
1906
1907	BC->IsStripped = !HasSymbolTable;
1908
1909	if (BC->IsStripped && !opts::AllowStripped) {
1910	BC->errs()
1911	<< "BOLT-ERROR: stripped binaries are not supported. If you know "
1912	"what you're doing, use --allow-stripped to proceed";
1913	exit(status: 1);
1914	}
1915
1916	// Force non-relocation mode for heatmap generation
1917	if (opts::HeatmapMode)
1918	BC->HasRelocations = false;
1919
1920	if (BC->HasRelocations)
1921	BC->outs() << "BOLT-INFO: enabling " << (opts::StrictMode ? "strict " : "")
1922	<< "relocation mode\n";
1923
1924	// Read EH frame for function boundaries info.
1925	Expected<const DWARFDebugFrame *> EHFrameOrError = BC->DwCtx->getEHFrame();
1926	if (!EHFrameOrError)
1927	report_error(Message: "expected valid eh_frame section", E: EHFrameOrError.takeError());
1928	CFIRdWrt.reset(p: new CFIReaderWriter(*BC, *EHFrameOrError.get()));
1929
1930	// Parse build-id
1931	parseBuildID();
1932	if (std::optional<std::string> FileBuildID = getPrintableBuildID())
1933	BC->setFileBuildID(*FileBuildID);
1934
1935	// Read .dynamic/PT_DYNAMIC.
1936	return readELFDynamic();
1937	}
1938
1939	void RewriteInstance::adjustCommandLineOptions() {
1940	if (BC->isAArch64() && !BC->HasRelocations)
1941	BC->errs() << "BOLT-WARNING: non-relocation mode for AArch64 is not fully "
1942	"supported\n";
1943
1944	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
1945	RtLibrary->adjustCommandLineOptions(BC: *BC);
1946
1947	if (opts::AlignMacroOpFusion != MFT_NONE && !BC->isX86()) {
1948	BC->outs()
1949	<< "BOLT-INFO: disabling -align-macro-fusion on non-x86 platform\n";
1950	opts::AlignMacroOpFusion = MFT_NONE;
1951	}
1952
1953	if (BC->isX86() && BC->MAB->allowAutoPadding()) {
1954	if (!BC->HasRelocations) {
1955	BC->errs()
1956	<< "BOLT-ERROR: cannot apply mitigations for Intel JCC erratum in "
1957	"non-relocation mode\n";
1958	exit(status: 1);
1959	}
1960	BC->outs()
1961	<< "BOLT-WARNING: using mitigation for Intel JCC erratum, layout "
1962	"may take several minutes\n";
1963	opts::AlignMacroOpFusion = MFT_NONE;
1964	}
1965
1966	if (opts::AlignMacroOpFusion != MFT_NONE && !BC->HasRelocations) {
1967	BC->outs() << "BOLT-INFO: disabling -align-macro-fusion in non-relocation "
1968	"mode\n";
1969	opts::AlignMacroOpFusion = MFT_NONE;
1970	}
1971
1972	if (opts::SplitEH && !BC->HasRelocations) {
1973	BC->errs() << "BOLT-WARNING: disabling -split-eh in non-relocation mode\n";
1974	opts::SplitEH = false;
1975	}
1976
1977	if (opts::StrictMode && !BC->HasRelocations) {
1978	BC->errs()
1979	<< "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "
1980	"mode\n";
1981	opts::StrictMode = false;
1982	}
1983
1984	if (BC->HasRelocations && opts::AggregateOnly &&
1985	!opts::StrictMode.getNumOccurrences()) {
1986	BC->outs() << "BOLT-INFO: enabling strict relocation mode for aggregation "
1987	"purposes\n";
1988	opts::StrictMode = true;
1989	}
1990
1991	if (BC->isX86() && BC->HasRelocations &&
1992	opts::AlignMacroOpFusion == MFT_HOT && !ProfileReader) {
1993	BC->outs()
1994	<< "BOLT-INFO: enabling -align-macro-fusion=all since no profile "
1995	"was specified\n";
1996	opts::AlignMacroOpFusion = MFT_ALL;
1997	}
1998
1999	if (!BC->HasRelocations &&
2000	opts::ReorderFunctions != ReorderFunctions::RT_NONE) {
2001	BC->errs() << "BOLT-ERROR: function reordering only works when "
2002	<< "relocations are enabled\n";
2003	exit(status: 1);
2004	}
2005
2006	if (opts::Instrument ||
2007	(opts::ReorderFunctions != ReorderFunctions::RT_NONE &&
2008	!opts::HotText.getNumOccurrences())) {
2009	opts::HotText = true;
2010	} else if (opts::HotText && !BC->HasRelocations) {
2011	BC->errs() << "BOLT-WARNING: hot text is disabled in non-relocation mode\n";
2012	opts::HotText = false;
2013	}
2014
2015	if (opts::HotText && opts::HotTextMoveSections.getNumOccurrences() == 0) {
2016	opts::HotTextMoveSections.addValue(V: ".stub");
2017	opts::HotTextMoveSections.addValue(V: ".mover");
2018	opts::HotTextMoveSections.addValue(V: ".never_hugify");
2019	}
2020
2021	if (opts::UseOldText && !BC->OldTextSectionAddress) {
2022	BC->errs()
2023	<< "BOLT-WARNING: cannot use old .text as the section was not found"
2024	"\n";
2025	opts::UseOldText = false;
2026	}
2027	if (opts::UseOldText && !BC->HasRelocations) {
2028	BC->errs() << "BOLT-WARNING: cannot use old .text in non-relocation mode\n";
2029	opts::UseOldText = false;
2030	}
2031
2032	if (!opts::AlignText.getNumOccurrences())
2033	opts::AlignText = BC->PageAlign;
2034
2035	if (opts::AlignText < opts::AlignFunctions)
2036	opts::AlignText = (unsigned)opts::AlignFunctions;
2037
2038	if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode &&
2039	!opts::UseOldText)
2040	opts::Lite = true;
2041
2042	if (opts::Lite && opts::UseOldText) {
2043	BC->errs() << "BOLT-WARNING: cannot combine -lite with -use-old-text. "
2044	"Disabling -use-old-text.\n";
2045	opts::UseOldText = false;
2046	}
2047
2048	if (opts::Lite && opts::StrictMode) {
2049	BC->errs()
2050	<< "BOLT-ERROR: -strict and -lite cannot be used at the same time\n";
2051	exit(status: 1);
2052	}
2053
2054	if (opts::Lite)
2055	BC->outs() << "BOLT-INFO: enabling lite mode\n";
2056
2057	if (BC->IsLinuxKernel) {
2058	if (!opts::KeepNops.getNumOccurrences())
2059	opts::KeepNops = true;
2060
2061	// Linux kernel may resume execution after a trap instruction in some cases.
2062	if (!opts::TerminalTrap.getNumOccurrences())
2063	opts::TerminalTrap = false;
2064	}
2065	}
2066
2067	namespace {
2068	template <typename ELFT>
2069	int64_t getRelocationAddend(const ELFObjectFile<ELFT> *Obj,
2070	const RelocationRef &RelRef) {
2071	using ELFShdrTy = typename ELFT::Shdr;
2072	using Elf_Rela = typename ELFT::Rela;
2073	int64_t Addend = 0;
2074	const ELFFile<ELFT> &EF = Obj->getELFFile();
2075	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2076	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2077	switch (RelocationSection->sh_type) {
2078	default:
2079	llvm_unreachable("unexpected relocation section type");
2080	case ELF::SHT_REL:
2081	break;
2082	case ELF::SHT_RELA: {
2083	const Elf_Rela *RelA = Obj->getRela(Rel);
2084	Addend = RelA->r_addend;
2085	break;
2086	}
2087	}
2088
2089	return Addend;
2090	}
2091
2092	int64_t getRelocationAddend(const ELFObjectFileBase *Obj,
2093	const RelocationRef &Rel) {
2094	return getRelocationAddend(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2095	}
2096
2097	template <typename ELFT>
2098	uint32_t getRelocationSymbol(const ELFObjectFile<ELFT> *Obj,
2099	const RelocationRef &RelRef) {
2100	using ELFShdrTy = typename ELFT::Shdr;
2101	uint32_t Symbol = 0;
2102	const ELFFile<ELFT> &EF = Obj->getELFFile();
2103	DataRefImpl Rel = RelRef.getRawDataRefImpl();
2104	const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));
2105	switch (RelocationSection->sh_type) {
2106	default:
2107	llvm_unreachable("unexpected relocation section type");
2108	case ELF::SHT_REL:
2109	Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL());
2110	break;
2111	case ELF::SHT_RELA:
2112	Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL());
2113	break;
2114	}
2115
2116	return Symbol;
2117	}
2118
2119	uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj,
2120	const RelocationRef &Rel) {
2121	return getRelocationSymbol(Obj: cast<ELF64LEObjectFile>(Val: Obj), RelRef: Rel);
2122	}
2123	} // anonymous namespace
2124
2125	bool RewriteInstance::analyzeRelocation(
2126	const RelocationRef &Rel, uint64_t &RType, std::string &SymbolName,
2127	bool &IsSectionRelocation, uint64_t &SymbolAddress, int64_t &Addend,
2128	uint64_t &ExtractedValue, bool &Skip) const {
2129	Skip = false;
2130	if (!Relocation::isSupported(Type: RType))
2131	return false;
2132
2133	const bool IsAArch64 = BC->isAArch64();
2134
2135	const size_t RelSize = Relocation::getSizeForType(Type: RType);
2136
2137	ErrorOr<uint64_t> Value =
2138	BC->getUnsignedValueAtAddress(Address: Rel.getOffset(), Size: RelSize);
2139	assert(Value && "failed to extract relocated value");
2140	if ((Skip = Relocation::skipRelocationProcess(Type&: RType, Contents: *Value)))
2141	return true;
2142
2143	ExtractedValue = Relocation::extractValue(Type: RType, Contents: *Value, PC: Rel.getOffset());
2144	Addend = getRelocationAddend(Obj: InputFile, Rel);
2145
2146	const bool IsPCRelative = Relocation::isPCRelative(Type: RType);
2147	const uint64_t PCRelOffset = IsPCRelative && !IsAArch64 ? Rel.getOffset() : 0;
2148	bool SkipVerification = false;
2149	auto SymbolIter = Rel.getSymbol();
2150	if (SymbolIter == InputFile->symbol_end()) {
2151	SymbolAddress = ExtractedValue - Addend + PCRelOffset;
2152	MCSymbol *RelSymbol =
2153	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "RELSYMat");
2154	SymbolName = std::string(RelSymbol->getName());
2155	IsSectionRelocation = false;
2156	} else {
2157	const SymbolRef &Symbol = *SymbolIter;
2158	SymbolName = std::string(cantFail(ValOrErr: Symbol.getName()));
2159	SymbolAddress = cantFail(ValOrErr: Symbol.getAddress());
2160	SkipVerification = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Other);
2161	// Section symbols are marked as ST_Debug.
2162	IsSectionRelocation = (cantFail(ValOrErr: Symbol.getType()) == SymbolRef::ST_Debug);
2163	// Check for PLT entry registered with symbol name
2164	if (!SymbolAddress && (IsAArch64 || BC->isRISCV())) {
2165	const BinaryData *BD = BC->getPLTBinaryDataByName(Name: SymbolName);
2166	SymbolAddress = BD ? BD->getAddress() : 0;
2167	}
2168	}
2169	// For PIE or dynamic libs, the linker may choose not to put the relocation
2170	// result at the address if it is a X86_64_64 one because it will emit a
2171	// dynamic relocation (X86_RELATIVE) for the dynamic linker and loader to
2172	// resolve it at run time. The static relocation result goes as the addend
2173	// of the dynamic relocation in this case. We can't verify these cases.
2174	// FIXME: perhaps we can try to find if it really emitted a corresponding
2175	// RELATIVE relocation at this offset with the correct value as the addend.
2176	if (!BC->HasFixedLoadAddress && RelSize == 8)
2177	SkipVerification = true;
2178
2179	if (IsSectionRelocation && !IsAArch64) {
2180	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2181	assert(Section && "section expected for section relocation");
2182	SymbolName = "section " + std::string(Section->getName());
2183	// Convert section symbol relocations to regular relocations inside
2184	// non-section symbols.
2185	if (Section->containsAddress(Address: ExtractedValue) && !IsPCRelative) {
2186	SymbolAddress = ExtractedValue;
2187	Addend = 0;
2188	} else {
2189	Addend = ExtractedValue - (SymbolAddress - PCRelOffset);
2190	}
2191	}
2192
2193	// If no symbol has been found or if it is a relocation requiring the
2194	// creation of a GOT entry, do not link against the symbol but against
2195	// whatever address was extracted from the instruction itself. We are
2196	// not creating a GOT entry as this was already processed by the linker.
2197	// For GOT relocs, do not subtract addend as the addend does not refer
2198	// to this instruction's target, but it refers to the target in the GOT
2199	// entry.
2200	if (Relocation::isGOT(Type: RType)) {
2201	Addend = 0;
2202	SymbolAddress = ExtractedValue + PCRelOffset;
2203	} else if (Relocation::isTLS(Type: RType)) {
2204	SkipVerification = true;
2205	} else if (!SymbolAddress) {
2206	assert(!IsSectionRelocation);
2207	if (ExtractedValue || Addend == 0 || IsPCRelative) {
2208	SymbolAddress =
2209	truncateToSize(Value: ExtractedValue - Addend + PCRelOffset, Bytes: RelSize);
2210	} else {
2211	// This is weird case. The extracted value is zero but the addend is
2212	// non-zero and the relocation is not pc-rel. Using the previous logic,
2213	// the SymbolAddress would end up as a huge number. Seen in
2214	// exceptions_pic.test.
2215	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocation @ 0x"
2216	<< Twine::utohexstr(Rel.getOffset())
2217	<< " value does not match addend for "
2218	<< "relocation to undefined symbol.\n");
2219	return true;
2220	}
2221	}
2222
2223	auto verifyExtractedValue = [&]() {
2224	if (SkipVerification)
2225	return true;
2226
2227	if (IsAArch64 || BC->isRISCV())
2228	return true;
2229
2230	if (SymbolName == "__hot_start" || SymbolName == "__hot_end")
2231	return true;
2232
2233	if (RType == ELF::R_X86_64_PLT32)
2234	return true;
2235
2236	return truncateToSize(Value: ExtractedValue, Bytes: RelSize) ==
2237	truncateToSize(Value: SymbolAddress + Addend - PCRelOffset, Bytes: RelSize);
2238	};
2239
2240	(void)verifyExtractedValue;
2241	assert(verifyExtractedValue() && "mismatched extracted relocation value");
2242
2243	return true;
2244	}
2245
2246	void RewriteInstance::processDynamicRelocations() {
2247	// Read .relr.dyn section containing compressed R_*_RELATIVE relocations.
2248	if (DynamicRelrSize > 0) {
2249	ErrorOr<BinarySection &> DynamicRelrSectionOrErr =
2250	BC->getSectionForAddress(Address: *DynamicRelrAddress);
2251	if (!DynamicRelrSectionOrErr)
2252	report_error(Message: "unable to find section corresponding to DT_RELR",
2253	EC: DynamicRelrSectionOrErr.getError());
2254	if (DynamicRelrSectionOrErr->getSize() != DynamicRelrSize)
2255	report_error(Message: "section size mismatch for DT_RELRSZ",
2256	EC: errc::executable_format_error);
2257	readDynamicRelrRelocations(Section&: *DynamicRelrSectionOrErr);
2258	}
2259
2260	// Read relocations for PLT - DT_JMPREL.
2261	if (PLTRelocationsSize > 0) {
2262	ErrorOr<BinarySection &> PLTRelSectionOrErr =
2263	BC->getSectionForAddress(Address: *PLTRelocationsAddress);
2264	if (!PLTRelSectionOrErr)
2265	report_error(Message: "unable to find section corresponding to DT_JMPREL",
2266	EC: PLTRelSectionOrErr.getError());
2267	if (PLTRelSectionOrErr->getSize() != PLTRelocationsSize)
2268	report_error(Message: "section size mismatch for DT_PLTRELSZ",
2269	EC: errc::executable_format_error);
2270	readDynamicRelocations(Section: PLTRelSectionOrErr->getSectionRef(),
2271	/*IsJmpRel*/ true);
2272	}
2273
2274	// The rest of dynamic relocations - DT_RELA.
2275	// The static executable might have .rela.dyn secion and not have PT_DYNAMIC
2276	if (!DynamicRelocationsSize && BC->IsStaticExecutable) {
2277	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2278	BC->getUniqueSectionByName(SectionName: getRelaDynSectionName());
2279	if (DynamicRelSectionOrErr) {
2280	DynamicRelocationsAddress = DynamicRelSectionOrErr->getAddress();
2281	DynamicRelocationsSize = DynamicRelSectionOrErr->getSize();
2282	const SectionRef &SectionRef = DynamicRelSectionOrErr->getSectionRef();
2283	DynamicRelativeRelocationsCount = std::distance(
2284	first: SectionRef.relocation_begin(), last: SectionRef.relocation_end());
2285	}
2286	}
2287
2288	if (DynamicRelocationsSize > 0) {
2289	ErrorOr<BinarySection &> DynamicRelSectionOrErr =
2290	BC->getSectionForAddress(Address: *DynamicRelocationsAddress);
2291	if (!DynamicRelSectionOrErr)
2292	report_error(Message: "unable to find section corresponding to DT_RELA",
2293	EC: DynamicRelSectionOrErr.getError());
2294	auto DynamicRelSectionSize = DynamicRelSectionOrErr->getSize();
2295	// On RISC-V DT_RELASZ seems to include both .rela.dyn and .rela.plt
2296	if (DynamicRelocationsSize == DynamicRelSectionSize + PLTRelocationsSize)
2297	DynamicRelocationsSize = DynamicRelSectionSize;
2298	if (DynamicRelSectionSize != DynamicRelocationsSize)
2299	report_error(Message: "section size mismatch for DT_RELASZ",
2300	EC: errc::executable_format_error);
2301	readDynamicRelocations(Section: DynamicRelSectionOrErr->getSectionRef(),
2302	/*IsJmpRel*/ false);
2303	}
2304	}
2305
2306	void RewriteInstance::processRelocations() {
2307	if (!BC->HasRelocations)
2308	return;
2309
2310	for (const SectionRef &Section : InputFile->sections()) {
2311	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2312	if (SecIter == InputFile->section_end())
2313	continue;
2314	if (BinarySection(*BC, Section).isAllocatable())
2315	continue;
2316
2317	readRelocations(Section);
2318	}
2319
2320	if (NumFailedRelocations)
2321	BC->errs() << "BOLT-WARNING: Failed to analyze " << NumFailedRelocations
2322	<< " relocations\n";
2323	}
2324
2325	void RewriteInstance::readDynamicRelocations(const SectionRef &Section,
2326	bool IsJmpRel) {
2327	assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected");
2328
2329	LLVM_DEBUG({
2330	StringRef SectionName = cantFail(Section.getName());
2331	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2332	<< ":\n";
2333	});
2334
2335	for (const RelocationRef &Rel : Section.relocations()) {
2336	const uint64_t RType = Rel.getType();
2337	if (Relocation::isNone(Type: RType))
2338	continue;
2339
2340	StringRef SymbolName = "<none>";
2341	MCSymbol *Symbol = nullptr;
2342	uint64_t SymbolAddress = 0;
2343	const uint64_t Addend = getRelocationAddend(Obj: InputFile, Rel);
2344
2345	symbol_iterator SymbolIter = Rel.getSymbol();
2346	if (SymbolIter != InputFile->symbol_end()) {
2347	SymbolName = cantFail(ValOrErr: SymbolIter->getName());
2348	BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName);
2349	Symbol = BD ? BD->getSymbol()
2350	: BC->getOrCreateUndefinedGlobalSymbol(Name: SymbolName);
2351	SymbolAddress = cantFail(ValOrErr: SymbolIter->getAddress());
2352	(void)SymbolAddress;
2353	}
2354
2355	LLVM_DEBUG(
2356	SmallString<16> TypeName;
2357	Rel.getTypeName(TypeName);
2358	dbgs() << "BOLT-DEBUG: dynamic relocation at 0x"
2359	<< Twine::utohexstr(Rel.getOffset()) << " : " << TypeName
2360	<< " : " << SymbolName << " : " << Twine::utohexstr(SymbolAddress)
2361	<< " : + 0x" << Twine::utohexstr(Addend) << '\n'
2362	);
2363
2364	if (IsJmpRel)
2365	IsJmpRelocation[RType] = true;
2366
2367	if (Symbol)
2368	SymbolIndex[Symbol] = getRelocationSymbol(Obj: InputFile, Rel);
2369
2370	BC->addDynamicRelocation(Address: Rel.getOffset(), Symbol, Type: RType, Addend);
2371	}
2372	}
2373
2374	void RewriteInstance::readDynamicRelrRelocations(BinarySection &Section) {
2375	assert(Section.isAllocatable() && "allocatable expected");
2376
2377	LLVM_DEBUG({
2378	StringRef SectionName = Section.getName();
2379	dbgs() << "BOLT-DEBUG: reading relocations in section " << SectionName
2380	<< ":\n";
2381	});
2382
2383	const uint64_t RType = Relocation::getRelative();
2384	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
2385	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
2386
2387	auto ExtractAddendValue = [&](uint64_t Address) -> uint64_t {
2388	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
2389	assert(Section && "cannot get section for data address from RELR");
2390	DataExtractor DE = DataExtractor(Section->getContents(),
2391	BC->AsmInfo->isLittleEndian(), PSize);
2392	uint64_t Offset = Address - Section->getAddress();
2393	return DE.getUnsigned(offset_ptr: &Offset, byte_size: PSize);
2394	};
2395
2396	auto AddRelocation = [&](uint64_t Address) {
2397	uint64_t Addend = ExtractAddendValue(Address);
2398	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: R_*_RELATIVE relocation at 0x"
2399	<< Twine::utohexstr(Address) << " to 0x"
2400	<< Twine::utohexstr(Addend) << '\n';);
2401	BC->addDynamicRelocation(Address, Symbol: nullptr, Type: RType, Addend);
2402	};
2403
2404	DataExtractor DE = DataExtractor(Section.getContents(),
2405	BC->AsmInfo->isLittleEndian(), PSize);
2406	uint64_t Offset = 0, Address = 0;
2407	uint64_t RelrCount = DynamicRelrSize / DynamicRelrEntrySize;
2408	while (RelrCount--) {
2409	assert(DE.isValidOffset(Offset));
2410	uint64_t Entry = DE.getUnsigned(offset_ptr: &Offset, byte_size: DynamicRelrEntrySize);
2411	if ((Entry & 1) == 0) {
2412	AddRelocation(Entry);
2413	Address = Entry + PSize;
2414	} else {
2415	const uint64_t StartAddress = Address;
2416	while (Entry >>= 1) {
2417	if (Entry & 1)
2418	AddRelocation(Address);
2419
2420	Address += PSize;
2421	}
2422
2423	Address = StartAddress + MaxDelta;
2424	}
2425	}
2426	}
2427
2428	void RewriteInstance::printRelocationInfo(const RelocationRef &Rel,
2429	StringRef SymbolName,
2430	uint64_t SymbolAddress,
2431	uint64_t Addend,
2432	uint64_t ExtractedValue) const {
2433	SmallString<16> TypeName;
2434	Rel.getTypeName(Result&: TypeName);
2435	const uint64_t Address = SymbolAddress + Addend;
2436	const uint64_t Offset = Rel.getOffset();
2437	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address: SymbolAddress);
2438	BinaryFunction *Func =
2439	BC->getBinaryFunctionContainingAddress(Address: Offset, CheckPastEnd: false, UseMaxSize: BC->isAArch64());
2440	dbgs() << formatv(Fmt: "Relocation: offset = {0:x}; type = {1}; value = {2:x}; ",
2441	Vals: Offset, Vals&: TypeName, Vals&: ExtractedValue)
2442	<< formatv(Fmt: "symbol = {0} ({1}); symbol address = {2:x}; ", Vals&: SymbolName,
2443	Vals: Section ? Section->getName() : "", Vals&: SymbolAddress)
2444	<< formatv(Fmt: "addend = {0:x}; address = {1:x}; in = ", Vals&: Addend, Vals: Address);
2445	if (Func)
2446	dbgs() << Func->getPrintName();
2447	else
2448	dbgs() << BC->getSectionForAddress(Address: Rel.getOffset())->getName();
2449	dbgs() << '\n';
2450	}
2451
2452	void RewriteInstance::readRelocations(const SectionRef &Section) {
2453	LLVM_DEBUG({
2454	StringRef SectionName = cantFail(Section.getName());
2455	dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName
2456	<< ":\n";
2457	});
2458	if (BinarySection(*BC, Section).isAllocatable()) {
2459	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring runtime relocations\n");
2460	return;
2461	}
2462	section_iterator SecIter = cantFail(ValOrErr: Section.getRelocatedSection());
2463	assert(SecIter != InputFile->section_end() && "relocated section expected");
2464	SectionRef RelocatedSection = *SecIter;
2465
2466	StringRef RelocatedSectionName = cantFail(ValOrErr: RelocatedSection.getName());
2467	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is "
2468	<< RelocatedSectionName << '\n');
2469
2470	if (!BinarySection(*BC, RelocatedSection).isAllocatable()) {
2471	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocations against "
2472	<< "non-allocatable section\n");
2473	return;
2474	}
2475	const bool SkipRelocs = StringSwitch<bool>(RelocatedSectionName)
2476	.Cases(S0: ".plt", S1: ".rela.plt", S2: ".got.plt",
2477	S3: ".eh_frame", S4: ".gcc_except_table", Value: true)
2478	.Default(Value: false);
2479	if (SkipRelocs) {
2480	LLVM_DEBUG(
2481	dbgs() << "BOLT-DEBUG: ignoring relocations against known section\n");
2482	return;
2483	}
2484
2485	for (const RelocationRef &Rel : Section.relocations())
2486	handleRelocation(RelocatedSection, Rel);
2487	}
2488
2489	void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
2490	const RelocationRef &Rel) {
2491	const bool IsAArch64 = BC->isAArch64();
2492	const bool IsFromCode = RelocatedSection.isText();
2493
2494	SmallString<16> TypeName;
2495	Rel.getTypeName(Result&: TypeName);
2496	uint64_t RType = Rel.getType();
2497	if (Relocation::skipRelocationType(Type: RType))
2498	return;
2499
2500	// Adjust the relocation type as the linker might have skewed it.
2501	if (BC->isX86() && (RType & ELF::R_X86_64_converted_reloc_bit)) {
2502	if (opts::Verbosity >= 1)
2503	dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n";
2504	RType &= ~ELF::R_X86_64_converted_reloc_bit;
2505	}
2506
2507	if (Relocation::isTLS(Type: RType)) {
2508	// No special handling required for TLS relocations on X86.
2509	if (BC->isX86())
2510	return;
2511
2512	// The non-got related TLS relocations on AArch64 and RISC-V also could be
2513	// skipped.
2514	if (!Relocation::isGOT(Type: RType))
2515	return;
2516	}
2517
2518	if (!IsAArch64 && BC->getDynamicRelocationAt(Address: Rel.getOffset())) {
2519	LLVM_DEBUG({
2520	dbgs() << formatv("BOLT-DEBUG: address {0:x} has a ", Rel.getOffset())
2521	<< "dynamic relocation against it. Ignoring static relocation.\n";
2522	});
2523	return;
2524	}
2525
2526	std::string SymbolName;
2527	uint64_t SymbolAddress;
2528	int64_t Addend;
2529	uint64_t ExtractedValue;
2530	bool IsSectionRelocation;
2531	bool Skip;
2532	if (!analyzeRelocation(Rel, RType, SymbolName, IsSectionRelocation,
2533	SymbolAddress, Addend, ExtractedValue, Skip)) {
2534	LLVM_DEBUG({
2535	dbgs() << "BOLT-WARNING: failed to analyze relocation @ offset = "
2536	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2537	});
2538	++NumFailedRelocations;
2539	return;
2540	}
2541
2542	if (Skip) {
2543	LLVM_DEBUG({
2544	dbgs() << "BOLT-DEBUG: skipping relocation @ offset = "
2545	<< formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);
2546	});
2547	return;
2548	}
2549
2550	const uint64_t Address = SymbolAddress + Addend;
2551
2552	LLVM_DEBUG({
2553	dbgs() << "BOLT-DEBUG: ";
2554	printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend, ExtractedValue);
2555	});
2556
2557	BinaryFunction *ContainingBF = nullptr;
2558	if (IsFromCode) {
2559	ContainingBF =
2560	BC->getBinaryFunctionContainingAddress(Address: Rel.getOffset(),
2561	/*CheckPastEnd*/ false,
2562	/*UseMaxSize*/ true);
2563	assert(ContainingBF && "cannot find function for address in code");
2564	if (!IsAArch64 && !ContainingBF->containsAddress(PC: Rel.getOffset())) {
2565	if (opts::Verbosity >= 1)
2566	BC->outs() << formatv(
2567	Fmt: "BOLT-INFO: {0} has relocations in padding area\n", Vals&: *ContainingBF);
2568	ContainingBF->setSize(ContainingBF->getMaxSize());
2569	ContainingBF->setSimple(false);
2570	return;
2571	}
2572	}
2573
2574	MCSymbol *ReferencedSymbol = nullptr;
2575	if (!IsSectionRelocation) {
2576	if (BinaryData *BD = BC->getBinaryDataByName(Name: SymbolName))
2577	ReferencedSymbol = BD->getSymbol();
2578	else if (BC->isGOTSymbol(SymName: SymbolName))
2579	if (BinaryData *BD = BC->getGOTSymbol())
2580	ReferencedSymbol = BD->getSymbol();
2581	}
2582
2583	ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};
2584	symbol_iterator SymbolIter = Rel.getSymbol();
2585	if (SymbolIter != InputFile->symbol_end()) {
2586	SymbolRef Symbol = *SymbolIter;
2587	section_iterator Section =
2588	cantFail(ValOrErr: Symbol.getSection(), Msg: "cannot get symbol section");
2589	if (Section != InputFile->section_end()) {
2590	Expected<StringRef> SectionName = Section->getName();
2591	if (SectionName && !SectionName->empty())
2592	ReferencedSection = BC->getUniqueSectionByName(SectionName: *SectionName);
2593	} else if (ReferencedSymbol && ContainingBF &&
2594	(cantFail(ValOrErr: Symbol.getFlags()) & SymbolRef::SF_Absolute)) {
2595	// This might be a relocation for an ABS symbols like __global_pointer$ on
2596	// RISC-V
2597	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol,
2598	RelType: Rel.getType(), Addend: 0,
2599	Value: cantFail(ValOrErr: Symbol.getValue()));
2600	return;
2601	}
2602	}
2603
2604	if (!ReferencedSection)
2605	ReferencedSection = BC->getSectionForAddress(Address: SymbolAddress);
2606
2607	const bool IsToCode = ReferencedSection && ReferencedSection->isText();
2608
2609	// Special handling of PC-relative relocations.
2610	if (BC->isX86() && Relocation::isPCRelative(Type: RType)) {
2611	if (!IsFromCode && IsToCode) {
2612	// PC-relative relocations from data to code are tricky since the
2613	// original information is typically lost after linking, even with
2614	// '--emit-relocs'. Such relocations are normally used by PIC-style
2615	// jump tables and they reference both the jump table and jump
2616	// targets by computing the difference between the two. If we blindly
2617	// apply the relocation, it will appear that it references an arbitrary
2618	// location in the code, possibly in a different function from the one
2619	// containing the jump table.
2620	//
2621	// For that reason, we only register the fact that there is a
2622	// PC-relative relocation at a given address against the code.
2623	// The actual referenced label/address will be determined during jump
2624	// table analysis.
2625	BC->addPCRelativeDataRelocation(Address: Rel.getOffset());
2626	} else if (ContainingBF && !IsSectionRelocation && ReferencedSymbol) {
2627	// If we know the referenced symbol, register the relocation from
2628	// the code. It's required to properly handle cases where
2629	// "symbol + addend" references an object different from "symbol".
2630	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
2631	Addend, Value: ExtractedValue);
2632	} else {
2633	LLVM_DEBUG({
2634	dbgs() << "BOLT-DEBUG: not creating PC-relative relocation at"
2635	<< formatv("{0:x} for {1}\n", Rel.getOffset(), SymbolName);
2636	});
2637	}
2638
2639	return;
2640	}
2641
2642	bool ForceRelocation = BC->forceSymbolRelocations(SymbolName);
2643	if ((BC->isAArch64() || BC->isRISCV()) && Relocation::isGOT(Type: RType))
2644	ForceRelocation = true;
2645
2646	if (!ReferencedSection && !ForceRelocation) {
2647	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: cannot determine referenced section.\n");
2648	return;
2649	}
2650
2651	// Occasionally we may see a reference past the last byte of the function
2652	// typically as a result of __builtin_unreachable(). Check it here.
2653	BinaryFunction *ReferencedBF = BC->getBinaryFunctionContainingAddress(
2654	Address, /*CheckPastEnd*/ true, /*UseMaxSize*/ IsAArch64);
2655
2656	if (!IsSectionRelocation) {
2657	if (BinaryFunction *BF =
2658	BC->getBinaryFunctionContainingAddress(Address: SymbolAddress)) {
2659	if (BF != ReferencedBF) {
2660	// It's possible we are referencing a function without referencing any
2661	// code, e.g. when taking a bitmask action on a function address.
2662	BC->errs()
2663	<< "BOLT-WARNING: non-standard function reference (e.g. bitmask)"
2664	<< formatv(Fmt: " detected against function {0} from ", Vals&: *BF);
2665	if (IsFromCode)
2666	BC->errs() << formatv(Fmt: "function {0}\n", Vals&: *ContainingBF);
2667	else
2668	BC->errs() << formatv(Fmt: "data section at {0:x}\n", Vals: Rel.getOffset());
2669	LLVM_DEBUG(printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend,
2670	ExtractedValue));
2671	ReferencedBF = BF;
2672	}
2673	}
2674	} else if (ReferencedBF) {
2675	assert(ReferencedSection && "section expected for section relocation");
2676	if (*ReferencedBF->getOriginSection() != *ReferencedSection) {
2677	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring false function reference\n");
2678	ReferencedBF = nullptr;
2679	}
2680	}
2681
2682	// Workaround for a member function pointer de-virtualization bug. We check
2683	// if a non-pc-relative relocation in the code is pointing to (fptr - 1).
2684	if (IsToCode && ContainingBF && !Relocation::isPCRelative(Type: RType) &&
2685	(!ReferencedBF || (ReferencedBF->getAddress() != Address))) {
2686	if (const BinaryFunction *RogueBF =
2687	BC->getBinaryFunctionAtAddress(Address: Address + 1)) {
2688	// Do an extra check that the function was referenced previously.
2689	// It's a linear search, but it should rarely happen.
2690	auto CheckReloc = [&](const Relocation &Rel) {
2691	return Rel.Symbol == RogueBF->getSymbol() &&
2692	!Relocation::isPCRelative(Type: Rel.Type);
2693	};
2694	bool Found = llvm::any_of(
2695	Range: llvm::make_second_range(c&: ContainingBF->Relocations), P: CheckReloc);
2696
2697	if (Found) {
2698	BC->errs()
2699	<< "BOLT-WARNING: detected possible compiler de-virtualization "
2700	"bug: -1 addend used with non-pc-relative relocation against "
2701	<< formatv(Fmt: "function {0} in function {1}\n", Vals: *RogueBF,
2702	Vals&: *ContainingBF);
2703	return;
2704	}
2705	}
2706	}
2707
2708	if (ForceRelocation) {
2709	std::string Name =
2710	Relocation::isGOT(Type: RType) ? "__BOLT_got_zero" : SymbolName;
2711	ReferencedSymbol = BC->registerNameAtAddress(Name, Address: 0, Size: 0, Alignment: 0);
2712	SymbolAddress = 0;
2713	if (Relocation::isGOT(Type: RType))
2714	Addend = Address;
2715	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: forcing relocation against symbol "
2716	<< SymbolName << " with addend " << Addend << '\n');
2717	} else if (ReferencedBF) {
2718	ReferencedSymbol = ReferencedBF->getSymbol();
2719	uint64_t RefFunctionOffset = 0;
2720
2721	// Adjust the point of reference to a code location inside a function.
2722	if (ReferencedBF->containsAddress(PC: Address, /*UseMaxSize = */ true)) {
2723	RefFunctionOffset = Address - ReferencedBF->getAddress();
2724	if (Relocation::isInstructionReference(Type: RType)) {
2725	// Instruction labels are created while disassembling so we just leave
2726	// the symbol empty for now. Since the extracted value is typically
2727	// unrelated to the referenced symbol (e.g., %pcrel_lo in RISC-V
2728	// references an instruction but the patched value references the low
2729	// bits of a data address), we set the extracted value to the symbol
2730	// address in order to be able to correctly reconstruct the reference
2731	// later.
2732	ReferencedSymbol = nullptr;
2733	ExtractedValue = Address;
2734	} else if (RefFunctionOffset) {
2735	if (ContainingBF && ContainingBF != ReferencedBF) {
2736	ReferencedSymbol =
2737	ReferencedBF->addEntryPointAtOffset(Offset: RefFunctionOffset);
2738	} else {
2739	ReferencedSymbol =
2740	ReferencedBF->getOrCreateLocalLabel(Address,
2741	/*CreatePastEnd =*/true);
2742
2743	// If ContainingBF != nullptr, it equals ReferencedBF (see
2744	// if-condition above) so we're handling a relocation from a function
2745	// to itself. RISC-V uses such relocations for branches, for example.
2746	// These should not be registered as externally references offsets.
2747	if (!ContainingBF)
2748	ReferencedBF->registerReferencedOffset(Offset: RefFunctionOffset);
2749	}
2750	if (opts::Verbosity > 1 &&
2751	BinarySection(*BC, RelocatedSection).isWritable())
2752	BC->errs()
2753	<< "BOLT-WARNING: writable reference into the middle of the "
2754	<< formatv(Fmt: "function {0} detected at address {1:x}\n",
2755	Vals&: *ReferencedBF, Vals: Rel.getOffset());
2756	}
2757	SymbolAddress = Address;
2758	Addend = 0;
2759	}
2760	LLVM_DEBUG({
2761	dbgs() << " referenced function " << *ReferencedBF;
2762	if (Address != ReferencedBF->getAddress())
2763	dbgs() << formatv(" at offset {0:x}", RefFunctionOffset);
2764	dbgs() << '\n';
2765	});
2766	} else {
2767	if (IsToCode && SymbolAddress) {
2768	// This can happen e.g. with PIC-style jump tables.
2769	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: no corresponding function for "
2770	"relocation against code\n");
2771	}
2772
2773	// In AArch64 there are zero reasons to keep a reference to the
2774	// "original" symbol plus addend. The original symbol is probably just a
2775	// section symbol. If we are here, this means we are probably accessing
2776	// data, so it is imperative to keep the original address.
2777	if (IsAArch64) {
2778	SymbolName = formatv(Fmt: "SYMBOLat{0:x}", Vals: Address);
2779	SymbolAddress = Address;
2780	Addend = 0;
2781	}
2782
2783	if (BinaryData *BD = BC->getBinaryDataContainingAddress(Address: SymbolAddress)) {
2784	// Note: this assertion is trying to check sanity of BinaryData objects
2785	// but AArch64 has inferred and incomplete object locations coming from
2786	// GOT/TLS or any other non-trivial relocation (that requires creation
2787	// of sections and whose symbol address is not really what should be
2788	// encoded in the instruction). So we essentially disabled this check
2789	// for AArch64 and live with bogus names for objects.
2790	assert((IsAArch64 || IsSectionRelocation ||
2791	BD->nameStartsWith(SymbolName) ||
2792	BD->nameStartsWith("PG" + SymbolName) ||
2793	(BD->nameStartsWith("ANONYMOUS") &&
2794	(BD->getSectionName().starts_with(".plt") ||
2795	BD->getSectionName().ends_with(".plt")))) &&
2796	"BOLT symbol names of all non-section relocations must match up "
2797	"with symbol names referenced in the relocation");
2798
2799	if (IsSectionRelocation)
2800	BC->markAmbiguousRelocations(BD&: *BD, Address);
2801
2802	ReferencedSymbol = BD->getSymbol();
2803	Addend += (SymbolAddress - BD->getAddress());
2804	SymbolAddress = BD->getAddress();
2805	assert(Address == SymbolAddress + Addend);
2806	} else {
2807	// These are mostly local data symbols but undefined symbols
2808	// in relocation sections can get through here too, from .plt.
2809	assert(
2810	(IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
2811	BC->getSectionNameForAddress(SymbolAddress)->starts_with(".plt")) &&
2812	"known symbols should not resolve to anonymous locals");
2813
2814	if (IsSectionRelocation) {
2815	ReferencedSymbol =
2816	BC->getOrCreateGlobalSymbol(Address: SymbolAddress, Prefix: "SYMBOLat");
2817	} else {
2818	SymbolRef Symbol = *Rel.getSymbol();
2819	const uint64_t SymbolSize =
2820	IsAArch64 ? 0 : ELFSymbolRef(Symbol).getSize();
2821	const uint64_t SymbolAlignment = IsAArch64 ? 1 : Symbol.getAlignment();
2822	const uint32_t SymbolFlags = cantFail(ValOrErr: Symbol.getFlags());
2823	std::string Name;
2824	if (SymbolFlags & SymbolRef::SF_Global) {
2825	Name = SymbolName;
2826	} else {
2827	if (StringRef(SymbolName)
2828	.starts_with(Prefix: BC->AsmInfo->getPrivateGlobalPrefix()))
2829	Name = NR.uniquify(Name: "PG" + SymbolName);
2830	else
2831	Name = NR.uniquify(Name: SymbolName);
2832	}
2833	ReferencedSymbol = BC->registerNameAtAddress(
2834	Name, Address: SymbolAddress, Size: SymbolSize, Alignment: SymbolAlignment, Flags: SymbolFlags);
2835	}
2836
2837	if (IsSectionRelocation) {
2838	BinaryData *BD = BC->getBinaryDataByName(Name: ReferencedSymbol->getName());
2839	BC->markAmbiguousRelocations(BD&: *BD, Address);
2840	}
2841	}
2842	}
2843
2844	auto checkMaxDataRelocations = [&]() {
2845	++NumDataRelocations;
2846	LLVM_DEBUG(if (opts::MaxDataRelocations &&
2847	NumDataRelocations + 1 == opts::MaxDataRelocations) {
2848	dbgs() << "BOLT-DEBUG: processing ending on data relocation "
2849	<< NumDataRelocations << ": ";
2850	printRelocationInfo(Rel, ReferencedSymbol->getName(), SymbolAddress,
2851	Addend, ExtractedValue);
2852	});
2853
2854	return (!opts::MaxDataRelocations ||
2855	NumDataRelocations < opts::MaxDataRelocations);
2856	};
2857
2858	if ((ReferencedSection && refersToReorderedSection(Section: ReferencedSection)) ||
2859	(opts::ForceToDataRelocations && checkMaxDataRelocations()) ||
2860	// RISC-V has ADD/SUB data-to-data relocations
2861	BC->isRISCV())
2862	ForceRelocation = true;
2863
2864	if (IsFromCode)
2865	ContainingBF->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, RelType: RType,
2866	Addend, Value: ExtractedValue);
2867	else if (IsToCode || ForceRelocation)
2868	BC->addRelocation(Address: Rel.getOffset(), Symbol: ReferencedSymbol, Type: RType, Addend,
2869	Value: ExtractedValue);
2870	else
2871	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");
2872	}
2873
2874	void RewriteInstance::selectFunctionsToProcess() {
2875	// Extend the list of functions to process or skip from a file.
2876	auto populateFunctionNames = [](cl::opt<std::string> &FunctionNamesFile,
2877	cl::list<std::string> &FunctionNames) {
2878	if (FunctionNamesFile.empty())
2879	return;
2880	std::ifstream FuncsFile(FunctionNamesFile, std::ios::in);
2881	std::string FuncName;
2882	while (std::getline(is&: FuncsFile, str&: FuncName))
2883	FunctionNames.push_back(value: FuncName);
2884	};
2885	populateFunctionNames(opts::FunctionNamesFile, opts::ForceFunctionNames);
2886	populateFunctionNames(opts::SkipFunctionNamesFile, opts::SkipFunctionNames);
2887	populateFunctionNames(opts::FunctionNamesFileNR, opts::ForceFunctionNamesNR);
2888
2889	// Make a set of functions to process to speed up lookups.
2890	std::unordered_set<std::string> ForceFunctionsNR(
2891	opts::ForceFunctionNamesNR.begin(), opts::ForceFunctionNamesNR.end());
2892
2893	if ((!opts::ForceFunctionNames.empty() ||
2894	!opts::ForceFunctionNamesNR.empty()) &&
2895	!opts::SkipFunctionNames.empty()) {
2896	BC->errs()
2897	<< "BOLT-ERROR: cannot select functions to process and skip at the "
2898	"same time. Please use only one type of selection.\n";
2899	exit(status: 1);
2900	}
2901
2902	uint64_t LiteThresholdExecCount = 0;
2903	if (opts::LiteThresholdPct) {
2904	if (opts::LiteThresholdPct > 100)
2905	opts::LiteThresholdPct = 100;
2906
2907	std::vector<const BinaryFunction *> TopFunctions;
2908	for (auto &BFI : BC->getBinaryFunctions()) {
2909	const BinaryFunction &Function = BFI.second;
2910	if (ProfileReader->mayHaveProfileData(BF: Function))
2911	TopFunctions.push_back(x: &Function);
2912	}
2913	llvm::sort(
2914	C&: TopFunctions, Comp: [](const BinaryFunction *A, const BinaryFunction *B) {
2915	return A->getKnownExecutionCount() < B->getKnownExecutionCount();
2916	});
2917
2918	size_t Index = TopFunctions.size() * opts::LiteThresholdPct / 100;
2919	if (Index)
2920	--Index;
2921	LiteThresholdExecCount = TopFunctions[Index]->getKnownExecutionCount();
2922	BC->outs() << "BOLT-INFO: limiting processing to functions with at least "
2923	<< LiteThresholdExecCount << " invocations\n";
2924	}
2925	LiteThresholdExecCount = std::max(
2926	a: LiteThresholdExecCount, b: static_cast<uint64_t>(opts::LiteThresholdCount));
2927
2928	StringSet<> ReorderFunctionsUserSet;
2929	StringSet<> ReorderFunctionsLTOCommonSet;
2930	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
2931	std::vector<std::string> FunctionNames;
2932	BC->logBOLTErrorsAndQuitOnFatal(
2933	E: ReorderFunctions::readFunctionOrderFile(FunctionNames));
2934	for (const std::string &Function : FunctionNames) {
2935	ReorderFunctionsUserSet.insert(key: Function);
2936	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name: Function))
2937	ReorderFunctionsLTOCommonSet.insert(key: *LTOCommonName);
2938	}
2939	}
2940
2941	uint64_t NumFunctionsToProcess = 0;
2942	auto mustSkip = [&](const BinaryFunction &Function) {
2943	if (opts::MaxFunctions.getNumOccurrences() &&
2944	NumFunctionsToProcess >= opts::MaxFunctions)
2945	return true;
2946	for (std::string &Name : opts::SkipFunctionNames)
2947	if (Function.hasNameRegex(NameRegex: Name))
2948	return true;
2949
2950	return false;
2951	};
2952
2953	auto shouldProcess = [&](const BinaryFunction &Function) {
2954	if (mustSkip(Function))
2955	return false;
2956
2957	// If the list is not empty, only process functions from the list.
2958	if (!opts::ForceFunctionNames.empty() || !ForceFunctionsNR.empty()) {
2959	// Regex check (-funcs and -funcs-file options).
2960	for (std::string &Name : opts::ForceFunctionNames)
2961	if (Function.hasNameRegex(NameRegex: Name))
2962	return true;
2963
2964	// Non-regex check (-funcs-no-regex and -funcs-file-no-regex).
2965	for (const StringRef Name : Function.getNames())
2966	if (ForceFunctionsNR.count(x: Name.str()))
2967	return true;
2968
2969	return false;
2970	}
2971
2972	if (opts::Lite) {
2973	// Forcibly include functions specified in the -function-order file.
2974	if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {
2975	for (const StringRef Name : Function.getNames())
2976	if (ReorderFunctionsUserSet.contains(key: Name))
2977	return true;
2978	for (const StringRef Name : Function.getNames())
2979	if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name))
2980	if (ReorderFunctionsLTOCommonSet.contains(key: *LTOCommonName))
2981	return true;
2982	}
2983
2984	if (ProfileReader && !ProfileReader->mayHaveProfileData(BF: Function))
2985	return false;
2986
2987	if (Function.getKnownExecutionCount() < LiteThresholdExecCount)
2988	return false;
2989	}
2990
2991	return true;
2992	};
2993
2994	for (auto &BFI : BC->getBinaryFunctions()) {
2995	BinaryFunction &Function = BFI.second;
2996
2997	// Pseudo functions are explicitly marked by us not to be processed.
2998	if (Function.isPseudo()) {
2999	Function.IsIgnored = true;
3000	Function.HasExternalRefRelocations = true;
3001	continue;
3002	}
3003
3004	// Decide what to do with fragments after parent functions are processed.
3005	if (Function.isFragment())
3006	continue;
3007
3008	if (!shouldProcess(Function)) {
3009	if (opts::Verbosity >= 1) {
3010	BC->outs() << "BOLT-INFO: skipping processing " << Function
3011	<< " per user request\n";
3012	}
3013	Function.setIgnored();
3014	} else {
3015	++NumFunctionsToProcess;
3016	if (opts::MaxFunctions.getNumOccurrences() &&
3017	NumFunctionsToProcess == opts::MaxFunctions)
3018	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3019	}
3020	}
3021
3022	if (!BC->HasSplitFunctions)
3023	return;
3024
3025	// Fragment overrides:
3026	// - If the fragment must be skipped, then the parent must be skipped as well.
3027	// Otherwise, fragment should follow the parent function:
3028	// - if the parent is skipped, skip fragment,
3029	// - if the parent is processed, process the fragment(s) as well.
3030	for (auto &BFI : BC->getBinaryFunctions()) {
3031	BinaryFunction &Function = BFI.second;
3032	if (!Function.isFragment())
3033	continue;
3034	if (mustSkip(Function)) {
3035	for (BinaryFunction *Parent : Function.ParentFragments) {
3036	if (opts::Verbosity >= 1) {
3037	BC->outs() << "BOLT-INFO: skipping processing " << *Parent
3038	<< " together with fragment function\n";
3039	}
3040	Parent->setIgnored();
3041	--NumFunctionsToProcess;
3042	}
3043	Function.setIgnored();
3044	continue;
3045	}
3046
3047	bool IgnoredParent =
3048	llvm::any_of(Range&: Function.ParentFragments, P: [&](BinaryFunction *Parent) {
3049	return Parent->isIgnored();
3050	});
3051	if (IgnoredParent) {
3052	if (opts::Verbosity >= 1) {
3053	BC->outs() << "BOLT-INFO: skipping processing " << Function
3054	<< " together with parent function\n";
3055	}
3056	Function.setIgnored();
3057	} else {
3058	++NumFunctionsToProcess;
3059	if (opts::Verbosity >= 1) {
3060	BC->outs() << "BOLT-INFO: processing " << Function
3061	<< " as a sibling of non-ignored function\n";
3062	}
3063	if (opts::MaxFunctions && NumFunctionsToProcess == opts::MaxFunctions)
3064	BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';
3065	}
3066	}
3067	}
3068
3069	void RewriteInstance::readDebugInfo() {
3070	NamedRegionTimer T("readDebugInfo", "read debug info", TimerGroupName,
3071	TimerGroupDesc, opts::TimeRewrite);
3072	if (!opts::UpdateDebugSections)
3073	return;
3074
3075	BC->preprocessDebugInfo();
3076	}
3077
3078	void RewriteInstance::preprocessProfileData() {
3079	if (!ProfileReader)
3080	return;
3081
3082	NamedRegionTimer T("preprocessprofile", "pre-process profile data",
3083	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3084
3085	BC->outs() << "BOLT-INFO: pre-processing profile using "
3086	<< ProfileReader->getReaderName() << '\n';
3087
3088	if (BAT->enabledFor(InputFile)) {
3089	BC->outs() << "BOLT-INFO: profile collection done on a binary already "
3090	"processed by BOLT\n";
3091	ProfileReader->setBAT(&*BAT);
3092	}
3093
3094	if (Error E = ProfileReader->preprocessProfile(BC&: *BC.get()))
3095	report_error(Message: "cannot pre-process profile", E: std::move(E));
3096
3097	if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName()) {
3098	BC->errs()
3099	<< "BOLT-ERROR: input binary does not have local file symbols "
3100	"but profile data includes function names with embedded file "
3101	"names. It appears that the input binary was stripped while a "
3102	"profiled binary was not\n";
3103	exit(status: 1);
3104	}
3105	}
3106
3107	void RewriteInstance::initializeMetadataManager() {
3108	if (BC->IsLinuxKernel)
3109	MetadataManager.registerRewriter(Rewriter: createLinuxKernelRewriter(*BC));
3110
3111	MetadataManager.registerRewriter(Rewriter: createPseudoProbeRewriter(*BC));
3112
3113	MetadataManager.registerRewriter(Rewriter: createSDTRewriter(*BC));
3114	}
3115
3116	void RewriteInstance::processMetadataPreCFG() {
3117	initializeMetadataManager();
3118
3119	MetadataManager.runInitializersPreCFG();
3120
3121	processProfileDataPreCFG();
3122	}
3123
3124	void RewriteInstance::processMetadataPostCFG() {
3125	MetadataManager.runInitializersPostCFG();
3126	}
3127
3128	void RewriteInstance::processProfileDataPreCFG() {
3129	if (!ProfileReader)
3130	return;
3131
3132	NamedRegionTimer T("processprofile-precfg", "process profile data pre-CFG",
3133	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3134
3135	if (Error E = ProfileReader->readProfilePreCFG(BC&: *BC.get()))
3136	report_error(Message: "cannot read profile pre-CFG", E: std::move(E));
3137	}
3138
3139	void RewriteInstance::processProfileData() {
3140	if (!ProfileReader)
3141	return;
3142
3143	NamedRegionTimer T("processprofile", "process profile data", TimerGroupName,
3144	TimerGroupDesc, opts::TimeRewrite);
3145
3146	if (Error E = ProfileReader->readProfile(BC&: *BC.get()))
3147	report_error(Message: "cannot read profile", E: std::move(E));
3148
3149	if (opts::PrintProfile || opts::PrintAll) {
3150	for (auto &BFI : BC->getBinaryFunctions()) {
3151	BinaryFunction &Function = BFI.second;
3152	if (Function.empty())
3153	continue;
3154
3155	Function.print(OS&: BC->outs(), Annotation: "after attaching profile");
3156	}
3157	}
3158
3159	if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) {
3160	YAMLProfileWriter PW(opts::SaveProfile);
3161	PW.writeProfile(RI: *this);
3162	}
3163	if (opts::AggregateOnly &&
3164	opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML &&
3165	!BAT->enabledFor(InputFile)) {
3166	YAMLProfileWriter PW(opts::OutputFilename);
3167	PW.writeProfile(RI: *this);
3168	}
3169
3170	// Release memory used by profile reader.
3171	ProfileReader.reset();
3172
3173	if (opts::AggregateOnly)
3174	exit(status: 0);
3175	}
3176
3177	void RewriteInstance::disassembleFunctions() {
3178	NamedRegionTimer T("disassembleFunctions", "disassemble functions",
3179	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3180	for (auto &BFI : BC->getBinaryFunctions()) {
3181	BinaryFunction &Function = BFI.second;
3182
3183	ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
3184	if (!FunctionData) {
3185	BC->errs() << "BOLT-ERROR: corresponding section is non-executable or "
3186	<< "empty for function " << Function << '\n';
3187	exit(status: 1);
3188	}
3189
3190	// Treat zero-sized functions as non-simple ones.
3191	if (Function.getSize() == 0) {
3192	Function.setSimple(false);
3193	continue;
3194	}
3195
3196	// Offset of the function in the file.
3197	const auto *FileBegin =
3198	reinterpret_cast<const uint8_t *>(InputFile->getData().data());
3199	Function.setFileOffset(FunctionData->begin() - FileBegin);
3200
3201	if (!shouldDisassemble(BF: Function)) {
3202	NamedRegionTimer T("scan", "scan functions", "buildfuncs",
3203	"Scan Binary Functions", opts::TimeBuild);
3204	Function.scanExternalRefs();
3205	Function.setSimple(false);
3206	continue;
3207	}
3208
3209	bool DisasmFailed{false};
3210	handleAllErrors(E: Function.disassemble(), Handlers: [&](const BOLTError &E) {
3211	DisasmFailed = true;
3212	if (E.isFatal()) {
3213	E.log(OS&: BC->errs());
3214	exit(status: 1);
3215	}
3216	if (opts::processAllFunctions()) {
3217	BC->errs() << BC->generateBugReportMessage(
3218	Message: "function cannot be properly disassembled. "
3219	"Unable to continue in relocation mode.",
3220	Function);
3221	exit(status: 1);
3222	}
3223	if (opts::Verbosity >= 1)
3224	BC->outs() << "BOLT-INFO: could not disassemble function " << Function
3225	<< ". Will ignore.\n";
3226	// Forcefully ignore the function.
3227	Function.setIgnored();
3228	});
3229
3230	if (DisasmFailed)
3231	continue;
3232
3233	if (opts::PrintAll || opts::PrintDisasm)
3234	Function.print(OS&: BC->outs(), Annotation: "after disassembly");
3235	}
3236
3237	BC->processInterproceduralReferences();
3238	BC->populateJumpTables();
3239
3240	for (auto &BFI : BC->getBinaryFunctions()) {
3241	BinaryFunction &Function = BFI.second;
3242
3243	if (!shouldDisassemble(BF: Function))
3244	continue;
3245
3246	Function.postProcessEntryPoints();
3247	Function.postProcessJumpTables();
3248	}
3249
3250	BC->clearJumpTableTempData();
3251	BC->adjustCodePadding();
3252
3253	for (auto &BFI : BC->getBinaryFunctions()) {
3254	BinaryFunction &Function = BFI.second;
3255
3256	if (!shouldDisassemble(BF: Function))
3257	continue;
3258
3259	if (!Function.isSimple()) {
3260	assert((!BC->HasRelocations || Function.getSize() == 0 ||
3261	Function.hasIndirectTargetToSplitFragment()) &&
3262	"unexpected non-simple function in relocation mode");
3263	continue;
3264	}
3265
3266	// Fill in CFI information for this function
3267	if (!Function.trapsOnEntry() && !CFIRdWrt->fillCFIInfoFor(Function)) {
3268	if (BC->HasRelocations) {
3269	BC->errs() << BC->generateBugReportMessage(Message: "unable to fill CFI.",
3270	Function);
3271	exit(status: 1);
3272	} else {
3273	BC->errs() << "BOLT-WARNING: unable to fill CFI for function "
3274	<< Function << ". Skipping.\n";
3275	Function.setSimple(false);
3276	continue;
3277	}
3278	}
3279
3280	// Parse LSDA.
3281	if (Function.getLSDAAddress() != 0 &&
3282	!BC->getFragmentsToSkip().count(x: &Function)) {
3283	ErrorOr<BinarySection &> LSDASection =
3284	BC->getSectionForAddress(Address: Function.getLSDAAddress());
3285	check_error(EC: LSDASection.getError(), Message: "failed to get LSDA section");
3286	ArrayRef<uint8_t> LSDAData = ArrayRef<uint8_t>(
3287	LSDASection->getData(), LSDASection->getContents().size());
3288	BC->logBOLTErrorsAndQuitOnFatal(
3289	E: Function.parseLSDA(LSDAData, LSDAAddress: LSDASection->getAddress()));
3290	}
3291	}
3292	}
3293
3294	void RewriteInstance::buildFunctionsCFG() {
3295	NamedRegionTimer T("buildCFG", "buildCFG", "buildfuncs",
3296	"Build Binary Functions", opts::TimeBuild);
3297
3298	// Create annotation indices to allow lock-free execution
3299	BC->MIB->getOrCreateAnnotationIndex(Name: "JTIndexReg");
3300	BC->MIB->getOrCreateAnnotationIndex(Name: "NOP");
3301
3302	ParallelUtilities::WorkFuncWithAllocTy WorkFun =
3303	[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {
3304	bool HadErrors{false};
3305	handleAllErrors(E: BF.buildCFG(AllocId), Handlers: [&](const BOLTError &E) {
3306	if (!E.getMessage().empty())
3307	E.log(OS&: BC->errs());
3308	if (E.isFatal())
3309	exit(status: 1);
3310	HadErrors = true;
3311	});
3312
3313	if (HadErrors)
3314	return;
3315
3316	if (opts::PrintAll) {
3317	auto L = BC->scopeLock();
3318	BF.print(OS&: BC->outs(), Annotation: "while building cfg");
3319	}
3320	};
3321
3322	ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
3323	return !shouldDisassemble(BF) || !BF.isSimple();
3324	};
3325
3326	ParallelUtilities::runOnEachFunctionWithUniqueAllocId(
3327	BC&: *BC, SchedPolicy: ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFunction: WorkFun,
3328	SkipPredicate, LogName: "disassembleFunctions-buildCFG",
3329	/*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll);
3330
3331	BC->postProcessSymbolTable();
3332	}
3333
3334	void RewriteInstance::postProcessFunctions() {
3335	// We mark fragments as non-simple here, not during disassembly,
3336	// So we can build their CFGs.
3337	BC->skipMarkedFragments();
3338	BC->clearFragmentsToSkip();
3339
3340	BC->TotalScore = 0;
3341	BC->SumExecutionCount = 0;
3342	for (auto &BFI : BC->getBinaryFunctions()) {
3343	BinaryFunction &Function = BFI.second;
3344
3345	// Set function as non-simple if it has dynamic relocations
3346	// in constant island, we don't want this function to be optimized
3347	// e.g. function splitting is unsupported.
3348	if (Function.hasDynamicRelocationAtIsland())
3349	Function.setSimple(false);
3350
3351	if (Function.empty())
3352	continue;
3353
3354	Function.postProcessCFG();
3355
3356	if (opts::PrintAll || opts::PrintCFG)
3357	Function.print(OS&: BC->outs(), Annotation: "after building cfg");
3358
3359	if (opts::DumpDotAll)
3360	Function.dumpGraphForPass(Annotation: "00_build-cfg");
3361
3362	if (opts::PrintLoopInfo) {
3363	Function.calculateLoopInfo();
3364	Function.printLoopInfo(OS&: BC->outs());
3365	}
3366
3367	BC->TotalScore += Function.getFunctionScore();
3368	BC->SumExecutionCount += Function.getKnownExecutionCount();
3369	}
3370
3371	if (opts::PrintGlobals) {
3372	BC->outs() << "BOLT-INFO: Global symbols:\n";
3373	BC->printGlobalSymbols(OS&: BC->outs());
3374	}
3375	}
3376
3377	void RewriteInstance::runOptimizationPasses() {
3378	NamedRegionTimer T("runOptimizationPasses", "run optimization passes",
3379	TimerGroupName, TimerGroupDesc, opts::TimeRewrite);
3380	BC->logBOLTErrorsAndQuitOnFatal(E: BinaryFunctionPassManager::runAllPasses(BC&: *BC));
3381	}
3382
3383	void RewriteInstance::preregisterSections() {
3384	// Preregister sections before emission to set their order in the output.
3385	const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true,
3386	/*IsText*/ false,
3387	/*IsAllocatable*/ true);
3388	if (BinarySection *EHFrameSection = getSection(Name: getEHFrameSectionName())) {
3389	// New .eh_frame.
3390	BC->registerOrUpdateSection(Name: getNewSecPrefix() + getEHFrameSectionName(),
3391	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3392	// Fully register a relocatable copy of the original .eh_frame.
3393	BC->registerSection(SectionName: ".relocated.eh_frame", OriginalSection: *EHFrameSection);
3394	}
3395	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".gcc_except_table",
3396	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3397	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata", ELFType: ELF::SHT_PROGBITS,
3398	ELFFlags: ROFlags);
3399	BC->registerOrUpdateSection(Name: getNewSecPrefix() + ".rodata.cold",
3400	ELFType: ELF::SHT_PROGBITS, ELFFlags: ROFlags);
3401	}
3402
3403	void RewriteInstance::emitAndLink() {
3404	NamedRegionTimer T("emitAndLink", "emit and link", TimerGroupName,
3405	TimerGroupDesc, opts::TimeRewrite);
3406
3407	SmallString<0> ObjectBuffer;
3408	raw_svector_ostream OS(ObjectBuffer);
3409
3410	// Implicitly MCObjectStreamer takes ownership of MCAsmBackend (MAB)
3411	// and MCCodeEmitter (MCE). ~MCObjectStreamer() will delete these
3412	// two instances.
3413	std::unique_ptr<MCStreamer> Streamer = BC->createStreamer(OS);
3414
3415	if (EHFrameSection) {
3416	if (opts::UseOldText || opts::StrictMode) {
3417	// The section is going to be regenerated from scratch.
3418	// Empty the contents, but keep the section reference.
3419	EHFrameSection->clearContents();
3420	} else {
3421	// Make .eh_frame relocatable.
3422	relocateEHFrameSection();
3423	}
3424	}
3425
3426	emitBinaryContext(Streamer&: *Streamer, BC&: *BC, OrgSecPrefix: getOrgSecPrefix());
3427
3428	Streamer->finish();
3429	if (Streamer->getContext().hadError()) {
3430	BC->errs() << "BOLT-ERROR: Emission failed.\n";
3431	exit(status: 1);
3432	}
3433
3434	if (opts::KeepTmp) {
3435	SmallString<128> OutObjectPath;
3436	sys::fs::getPotentiallyUniqueTempFileName(Prefix: "output", Suffix: "o", ResultPath&: OutObjectPath);
3437	std::error_code EC;
3438	raw_fd_ostream FOS(OutObjectPath, EC);
3439	check_error(EC, Message: "cannot create output object file");
3440	FOS << ObjectBuffer;
3441	BC->outs()
3442	<< "BOLT-INFO: intermediary output object file saved for debugging "
3443	"purposes: "
3444	<< OutObjectPath << "\n";
3445	}
3446
3447	ErrorOr<BinarySection &> TextSection =
3448	BC->getUniqueSectionByName(SectionName: BC->getMainCodeSectionName());
3449	if (BC->HasRelocations && TextSection)
3450	BC->renameSection(Section&: *TextSection,
3451	NewName: getOrgSecPrefix() + BC->getMainCodeSectionName());
3452
3453	//////////////////////////////////////////////////////////////////////////////
3454	// Assign addresses to new sections.
3455	//////////////////////////////////////////////////////////////////////////////
3456
3457	// Get output object as ObjectFile.
3458	std::unique_ptr<MemoryBuffer> ObjectMemBuffer =
3459	MemoryBuffer::getMemBuffer(InputData: ObjectBuffer, BufferName: "in-memory object file", RequiresNullTerminator: false);
3460
3461	auto EFMM = std::make_unique<ExecutableFileMemoryManager>(args&: *BC);
3462	EFMM->setNewSecPrefix(getNewSecPrefix());
3463	EFMM->setOrgSecPrefix(getOrgSecPrefix());
3464
3465	Linker = std::make_unique<JITLinkLinker>(args&: *BC, args: std::move(EFMM));
3466	Linker->loadObject(Obj: ObjectMemBuffer->getMemBufferRef(),
3467	MapSections: [this](auto MapSection) { mapFileSections(MapSection); });
3468
3469	// Update output addresses based on the new section map and
3470	// layout. Only do this for the object created by ourselves.
3471	updateOutputValues(Linker: *Linker);
3472
3473	if (opts::UpdateDebugSections) {
3474	MCAsmLayout FinalLayout(
3475	static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler());
3476	DebugInfoRewriter->updateLineTableOffsets(Layout: FinalLayout);
3477	}
3478
3479	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
3480	RtLibrary->link(BC&: *BC, ToolPath, Linker&: *Linker, MapSections: [this](auto MapSection) {
3481	// Map newly registered sections.
3482	this->mapAllocatableSections(MapSection);
3483	});
3484
3485	// Once the code is emitted, we can rename function sections to actual
3486	// output sections and de-register sections used for emission.
3487	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
3488	ErrorOr<BinarySection &> Section = Function->getCodeSection();
3489	if (Section &&
3490	(Function->getImageAddress() == 0 || Function->getImageSize() == 0))
3491	continue;
3492
3493	// Restore origin section for functions that were emitted or supposed to
3494	// be emitted to patch sections.
3495	if (Section)
3496	BC->deregisterSection(Section&: *Section);
3497	assert(Function->getOriginSectionName() && "expected origin section");
3498	Function->CodeSectionName = Function->getOriginSectionName()->str();
3499	for (const FunctionFragment &FF :
3500	Function->getLayout().getSplitFragments()) {
3501	if (ErrorOr<BinarySection &> ColdSection =
3502	Function->getCodeSection(Fragment: FF.getFragmentNum()))
3503	BC->deregisterSection(Section&: *ColdSection);
3504	}
3505	if (Function->getLayout().isSplit())
3506	Function->setColdCodeSectionName(getBOLTTextSectionName());
3507	}
3508
3509	if (opts::PrintCacheMetrics) {
3510	BC->outs() << "BOLT-INFO: cache metrics after emitting functions:\n";
3511	CacheMetrics::printAll(OS&: BC->outs(), BinaryFunctions: BC->getSortedFunctions());
3512	}
3513	}
3514
3515	void RewriteInstance::finalizeMetadataPreEmit() {
3516	MetadataManager.runFinalizersPreEmit();
3517	}
3518
3519	void RewriteInstance::updateMetadata() {
3520	MetadataManager.runFinalizersAfterEmit();
3521
3522	if (opts::UpdateDebugSections) {
3523	NamedRegionTimer T("updateDebugInfo", "update debug info", TimerGroupName,
3524	TimerGroupDesc, opts::TimeRewrite);
3525	DebugInfoRewriter->updateDebugInfo();
3526	}
3527
3528	if (opts::WriteBoltInfoSection)
3529	addBoltInfoSection();
3530	}
3531
3532	void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {
3533	BC->deregisterUnusedSections();
3534
3535	// If no new .eh_frame was written, remove relocated original .eh_frame.
3536	BinarySection *RelocatedEHFrameSection =
3537	getSection(Name: ".relocated" + getEHFrameSectionName());
3538	if (RelocatedEHFrameSection && RelocatedEHFrameSection->hasValidSectionID()) {
3539	BinarySection *NewEHFrameSection =
3540	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
3541	if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) {
3542	// JITLink will still have to process relocations for the section, hence
3543	// we need to assign it the address that wouldn't result in relocation
3544	// processing failure.
3545	MapSection(*RelocatedEHFrameSection, NextAvailableAddress);
3546	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
3547	}
3548	}
3549
3550	mapCodeSections(MapSection);
3551
3552	// Map the rest of the sections.
3553	mapAllocatableSections(MapSection);
3554	}
3555
3556	std::vector<BinarySection *> RewriteInstance::getCodeSections() {
3557	std::vector<BinarySection *> CodeSections;
3558	for (BinarySection &Section : BC->textSections())
3559	if (Section.hasValidSectionID())
3560	CodeSections.emplace_back(args: &Section);
3561
3562	auto compareSections = [&](const BinarySection *A, const BinarySection *B) {
3563	// If both A and B have names starting with ".text.cold", then
3564	// - if opts::HotFunctionsAtEnd is true, we want order
3565	// ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
3566	// - if opts::HotFunctionsAtEnd is false, we want order
3567	// ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
3568	if (A->getName().starts_with(Prefix: BC->getColdCodeSectionName()) &&
3569	B->getName().starts_with(Prefix: BC->getColdCodeSectionName())) {
3570	if (A->getName().size() != B->getName().size())
3571	return (opts::HotFunctionsAtEnd)
3572	? (A->getName().size() > B->getName().size())
3573	: (A->getName().size() < B->getName().size());
3574	return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName())
3575	: (A->getName() < B->getName());
3576	}
3577
3578	// Place movers before anything else.
3579	if (A->getName() == BC->getHotTextMoverSectionName())
3580	return true;
3581	if (B->getName() == BC->getHotTextMoverSectionName())
3582	return false;
3583
3584	// Depending on opts::HotFunctionsAtEnd, place main and warm sections in
3585	// order.
3586	if (opts::HotFunctionsAtEnd) {
3587	if (B->getName() == BC->getMainCodeSectionName())
3588	return true;
3589	if (A->getName() == BC->getMainCodeSectionName())
3590	return false;
3591	return (B->getName() == BC->getWarmCodeSectionName());
3592	} else {
3593	if (A->getName() == BC->getMainCodeSectionName())
3594	return true;
3595	if (B->getName() == BC->getMainCodeSectionName())
3596	return false;
3597	return (A->getName() == BC->getWarmCodeSectionName());
3598	}
3599	};
3600
3601	// Determine the order of sections.
3602	llvm::stable_sort(Range&: CodeSections, C: compareSections);
3603
3604	return CodeSections;
3605	}
3606
3607	void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {
3608	if (BC->HasRelocations) {
3609	// Map sections for functions with pre-assigned addresses.
3610	for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {
3611	const uint64_t OutputAddress = InjectedFunction->getOutputAddress();
3612	if (!OutputAddress)
3613	continue;
3614
3615	ErrorOr<BinarySection &> FunctionSection =
3616	InjectedFunction->getCodeSection();
3617	assert(FunctionSection && "function should have section");
3618	FunctionSection->setOutputAddress(OutputAddress);
3619	MapSection(*FunctionSection, OutputAddress);
3620	InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());
3621	InjectedFunction->setImageSize(FunctionSection->getOutputSize());
3622	}
3623
3624	// Populate the list of sections to be allocated.
3625	std::vector<BinarySection *> CodeSections = getCodeSections();
3626
3627	// Remove sections that were pre-allocated (patch sections).
3628	llvm::erase_if(C&: CodeSections, P: [](BinarySection *Section) {
3629	return Section->getOutputAddress();
3630	});
3631	LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";
3632	for (const BinarySection *Section : CodeSections)
3633	dbgs() << Section->getName() << '\n';
3634	);
3635
3636	uint64_t PaddingSize = 0; // size of padding required at the end
3637
3638	// Allocate sections starting at a given Address.
3639	auto allocateAt = [&](uint64_t Address) {
3640	const char *LastNonColdSectionName = BC->HasWarmSection
3641	? BC->getWarmCodeSectionName()
3642	: BC->getMainCodeSectionName();
3643	for (BinarySection *Section : CodeSections) {
3644	Address = alignTo(Value: Address, Align: Section->getAlignment());
3645	Section->setOutputAddress(Address);
3646	Address += Section->getOutputSize();
3647
3648	// Hugify: Additional huge page from right side due to
3649	// weird ASLR mapping addresses (4KB aligned)
3650	if (opts::Hugify && !BC->HasFixedLoadAddress &&
3651	Section->getName() == LastNonColdSectionName)
3652	Address = alignTo(Value: Address, Align: Section->getAlignment());
3653	}
3654
3655	// Make sure we allocate enough space for huge pages.
3656	ErrorOr<BinarySection &> TextSection =
3657	BC->getUniqueSectionByName(SectionName: LastNonColdSectionName);
3658	if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {
3659	uint64_t HotTextEnd =
3660	TextSection->getOutputAddress() + TextSection->getOutputSize();
3661	HotTextEnd = alignTo(Value: HotTextEnd, Align: BC->PageAlign);
3662	if (HotTextEnd > Address) {
3663	PaddingSize = HotTextEnd - Address;
3664	Address = HotTextEnd;
3665	}
3666	}
3667	return Address;
3668	};
3669
3670	// Check if we can fit code in the original .text
3671	bool AllocationDone = false;
3672	if (opts::UseOldText) {
3673	const uint64_t CodeSize =
3674	allocateAt(BC->OldTextSectionAddress) - BC->OldTextSectionAddress;
3675
3676	if (CodeSize <= BC->OldTextSectionSize) {
3677	BC->outs() << "BOLT-INFO: using original .text for new code with 0x"
3678	<< Twine::utohexstr(Val: opts::AlignText) << " alignment\n";
3679	AllocationDone = true;
3680	} else {
3681	BC->errs()
3682	<< "BOLT-WARNING: original .text too small to fit the new code"
3683	<< " using 0x" << Twine::utohexstr(Val: opts::AlignText)
3684	<< " alignment. " << CodeSize << " bytes needed, have "
3685	<< BC->OldTextSectionSize << " bytes available.\n";
3686	opts::UseOldText = false;
3687	}
3688	}
3689
3690	if (!AllocationDone)
3691	NextAvailableAddress = allocateAt(NextAvailableAddress);
3692
3693	// Do the mapping for ORC layer based on the allocation.
3694	for (BinarySection *Section : CodeSections) {
3695	LLVM_DEBUG(
3696	dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"
3697	<< Twine::utohexstr(Section->getAllocAddress()) << " to 0x"
3698	<< Twine::utohexstr(Section->getOutputAddress()) << '\n');
3699	MapSection(*Section, Section->getOutputAddress());
3700	Section->setOutputFileOffset(
3701	getFileOffsetForAddress(Address: Section->getOutputAddress()));
3702	}
3703
3704	// Check if we need to insert a padding section for hot text.
3705	if (PaddingSize && !opts::UseOldText)
3706	BC->outs() << "BOLT-INFO: padding code to 0x"
3707	<< Twine::utohexstr(Val: NextAvailableAddress)
3708	<< " to accommodate hot text\n";
3709
3710	return;
3711	}
3712
3713	// Processing in non-relocation mode.
3714	uint64_t NewTextSectionStartAddress = NextAvailableAddress;
3715
3716	for (auto &BFI : BC->getBinaryFunctions()) {
3717	BinaryFunction &Function = BFI.second;
3718	if (!Function.isEmitted())
3719	continue;
3720
3721	bool TooLarge = false;
3722	ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();
3723	assert(FuncSection && "cannot find section for function");
3724	FuncSection->setOutputAddress(Function.getAddress());
3725	LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
3726	<< Twine::utohexstr(FuncSection->getAllocAddress())
3727	<< " to 0x" << Twine::utohexstr(Function.getAddress())
3728	<< '\n');
3729	MapSection(*FuncSection, Function.getAddress());
3730	Function.setImageAddress(FuncSection->getAllocAddress());
3731	Function.setImageSize(FuncSection->getOutputSize());
3732	if (Function.getImageSize() > Function.getMaxSize()) {
3733	assert(!BC->isX86() && "Unexpected large function.");
3734	TooLarge = true;
3735	FailedAddresses.emplace_back(args: Function.getAddress());
3736	}
3737
3738	// Map jump tables if updating in-place.
3739	if (opts::JumpTables == JTS_BASIC) {
3740	for (auto &JTI : Function.JumpTables) {
3741	JumpTable *JT = JTI.second;
3742	BinarySection &Section = JT->getOutputSection();
3743	Section.setOutputAddress(JT->getAddress());
3744	Section.setOutputFileOffset(getFileOffsetForAddress(Address: JT->getAddress()));
3745	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: mapping JT " << Section.getName()
3746	<< " to 0x" << Twine::utohexstr(JT->getAddress())
3747	<< '\n');
3748	MapSection(Section, JT->getAddress());
3749	}
3750	}
3751
3752	if (!Function.isSplit())
3753	continue;
3754
3755	assert(Function.getLayout().isHotColdSplit() &&
3756	"Cannot allocate more than two fragments per function in "
3757	"non-relocation mode.");
3758
3759	FunctionFragment &FF =
3760	Function.getLayout().getFragment(Num: FragmentNum::cold());
3761	ErrorOr<BinarySection &> ColdSection =
3762	Function.getCodeSection(Fragment: FF.getFragmentNum());
3763	assert(ColdSection && "cannot find section for cold part");
3764	// Cold fragments are aligned at 16 bytes.
3765	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: 16);
3766	if (TooLarge) {
3767	// The corresponding FDE will refer to address 0.
3768	FF.setAddress(0);
3769	FF.setImageAddress(0);
3770	FF.setImageSize(0);
3771	FF.setFileOffset(0);
3772	} else {
3773	FF.setAddress(NextAvailableAddress);
3774	FF.setImageAddress(ColdSection->getAllocAddress());
3775	FF.setImageSize(ColdSection->getOutputSize());
3776	FF.setFileOffset(getFileOffsetForAddress(Address: NextAvailableAddress));
3777	ColdSection->setOutputAddress(FF.getAddress());
3778	}
3779
3780	LLVM_DEBUG(
3781	dbgs() << formatv(
3782	"BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n",
3783	FF.getImageAddress(), FF.getAddress(), FF.getImageSize()));
3784	MapSection(*ColdSection, FF.getAddress());
3785
3786	if (TooLarge)
3787	BC->deregisterSection(Section&: *ColdSection);
3788
3789	NextAvailableAddress += FF.getImageSize();
3790	}
3791
3792	// Add the new text section aggregating all existing code sections.
3793	// This is pseudo-section that serves a purpose of creating a corresponding
3794	// entry in section header table.
3795	int64_t NewTextSectionSize =
3796	NextAvailableAddress - NewTextSectionStartAddress;
3797	if (NewTextSectionSize) {
3798	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
3799	/*IsText=*/true,
3800	/*IsAllocatable=*/true);
3801	BinarySection &Section =
3802	BC->registerOrUpdateSection(Name: getBOLTTextSectionName(),
3803	ELFType: ELF::SHT_PROGBITS,
3804	ELFFlags: Flags,
3805	/*Data=*/nullptr,
3806	Size: NewTextSectionSize,
3807	Alignment: 16);
3808	Section.setOutputAddress(NewTextSectionStartAddress);
3809	Section.setOutputFileOffset(
3810	getFileOffsetForAddress(Address: NewTextSectionStartAddress));
3811	}
3812	}
3813
3814	void RewriteInstance::mapAllocatableSections(
3815	BOLTLinker::SectionMapper MapSection) {
3816	// Allocate read-only sections first, then writable sections.
3817	enum : uint8_t { ST_READONLY, ST_READWRITE };
3818	for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) {
3819	const uint64_t LastNextAvailableAddress = NextAvailableAddress;
3820	if (SType == ST_READWRITE) {
3821	// Align R+W segment to regular page size
3822	NextAvailableAddress = alignTo(Value: NextAvailableAddress, Align: BC->RegularPageSize);
3823	NewWritableSegmentAddress = NextAvailableAddress;
3824	}
3825
3826	for (BinarySection &Section : BC->allocatableSections()) {
3827	if (Section.isLinkOnly())
3828	continue;
3829
3830	if (!Section.hasValidSectionID())
3831	continue;
3832
3833	if (Section.isWritable() == (SType == ST_READONLY))
3834	continue;
3835
3836	if (Section.getOutputAddress()) {
3837	LLVM_DEBUG({
3838	dbgs() << "BOLT-DEBUG: section " << Section.getName()
3839	<< " is already mapped at 0x"
3840	<< Twine::utohexstr(Section.getOutputAddress()) << '\n';
3841	});
3842	continue;
3843	}
3844
3845	if (Section.hasSectionRef()) {
3846	LLVM_DEBUG({
3847	dbgs() << "BOLT-DEBUG: mapping original section " << Section.getName()
3848	<< " to 0x" << Twine::utohexstr(Section.getAddress()) << '\n';
3849	});
3850	Section.setOutputAddress(Section.getAddress());
3851	Section.setOutputFileOffset(Section.getInputFileOffset());
3852	MapSection(Section, Section.getAddress());
3853	} else {
3854	NextAvailableAddress =
3855	alignTo(Value: NextAvailableAddress, Align: Section.getAlignment());
3856	LLVM_DEBUG({
3857	dbgs() << "BOLT: mapping section " << Section.getName() << " (0x"
3858	<< Twine::utohexstr(Section.getAllocAddress()) << ") to 0x"
3859	<< Twine::utohexstr(NextAvailableAddress) << ":0x"
3860	<< Twine::utohexstr(NextAvailableAddress +
3861	Section.getOutputSize())
3862	<< '\n';
3863	});
3864
3865	MapSection(Section, NextAvailableAddress);
3866	Section.setOutputAddress(NextAvailableAddress);
3867	Section.setOutputFileOffset(
3868	getFileOffsetForAddress(Address: NextAvailableAddress));
3869
3870	NextAvailableAddress += Section.getOutputSize();
3871	}
3872	}
3873
3874	if (SType == ST_READONLY) {
3875	if (PHDRTableAddress) {
3876	// Segment size includes the size of the PHDR area.
3877	NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
3878	} else {
3879	// Existing PHDR table would be updated.
3880	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
3881	}
3882	} else if (SType == ST_READWRITE) {
3883	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
3884	// Restore NextAvailableAddress if no new writable sections
3885	if (!NewWritableSegmentSize)
3886	NextAvailableAddress = LastNextAvailableAddress;
3887	}
3888	}
3889	}
3890
3891	void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {
3892	if (std::optional<AddressMap> Map = AddressMap::parse(BC&: *BC))
3893	BC->setIOAddressMap(std::move(*Map));
3894
3895	for (BinaryFunction *Function : BC->getAllBinaryFunctions())
3896	Function->updateOutputValues(Linker);
3897	}
3898
3899	void RewriteInstance::patchELFPHDRTable() {
3900	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
3901	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
3902	raw_fd_ostream &OS = Out->os();
3903
3904	// Write/re-write program headers.
3905	Phnum = Obj.getHeader().e_phnum;
3906	if (PHDRTableOffset) {
3907	// Writing new pheader table and adding one new entry for R+X segment.
3908	Phnum += 1;
3909	if (NewWritableSegmentSize) {
3910	// Adding one more entry for R+W segment.
3911	Phnum += 1;
3912	}
3913	} else {
3914	assert(!PHDRTableAddress && "unexpected address for program header table");
3915	PHDRTableOffset = Obj.getHeader().e_phoff;
3916	if (NewWritableSegmentSize) {
3917	BC->errs() << "Unable to add writable segment with UseGnuStack option\n";
3918	exit(status: 1);
3919	}
3920	}
3921
3922	// NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
3923	// last segments size based on the NextAvailableAddress variable.
3924	if (!NewWritableSegmentSize) {
3925	if (PHDRTableAddress)
3926	NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
3927	else
3928	NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
3929	} else {
3930	NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
3931	}
3932
3933	OS.seek(off: PHDRTableOffset);
3934
3935	bool ModdedGnuStack = false;
3936	(void)ModdedGnuStack;
3937	bool AddedSegment = false;
3938	(void)AddedSegment;
3939
3940	auto createNewTextPhdr = [&]() {
3941	ELF64LEPhdrTy NewPhdr;
3942	NewPhdr.p_type = ELF::PT_LOAD;
3943	if (PHDRTableAddress) {
3944	NewPhdr.p_offset = PHDRTableOffset;
3945	NewPhdr.p_vaddr = PHDRTableAddress;
3946	NewPhdr.p_paddr = PHDRTableAddress;
3947	} else {
3948	NewPhdr.p_offset = NewTextSegmentOffset;
3949	NewPhdr.p_vaddr = NewTextSegmentAddress;
3950	NewPhdr.p_paddr = NewTextSegmentAddress;
3951	}
3952	NewPhdr.p_filesz = NewTextSegmentSize;
3953	NewPhdr.p_memsz = NewTextSegmentSize;
3954	NewPhdr.p_flags = ELF::PF_X | ELF::PF_R;
3955	// FIXME: Currently instrumentation is experimental and the runtime data
3956	// is emitted with code, thus everything needs to be writable
3957	if (opts::Instrument)
3958	NewPhdr.p_flags |= ELF::PF_W;
3959	NewPhdr.p_align = BC->PageAlign;
3960
3961	return NewPhdr;
3962	};
3963
3964	auto createNewWritableSectionsPhdr = [&]() {
3965	ELF64LEPhdrTy NewPhdr;
3966	NewPhdr.p_type = ELF::PT_LOAD;
3967	NewPhdr.p_offset = getFileOffsetForAddress(Address: NewWritableSegmentAddress);
3968	NewPhdr.p_vaddr = NewWritableSegmentAddress;
3969	NewPhdr.p_paddr = NewWritableSegmentAddress;
3970	NewPhdr.p_filesz = NewWritableSegmentSize;
3971	NewPhdr.p_memsz = NewWritableSegmentSize;
3972	NewPhdr.p_align = BC->RegularPageSize;
3973	NewPhdr.p_flags = ELF::PF_R | ELF::PF_W;
3974	return NewPhdr;
3975	};
3976
3977	// Copy existing program headers with modifications.
3978	for (const ELF64LE::Phdr &Phdr : cantFail(ValOrErr: Obj.program_headers())) {
3979	ELF64LE::Phdr NewPhdr = Phdr;
3980	if (PHDRTableAddress && Phdr.p_type == ELF::PT_PHDR) {
3981	NewPhdr.p_offset = PHDRTableOffset;
3982	NewPhdr.p_vaddr = PHDRTableAddress;
3983	NewPhdr.p_paddr = PHDRTableAddress;
3984	NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum;
3985	NewPhdr.p_memsz = sizeof(NewPhdr) * Phnum;
3986	} else if (Phdr.p_type == ELF::PT_GNU_EH_FRAME) {
3987	ErrorOr<BinarySection &> EHFrameHdrSec =
3988	BC->getUniqueSectionByName(SectionName: getNewSecPrefix() + ".eh_frame_hdr");
3989	if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() &&
3990	EHFrameHdrSec->isFinalized()) {
3991	NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset();
3992	NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress();
3993	NewPhdr.p_paddr = EHFrameHdrSec->getOutputAddress();
3994	NewPhdr.p_filesz = EHFrameHdrSec->getOutputSize();
3995	NewPhdr.p_memsz = EHFrameHdrSec->getOutputSize();
3996	}
3997	} else if (opts::UseGnuStack && Phdr.p_type == ELF::PT_GNU_STACK) {
3998	NewPhdr = createNewTextPhdr();
3999	ModdedGnuStack = true;
4000	} else if (!opts::UseGnuStack && Phdr.p_type == ELF::PT_DYNAMIC) {
4001	// Insert the new header before DYNAMIC.
4002	ELF64LE::Phdr NewTextPhdr = createNewTextPhdr();
4003	OS.write(Ptr: reinterpret_cast<const char *>(&NewTextPhdr),
4004	Size: sizeof(NewTextPhdr));
4005	if (NewWritableSegmentSize) {
4006	ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr();
4007	OS.write(Ptr: reinterpret_cast<const char *>(&NewWritablePhdr),
4008	Size: sizeof(NewWritablePhdr));
4009	}
4010	AddedSegment = true;
4011	}
4012	OS.write(Ptr: reinterpret_cast<const char *>(&NewPhdr), Size: sizeof(NewPhdr));
4013	}
4014
4015	if (!opts::UseGnuStack && !AddedSegment) {
4016	// Append the new header to the end of the table.
4017	ELF64LE::Phdr NewTextPhdr = createNewTextPhdr();
4018	OS.write(Ptr: reinterpret_cast<const char *>(&NewTextPhdr), Size: sizeof(NewTextPhdr));
4019	if (NewWritableSegmentSize) {
4020	ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr();
4021	OS.write(Ptr: reinterpret_cast<const char *>(&NewWritablePhdr),
4022	Size: sizeof(NewWritablePhdr));
4023	}
4024	}
4025
4026	assert((!opts::UseGnuStack || ModdedGnuStack) &&
4027	"could not find GNU_STACK program header to modify");
4028	}
4029
4030	namespace {
4031
4032	/// Write padding to \p OS such that its current \p Offset becomes aligned
4033	/// at \p Alignment. Return new (aligned) offset.
4034	uint64_t appendPadding(raw_pwrite_stream &OS, uint64_t Offset,
4035	uint64_t Alignment) {
4036	if (!Alignment)
4037	return Offset;
4038
4039	const uint64_t PaddingSize =
4040	offsetToAlignment(Value: Offset, Alignment: llvm::Align(Alignment));
4041	for (unsigned I = 0; I < PaddingSize; ++I)
4042	OS.write(C: (unsigned char)0);
4043	return Offset + PaddingSize;
4044	}
4045
4046	}
4047
4048	void RewriteInstance::rewriteNoteSections() {
4049	auto ELF64LEFile = cast<ELF64LEObjectFile>(Val: InputFile);
4050	const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
4051	raw_fd_ostream &OS = Out->os();
4052
4053	uint64_t NextAvailableOffset = getFileOffsetForAddress(Address: NextAvailableAddress);
4054	assert(NextAvailableOffset >= FirstNonAllocatableOffset &&
4055	"next available offset calculation failure");
4056	OS.seek(off: NextAvailableOffset);
4057
4058	// Copy over non-allocatable section contents and update file offsets.
4059	for (const ELF64LE::Shdr &Section : cantFail(ValOrErr: Obj.sections())) {
4060	if (Section.sh_type == ELF::SHT_NULL)
4061	continue;
4062	if (Section.sh_flags & ELF::SHF_ALLOC)
4063	continue;
4064
4065	SectionRef SecRef = ELF64LEFile->toSectionRef(Sec: &Section);
4066	BinarySection *BSec = BC->getSectionForSectionRef(Section: SecRef);
4067	assert(BSec && !BSec->isAllocatable() &&
4068	"Matching non-allocatable BinarySection should exist.");
4069
4070	StringRef SectionName =
4071	cantFail(ValOrErr: Obj.getSectionName(Section), Msg: "cannot get section name");
4072	if (shouldStrip(Section, SectionName))
4073	continue;
4074
4075	// Insert padding as needed.
4076	NextAvailableOffset =
4077	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.sh_addralign);
4078
4079	// New section size.
4080	uint64_t Size = 0;
4081	bool DataWritten = false;
4082	uint8_t *SectionData = nullptr;
4083	// Copy over section contents unless it's one of the sections we overwrite.
4084	if (!willOverwriteSection(SectionName)) {
4085	Size = Section.sh_size;
4086	StringRef Dataref = InputFile->getData().substr(Start: Section.sh_offset, N: Size);
4087	std::string Data;
4088	if (BSec->getPatcher()) {
4089	Data = BSec->getPatcher()->patchBinary(BinaryContents: Dataref);
4090	Dataref = StringRef(Data);
4091	}
4092
4093	// Section was expanded, so need to treat it as overwrite.
4094	if (Size != Dataref.size()) {
4095	BSec = &BC->registerOrUpdateNoteSection(
4096	Name: SectionName, Data: copyByteArray(Buffer: Dataref), Size: Dataref.size());
4097	Size = 0;
4098	} else {
4099	OS << Dataref;
4100	DataWritten = true;
4101
4102	// Add padding as the section extension might rely on the alignment.
4103	Size = appendPadding(OS, Offset: Size, Alignment: Section.sh_addralign);
4104	}
4105	}
4106
4107	// Perform section post-processing.
4108	assert(BSec->getAlignment() <= Section.sh_addralign &&
4109	"alignment exceeds value in file");
4110
4111	if (BSec->getAllocAddress()) {
4112	assert(!DataWritten && "Writing section twice.");
4113	(void)DataWritten;
4114	SectionData = BSec->getOutputData();
4115
4116	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: " << (Size ? "appending" : "writing")
4117	<< " contents to section " << SectionName << '\n');
4118	OS.write(Ptr: reinterpret_cast<char *>(SectionData), Size: BSec->getOutputSize());
4119	Size += BSec->getOutputSize();
4120	}
4121
4122	BSec->setOutputFileOffset(NextAvailableOffset);
4123	BSec->flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
4124	return getNewValueForSymbol(Name: S->getName());
4125	});
4126
4127	// Section contents are no longer needed, but we need to update the size so
4128	// that it will be reflected in the section header table.
4129	BSec->updateContents(NewData: nullptr, NewSize: Size);
4130
4131	NextAvailableOffset += Size;
4132	}
4133
4134	// Write new note sections.
4135	for (BinarySection &Section : BC->nonAllocatableSections()) {
4136	if (Section.getOutputFileOffset() || !Section.getAllocAddress())
4137	continue;
4138
4139	assert(!Section.hasPendingRelocations() && "cannot have pending relocs");
4140
4141	NextAvailableOffset =
4142	appendPadding(OS, Offset: NextAvailableOffset, Alignment: Section.getAlignment());
4143	Section.setOutputFileOffset(NextAvailableOffset);
4144
4145	LLVM_DEBUG(
4146	dbgs() << "BOLT-DEBUG: writing out new section " << Section.getName()
4147	<< " of size " << Section.getOutputSize() << " at offset 0x"
4148	<< Twine::utohexstr(Section.getOutputFileOffset()) << '\n');
4149
4150	OS.write(Ptr: Section.getOutputContents().data(), Size: Section.getOutputSize());
4151	NextAvailableOffset += Section.getOutputSize();
4152	}
4153	}
4154
4155	template <typename ELFT>
4156	void RewriteInstance::finalizeSectionStringTable(ELFObjectFile<ELFT> *File) {
4157	// Pre-populate section header string table.
4158	for (const BinarySection &Section : BC->sections())
4159	if (!Section.isAnonymous())
4160	SHStrTab.add(S: Section.getOutputName());
4161	SHStrTab.finalize();
4162
4163	const size_t SHStrTabSize = SHStrTab.getSize();
4164	uint8_t *DataCopy = new uint8_t[SHStrTabSize];
4165	memset(s: DataCopy, c: 0, n: SHStrTabSize);
4166	SHStrTab.write(Buf: DataCopy);
4167	BC->registerOrUpdateNoteSection(Name: ".shstrtab",
4168	Data: DataCopy,
4169	Size: SHStrTabSize,
4170	/*Alignment=*/1,
4171	/*IsReadOnly=*/true,
4172	ELFType: ELF::SHT_STRTAB);
4173	}
4174
4175	void RewriteInstance::addBoltInfoSection() {
4176	std::string DescStr;
4177	raw_string_ostream DescOS(DescStr);
4178
4179	DescOS << "BOLT revision: " << BoltRevision << ", "
4180	<< "command line:";
4181	for (int I = 0; I < Argc; ++I)
4182	DescOS << " " << Argv[I];
4183	DescOS.flush();
4184
4185	// Encode as GNU GOLD VERSION so it is easily printable by 'readelf -n'
4186	const std::string BoltInfo =
4187	BinarySection::encodeELFNote(NameStr: "GNU", DescStr, Type: 4 /*NT_GNU_GOLD_VERSION*/);
4188	BC->registerOrUpdateNoteSection(Name: ".note.bolt_info", Data: copyByteArray(Buffer: BoltInfo),
4189	Size: BoltInfo.size(),
4190	/*Alignment=*/1,
4191	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4192	}
4193
4194	void RewriteInstance::addBATSection() {
4195	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME, Data: nullptr,
4196	Size: 0,
4197	/*Alignment=*/1,
4198	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4199	}
4200
4201	void RewriteInstance::encodeBATSection() {
4202	std::string DescStr;
4203	raw_string_ostream DescOS(DescStr);
4204
4205	BAT->write(BC: *BC, OS&: DescOS);
4206	DescOS.flush();
4207
4208	const std::string BoltInfo =
4209	BinarySection::encodeELFNote(NameStr: "BOLT", DescStr, Type: BinarySection::NT_BOLT_BAT);
4210	BC->registerOrUpdateNoteSection(Name: BoltAddressTranslation::SECTION_NAME,
4211	Data: copyByteArray(Buffer: BoltInfo), Size: BoltInfo.size(),
4212	/*Alignment=*/1,
4213	/*IsReadOnly=*/true, ELFType: ELF::SHT_NOTE);
4214	BC->outs() << "BOLT-INFO: BAT section size (bytes): " << BoltInfo.size()
4215	<< '\n';
4216	}
4217
4218	template <typename ELFShdrTy>
4219	bool RewriteInstance::shouldStrip(const ELFShdrTy &Section,
4220	StringRef SectionName) {
4221	// Strip non-allocatable relocation sections.
4222	if (!(Section.sh_flags & ELF::SHF_ALLOC) && Section.sh_type == ELF::SHT_RELA)
4223	return true;
4224
4225	// Strip debug sections if not updating them.
4226	if (isDebugSection(SectionName) && !opts::UpdateDebugSections)
4227	return true;
4228
4229	// Strip symtab section if needed
4230	if (opts::RemoveSymtab && Section.sh_type == ELF::SHT_SYMTAB)
4231	return true;
4232
4233	return false;
4234	}
4235
4236	template <typename ELFT>
4237	std::vector<typename object::ELFObjectFile<ELFT>::Elf_Shdr>
4238	RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
4239	std::vector<uint32_t> &NewSectionIndex) {
4240	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4241	const ELFFile<ELFT> &Obj = File->getELFFile();
4242	typename ELFT::ShdrRange Sections = cantFail(Obj.sections());
4243
4244	// Keep track of section header entries attached to the corresponding section.
4245	std::vector<std::pair<BinarySection *, ELFShdrTy>> OutputSections;
4246	auto addSection = [&](const ELFShdrTy &Section, BinarySection &BinSec) {
4247	ELFShdrTy NewSection = Section;
4248	NewSection.sh_name = SHStrTab.getOffset(S: BinSec.getOutputName());
4249	OutputSections.emplace_back(&BinSec, std::move(NewSection));
4250	};
4251
4252	// Copy over entries for original allocatable sections using modified name.
4253	for (const ELFShdrTy &Section : Sections) {
4254	// Always ignore this section.
4255	if (Section.sh_type == ELF::SHT_NULL) {
4256	OutputSections.emplace_back(nullptr, Section);
4257	continue;
4258	}
4259
4260	if (!(Section.sh_flags & ELF::SHF_ALLOC))
4261	continue;
4262
4263	SectionRef SecRef = File->toSectionRef(&Section);
4264	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4265	assert(BinSec && "Matching BinarySection should exist.");
4266
4267	addSection(Section, *BinSec);
4268	}
4269
4270	for (BinarySection &Section : BC->allocatableSections()) {
4271	if (!Section.isFinalized())
4272	continue;
4273
4274	if (Section.hasSectionRef() || Section.isAnonymous()) {
4275	if (opts::Verbosity)
4276	BC->outs() << "BOLT-INFO: not writing section header for section "
4277	<< Section.getOutputName() << '\n';
4278	continue;
4279	}
4280
4281	if (opts::Verbosity >= 1)
4282	BC->outs() << "BOLT-INFO: writing section header for "
4283	<< Section.getOutputName() << '\n';
4284	ELFShdrTy NewSection;
4285	NewSection.sh_type = ELF::SHT_PROGBITS;
4286	NewSection.sh_addr = Section.getOutputAddress();
4287	NewSection.sh_offset = Section.getOutputFileOffset();
4288	NewSection.sh_size = Section.getOutputSize();
4289	NewSection.sh_entsize = 0;
4290	NewSection.sh_flags = Section.getELFFlags();
4291	NewSection.sh_link = 0;
4292	NewSection.sh_info = 0;
4293	NewSection.sh_addralign = Section.getAlignment();
4294	addSection(NewSection, Section);
4295	}
4296
4297	// Sort all allocatable sections by their offset.
4298	llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) {
4299	return A.second.sh_offset < B.second.sh_offset;
4300	});
4301
4302	// Fix section sizes to prevent overlapping.
4303	ELFShdrTy *PrevSection = nullptr;
4304	BinarySection *PrevBinSec = nullptr;
4305	for (auto &SectionKV : OutputSections) {
4306	ELFShdrTy &Section = SectionKV.second;
4307
4308	// Ignore NOBITS sections as they don't take any space in the file.
4309	if (Section.sh_type == ELF::SHT_NOBITS)
4310	continue;
4311
4312	// Note that address continuity is not guaranteed as sections could be
4313	// placed in different loadable segments.
4314	if (PrevSection &&
4315	PrevSection->sh_offset + PrevSection->sh_size > Section.sh_offset) {
4316	if (opts::Verbosity > 1)
4317	BC->outs() << "BOLT-INFO: adjusting size for section "
4318	<< PrevBinSec->getOutputName() << '\n';
4319	PrevSection->sh_size = Section.sh_offset - PrevSection->sh_offset;
4320	}
4321
4322	PrevSection = &Section;
4323	PrevBinSec = SectionKV.first;
4324	}
4325
4326	uint64_t LastFileOffset = 0;
4327
4328	// Copy over entries for non-allocatable sections performing necessary
4329	// adjustments.
4330	for (const ELFShdrTy &Section : Sections) {
4331	if (Section.sh_type == ELF::SHT_NULL)
4332	continue;
4333	if (Section.sh_flags & ELF::SHF_ALLOC)
4334	continue;
4335
4336	StringRef SectionName =
4337	cantFail(Obj.getSectionName(Section), "cannot get section name");
4338
4339	if (shouldStrip(Section, SectionName))
4340	continue;
4341
4342	SectionRef SecRef = File->toSectionRef(&Section);
4343	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4344	assert(BinSec && "Matching BinarySection should exist.");
4345
4346	ELFShdrTy NewSection = Section;
4347	NewSection.sh_offset = BinSec->getOutputFileOffset();
4348	NewSection.sh_size = BinSec->getOutputSize();
4349
4350	if (NewSection.sh_type == ELF::SHT_SYMTAB)
4351	NewSection.sh_info = NumLocalSymbols;
4352
4353	addSection(NewSection, *BinSec);
4354
4355	LastFileOffset = BinSec->getOutputFileOffset();
4356	}
4357
4358	// Create entries for new non-allocatable sections.
4359	for (BinarySection &Section : BC->nonAllocatableSections()) {
4360	if (Section.getOutputFileOffset() <= LastFileOffset)
4361	continue;
4362
4363	if (opts::Verbosity >= 1)
4364	BC->outs() << "BOLT-INFO: writing section header for "
4365	<< Section.getOutputName() << '\n';
4366
4367	ELFShdrTy NewSection;
4368	NewSection.sh_type = Section.getELFType();
4369	NewSection.sh_addr = 0;
4370	NewSection.sh_offset = Section.getOutputFileOffset();
4371	NewSection.sh_size = Section.getOutputSize();
4372	NewSection.sh_entsize = 0;
4373	NewSection.sh_flags = Section.getELFFlags();
4374	NewSection.sh_link = 0;
4375	NewSection.sh_info = 0;
4376	NewSection.sh_addralign = Section.getAlignment();
4377
4378	addSection(NewSection, Section);
4379	}
4380
4381	// Assign indices to sections.
4382	std::unordered_map<std::string, uint64_t> NameToIndex;
4383	for (uint32_t Index = 1; Index < OutputSections.size(); ++Index)
4384	OutputSections[Index].first->setIndex(Index);
4385
4386	// Update section index mapping
4387	NewSectionIndex.clear();
4388	NewSectionIndex.resize(Sections.size(), 0);
4389	for (const ELFShdrTy &Section : Sections) {
4390	if (Section.sh_type == ELF::SHT_NULL)
4391	continue;
4392
4393	size_t OrgIndex = std::distance(Sections.begin(), &Section);
4394
4395	SectionRef SecRef = File->toSectionRef(&Section);
4396	BinarySection *BinSec = BC->getSectionForSectionRef(Section: SecRef);
4397	assert(BinSec && "BinarySection should exist for an input section.");
4398
4399	// Some sections are stripped
4400	if (!BinSec->hasValidIndex())
4401	continue;
4402
4403	NewSectionIndex[OrgIndex] = BinSec->getIndex();
4404	}
4405
4406	std::vector<ELFShdrTy> SectionsOnly(OutputSections.size());
4407	llvm::copy(llvm::make_second_range(OutputSections), SectionsOnly.begin());
4408
4409	return SectionsOnly;
4410	}
4411
4412	// Rewrite section header table inserting new entries as needed. The sections
4413	// header table size itself may affect the offsets of other sections,
4414	// so we are placing it at the end of the binary.
4415	//
4416	// As we rewrite entries we need to track how many sections were inserted
4417	// as it changes the sh_link value. We map old indices to new ones for
4418	// existing sections.
4419	template <typename ELFT>
4420	void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {
4421	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4422	using ELFEhdrTy = typename ELFObjectFile<ELFT>::Elf_Ehdr;
4423	raw_fd_ostream &OS = Out->os();
4424	const ELFFile<ELFT> &Obj = File->getELFFile();
4425
4426	// Mapping from old section indices to new ones
4427	std::vector<uint32_t> NewSectionIndex;
4428	std::vector<ELFShdrTy> OutputSections =
4429	getOutputSections(File, NewSectionIndex);
4430	LLVM_DEBUG(
4431	dbgs() << "BOLT-DEBUG: old to new section index mapping:\n";
4432	for (uint64_t I = 0; I < NewSectionIndex.size(); ++I)
4433	dbgs() << " " << I << " -> " << NewSectionIndex[I] << '\n';
4434	);
4435
4436	// Align starting address for section header table. There's no architecutal
4437	// need to align this, it is just for pleasant human readability.
4438	uint64_t SHTOffset = OS.tell();
4439	SHTOffset = appendPadding(OS, Offset: SHTOffset, Alignment: 16);
4440
4441	// Write all section header entries while patching section references.
4442	for (ELFShdrTy &Section : OutputSections) {
4443	Section.sh_link = NewSectionIndex[Section.sh_link];
4444	if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA)
4445	Section.sh_info = NewSectionIndex[Section.sh_info];
4446	OS.write(Ptr: reinterpret_cast<const char *>(&Section), Size: sizeof(Section));
4447	}
4448
4449	// Fix ELF header.
4450	ELFEhdrTy NewEhdr = Obj.getHeader();
4451
4452	if (BC->HasRelocations) {
4453	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())
4454	NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();
4455	else
4456	NewEhdr.e_entry = getNewFunctionAddress(OldAddress: NewEhdr.e_entry);
4457	assert((NewEhdr.e_entry || !Obj.getHeader().e_entry) &&
4458	"cannot find new address for entry point");
4459	}
4460	if (PHDRTableOffset) {
4461	NewEhdr.e_phoff = PHDRTableOffset;
4462	NewEhdr.e_phnum = Phnum;
4463	}
4464	NewEhdr.e_shoff = SHTOffset;
4465	NewEhdr.e_shnum = OutputSections.size();
4466	NewEhdr.e_shstrndx = NewSectionIndex[NewEhdr.e_shstrndx];
4467	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewEhdr), Size: sizeof(NewEhdr), Offset: 0);
4468	}
4469
4470	template <typename ELFT, typename WriteFuncTy, typename StrTabFuncTy>
4471	void RewriteInstance::updateELFSymbolTable(
4472	ELFObjectFile<ELFT> *File, bool IsDynSym,
4473	const typename object::ELFObjectFile<ELFT>::Elf_Shdr &SymTabSection,
4474	const std::vector<uint32_t> &NewSectionIndex, WriteFuncTy Write,
4475	StrTabFuncTy AddToStrTab) {
4476	const ELFFile<ELFT> &Obj = File->getELFFile();
4477	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
4478
4479	StringRef StringSection =
4480	cantFail(Obj.getStringTableForSymtab(SymTabSection));
4481
4482	unsigned NumHotTextSymsUpdated = 0;
4483	unsigned NumHotDataSymsUpdated = 0;
4484
4485	std::map<const BinaryFunction *, uint64_t> IslandSizes;
4486	auto getConstantIslandSize = [&IslandSizes](const BinaryFunction &BF) {
4487	auto Itr = IslandSizes.find(x: &BF);
4488	if (Itr != IslandSizes.end())
4489	return Itr->second;
4490	return IslandSizes[&BF] = BF.estimateConstantIslandSize();
4491	};
4492
4493	// Symbols for the new symbol table.
4494	std::vector<ELFSymTy> Symbols;
4495
4496	auto getNewSectionIndex = [&](uint32_t OldIndex) {
4497	// For dynamic symbol table, the section index could be wrong on the input,
4498	// and its value is ignored by the runtime if it's different from
4499	// SHN_UNDEF and SHN_ABS.
4500	// However, we still need to update dynamic symbol table, so return a
4501	// section index, even though the index is broken.
4502	if (IsDynSym && OldIndex >= NewSectionIndex.size())
4503	return OldIndex;
4504
4505	assert(OldIndex < NewSectionIndex.size() && "section index out of bounds");
4506	const uint32_t NewIndex = NewSectionIndex[OldIndex];
4507
4508	// We may have stripped the section that dynsym was referencing due to
4509	// the linker bug. In that case return the old index avoiding marking
4510	// the symbol as undefined.
4511	if (IsDynSym && NewIndex != OldIndex && NewIndex == ELF::SHN_UNDEF)
4512	return OldIndex;
4513	return NewIndex;
4514	};
4515
4516	// Get the extra symbol name of a split fragment; used in addExtraSymbols.
4517	auto getSplitSymbolName = [&](const FunctionFragment &FF,
4518	const ELFSymTy &FunctionSymbol) {
4519	SmallString<256> SymbolName;
4520	if (BC->HasWarmSection)
4521	SymbolName =
4522	formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),
4523	FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");
4524	else
4525	SymbolName = formatv("{0}.cold.{1}",
4526	cantFail(FunctionSymbol.getName(StringSection)),
4527	FF.getFragmentNum().get() - 1);
4528	return SymbolName;
4529	};
4530
4531	// Add extra symbols for the function.
4532	//
4533	// Note that addExtraSymbols() could be called multiple times for the same
4534	// function with different FunctionSymbol matching the main function entry
4535	// point.
4536	auto addExtraSymbols = [&](const BinaryFunction &Function,
4537	const ELFSymTy &FunctionSymbol) {
4538	if (Function.isFolded()) {
4539	BinaryFunction *ICFParent = Function.getFoldedIntoFunction();
4540	while (ICFParent->isFolded())
4541	ICFParent = ICFParent->getFoldedIntoFunction();
4542	ELFSymTy ICFSymbol = FunctionSymbol;
4543	SmallVector<char, 256> Buf;
4544	ICFSymbol.st_name =
4545	AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection)))
4546	.concat(Suffix: ".icf.0")
4547	.toStringRef(Out&: Buf));
4548	ICFSymbol.st_value = ICFParent->getOutputAddress();
4549	ICFSymbol.st_size = ICFParent->getOutputSize();
4550	ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex();
4551	Symbols.emplace_back(ICFSymbol);
4552	}
4553	if (Function.isSplit()) {
4554	for (const FunctionFragment &FF :
4555	Function.getLayout().getSplitFragments()) {
4556	if (FF.getAddress()) {
4557	ELFSymTy NewColdSym = FunctionSymbol;
4558	const SmallString<256> SymbolName =
4559	getSplitSymbolName(FF, FunctionSymbol);
4560	NewColdSym.st_name = AddToStrTab(SymbolName);
4561	NewColdSym.st_shndx =
4562	Function.getCodeSection(Fragment: FF.getFragmentNum())->getIndex();
4563	NewColdSym.st_value = FF.getAddress();
4564	NewColdSym.st_size = FF.getImageSize();
4565	NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
4566	Symbols.emplace_back(NewColdSym);
4567	}
4568	}
4569	}
4570	if (Function.hasConstantIsland()) {
4571	uint64_t DataMark = Function.getOutputDataAddress();
4572	uint64_t CISize = getConstantIslandSize(Function);
4573	uint64_t CodeMark = DataMark + CISize;
4574	ELFSymTy DataMarkSym = FunctionSymbol;
4575	DataMarkSym.st_name = AddToStrTab("$d");
4576	DataMarkSym.st_value = DataMark;
4577	DataMarkSym.st_size = 0;
4578	DataMarkSym.setType(ELF::STT_NOTYPE);
4579	DataMarkSym.setBinding(ELF::STB_LOCAL);
4580	ELFSymTy CodeMarkSym = DataMarkSym;
4581	CodeMarkSym.st_name = AddToStrTab("$x");
4582	CodeMarkSym.st_value = CodeMark;
4583	Symbols.emplace_back(DataMarkSym);
4584	Symbols.emplace_back(CodeMarkSym);
4585	}
4586	if (Function.hasConstantIsland() && Function.isSplit()) {
4587	uint64_t DataMark = Function.getOutputColdDataAddress();
4588	uint64_t CISize = getConstantIslandSize(Function);
4589	uint64_t CodeMark = DataMark + CISize;
4590	ELFSymTy DataMarkSym = FunctionSymbol;
4591	DataMarkSym.st_name = AddToStrTab("$d");
4592	DataMarkSym.st_value = DataMark;
4593	DataMarkSym.st_size = 0;
4594	DataMarkSym.setType(ELF::STT_NOTYPE);
4595	DataMarkSym.setBinding(ELF::STB_LOCAL);
4596	ELFSymTy CodeMarkSym = DataMarkSym;
4597	CodeMarkSym.st_name = AddToStrTab("$x");
4598	CodeMarkSym.st_value = CodeMark;
4599	Symbols.emplace_back(DataMarkSym);
4600	Symbols.emplace_back(CodeMarkSym);
4601	}
4602	};
4603
4604	// For regular (non-dynamic) symbol table, exclude symbols referring
4605	// to non-allocatable sections.
4606	auto shouldStrip = [&](const ELFSymTy &Symbol) {
4607	if (Symbol.isAbsolute() || !Symbol.isDefined())
4608	return false;
4609
4610	// If we cannot link the symbol to a section, leave it as is.
4611	Expected<const typename ELFT::Shdr *> Section =
4612	Obj.getSection(Symbol.st_shndx);
4613	if (!Section)
4614	return false;
4615
4616	// Remove the section symbol iif the corresponding section was stripped.
4617	if (Symbol.getType() == ELF::STT_SECTION) {
4618	if (!getNewSectionIndex(Symbol.st_shndx))
4619	return true;
4620	return false;
4621	}
4622
4623	// Symbols in non-allocatable sections are typically remnants of relocations
4624	// emitted under "-emit-relocs" linker option. Delete those as we delete
4625	// relocations against non-allocatable sections.
4626	if (!((*Section)->sh_flags & ELF::SHF_ALLOC))
4627	return true;
4628
4629	return false;
4630	};
4631
4632	for (const ELFSymTy &Symbol : cantFail(Obj.symbols(&SymTabSection))) {
4633	// For regular (non-dynamic) symbol table strip unneeded symbols.
4634	if (!IsDynSym && shouldStrip(Symbol))
4635	continue;
4636
4637	const BinaryFunction *Function =
4638	BC->getBinaryFunctionAtAddress(Symbol.st_value);
4639	// Ignore false function references, e.g. when the section address matches
4640	// the address of the function.
4641	if (Function && Symbol.getType() == ELF::STT_SECTION)
4642	Function = nullptr;
4643
4644	// For non-dynamic symtab, make sure the symbol section matches that of
4645	// the function. It can mismatch e.g. if the symbol is a section marker
4646	// in which case we treat the symbol separately from the function.
4647	// For dynamic symbol table, the section index could be wrong on the input,
4648	// and its value is ignored by the runtime if it's different from
4649	// SHN_UNDEF and SHN_ABS.
4650	if (!IsDynSym && Function &&
4651	Symbol.st_shndx !=
4652	Function->getOriginSection()->getSectionRef().getIndex())
4653	Function = nullptr;
4654
4655	// Create a new symbol based on the existing symbol.
4656	ELFSymTy NewSymbol = Symbol;
4657
4658	if (Function) {
4659	// If the symbol matched a function that was not emitted, update the
4660	// corresponding section index but otherwise leave it unchanged.
4661	if (Function->isEmitted()) {
4662	NewSymbol.st_value = Function->getOutputAddress();
4663	NewSymbol.st_size = Function->getOutputSize();
4664	NewSymbol.st_shndx = Function->getCodeSection()->getIndex();
4665	} else if (Symbol.st_shndx < ELF::SHN_LORESERVE) {
4666	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
4667	}
4668
4669	// Add new symbols to the symbol table if necessary.
4670	if (!IsDynSym)
4671	addExtraSymbols(*Function, NewSymbol);
4672	} else {
4673	// Check if the function symbol matches address inside a function, i.e.
4674	// it marks a secondary entry point.
4675	Function =
4676	(Symbol.getType() == ELF::STT_FUNC)
4677	? BC->getBinaryFunctionContainingAddress(Symbol.st_value,
4678	/*CheckPastEnd=*/false,
4679	/*UseMaxSize=*/true)
4680	: nullptr;
4681
4682	if (Function && Function->isEmitted()) {
4683	assert(Function->getLayout().isHotColdSplit() &&
4684	"Adding symbols based on cold fragment when there are more than "
4685	"2 fragments");
4686	const uint64_t OutputAddress =
4687	Function->translateInputToOutputAddress(Address: Symbol.st_value);
4688
4689	NewSymbol.st_value = OutputAddress;
4690	// Force secondary entry points to have zero size.
4691	NewSymbol.st_size = 0;
4692
4693	// Find fragment containing entrypoint
4694	FunctionLayout::fragment_const_iterator FF = llvm::find_if(
4695	Function->getLayout().fragments(), [&](const FunctionFragment &FF) {
4696	uint64_t Lo = FF.getAddress();
4697	uint64_t Hi = Lo + FF.getImageSize();
4698	return Lo <= OutputAddress && OutputAddress < Hi;
4699	});
4700
4701	if (FF == Function->getLayout().fragment_end()) {
4702	assert(
4703	OutputAddress >= Function->getCodeSection()->getOutputAddress() &&
4704	OutputAddress < (Function->getCodeSection()->getOutputAddress() +
4705	Function->getCodeSection()->getOutputSize()) &&
4706	"Cannot locate fragment containing secondary entrypoint");
4707	FF = Function->getLayout().fragment_begin();
4708	}
4709
4710	NewSymbol.st_shndx =
4711	Function->getCodeSection(Fragment: FF->getFragmentNum())->getIndex();
4712	} else {
4713	// Check if the symbol belongs to moved data object and update it.
4714	BinaryData *BD = opts::ReorderData.empty()
4715	? nullptr
4716	: BC->getBinaryDataAtAddress(Symbol.st_value);
4717	if (BD && BD->isMoved() && !BD->isJumpTable()) {
4718	assert((!BD->getSize() || !Symbol.st_size ||
4719	Symbol.st_size == BD->getSize()) &&
4720	"sizes must match");
4721
4722	BinarySection &OutputSection = BD->getOutputSection();
4723	assert(OutputSection.getIndex());
4724	LLVM_DEBUG(dbgs()
4725	<< "BOLT-DEBUG: moving " << BD->getName() << " from "
4726	<< *BC->getSectionNameForAddress(Symbol.st_value) << " ("
4727	<< Symbol.st_shndx << ") to " << OutputSection.getName()
4728	<< " (" << OutputSection.getIndex() << ")\n");
4729	NewSymbol.st_shndx = OutputSection.getIndex();
4730	NewSymbol.st_value = BD->getOutputAddress();
4731	} else {
4732	// Otherwise just update the section for the symbol.
4733	if (Symbol.st_shndx < ELF::SHN_LORESERVE)
4734	NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);
4735	}
4736
4737	// Detect local syms in the text section that we didn't update
4738	// and that were preserved by the linker to support relocations against
4739	// .text. Remove them from the symtab.
4740	if (Symbol.getType() == ELF::STT_NOTYPE &&
4741	Symbol.getBinding() == ELF::STB_LOCAL && Symbol.st_size == 0) {
4742	if (BC->getBinaryFunctionContainingAddress(Symbol.st_value,
4743	/*CheckPastEnd=*/false,
4744	/*UseMaxSize=*/true)) {
4745	// Can only delete the symbol if not patching. Such symbols should
4746	// not exist in the dynamic symbol table.
4747	assert(!IsDynSym && "cannot delete symbol");
4748	continue;
4749	}
4750	}
4751	}
4752	}
4753
4754	// Handle special symbols based on their name.
4755	Expected<StringRef> SymbolName = Symbol.getName(StringSection);
4756	assert(SymbolName && "cannot get symbol name");
4757
4758	auto updateSymbolValue = [&](const StringRef Name,
4759	std::optional<uint64_t> Value = std::nullopt) {
4760	NewSymbol.st_value = Value ? *Value : getNewValueForSymbol(Name);
4761	NewSymbol.st_shndx = ELF::SHN_ABS;
4762	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
4763	<< Twine::utohexstr(Val: NewSymbol.st_value) << '\n';
4764	};
4765
4766	if (opts::HotText &&
4767	(*SymbolName == "__hot_start" || *SymbolName == "__hot_end")) {
4768	updateSymbolValue(*SymbolName);
4769	++NumHotTextSymsUpdated;
4770	}
4771
4772	if (opts::HotData && (*SymbolName == "__hot_data_start" ||
4773	*SymbolName == "__hot_data_end")) {
4774	updateSymbolValue(*SymbolName);
4775	++NumHotDataSymsUpdated;
4776	}
4777
4778	if (*SymbolName == "_end")
4779	updateSymbolValue(*SymbolName, NextAvailableAddress);
4780
4781	if (IsDynSym)
4782	Write((&Symbol - cantFail(Obj.symbols(&SymTabSection)).begin()) *
4783	sizeof(ELFSymTy),
4784	NewSymbol);
4785	else
4786	Symbols.emplace_back(NewSymbol);
4787	}
4788
4789	if (IsDynSym) {
4790	assert(Symbols.empty());
4791	return;
4792	}
4793
4794	// Add symbols of injected functions
4795	for (BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {
4796	ELFSymTy NewSymbol;
4797	BinarySection *OriginSection = Function->getOriginSection();
4798	NewSymbol.st_shndx =
4799	OriginSection
4800	? getNewSectionIndex(OriginSection->getSectionRef().getIndex())
4801	: Function->getCodeSection()->getIndex();
4802	NewSymbol.st_value = Function->getOutputAddress();
4803	NewSymbol.st_name = AddToStrTab(Function->getOneName());
4804	NewSymbol.st_size = Function->getOutputSize();
4805	NewSymbol.st_other = 0;
4806	NewSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);
4807	Symbols.emplace_back(NewSymbol);
4808
4809	if (Function->isSplit()) {
4810	assert(Function->getLayout().isHotColdSplit() &&
4811	"Adding symbols based on cold fragment when there are more than "
4812	"2 fragments");
4813	ELFSymTy NewColdSym = NewSymbol;
4814	NewColdSym.setType(ELF::STT_NOTYPE);
4815	SmallVector<char, 256> Buf;
4816	NewColdSym.st_name = AddToStrTab(
4817	Twine(Function->getPrintName()).concat(Suffix: ".cold.0").toStringRef(Out&: Buf));
4818	const FunctionFragment &ColdFF =
4819	Function->getLayout().getFragment(Num: FragmentNum::cold());
4820	NewColdSym.st_value = ColdFF.getAddress();
4821	NewColdSym.st_size = ColdFF.getImageSize();
4822	Symbols.emplace_back(NewColdSym);
4823	}
4824	}
4825
4826	auto AddSymbol = [&](const StringRef &Name, uint64_t Address) {
4827	if (!Address)
4828	return;
4829
4830	ELFSymTy Symbol;
4831	Symbol.st_value = Address;
4832	Symbol.st_shndx = ELF::SHN_ABS;
4833	Symbol.st_name = AddToStrTab(Name);
4834	Symbol.st_size = 0;
4835	Symbol.st_other = 0;
4836	Symbol.setBindingAndType(ELF::STB_WEAK, ELF::STT_NOTYPE);
4837
4838	BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"
4839	<< Twine::utohexstr(Val: Symbol.st_value) << '\n';
4840
4841	Symbols.emplace_back(Symbol);
4842	};
4843
4844	// Add runtime library start and fini address symbols
4845	if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {
4846	AddSymbol("__bolt_runtime_start", RtLibrary->getRuntimeStartAddress());
4847	AddSymbol("__bolt_runtime_fini", RtLibrary->getRuntimeFiniAddress());
4848	}
4849
4850	assert((!NumHotTextSymsUpdated || NumHotTextSymsUpdated == 2) &&
4851	"either none or both __hot_start/__hot_end symbols were expected");
4852	assert((!NumHotDataSymsUpdated || NumHotDataSymsUpdated == 2) &&
4853	"either none or both __hot_data_start/__hot_data_end symbols were "
4854	"expected");
4855
4856	auto AddEmittedSymbol = [&](const StringRef &Name) {
4857	AddSymbol(Name, getNewValueForSymbol(Name));
4858	};
4859
4860	if (opts::HotText && !NumHotTextSymsUpdated) {
4861	AddEmittedSymbol("__hot_start");
4862	AddEmittedSymbol("__hot_end");
4863	}
4864
4865	if (opts::HotData && !NumHotDataSymsUpdated) {
4866	AddEmittedSymbol("__hot_data_start");
4867	AddEmittedSymbol("__hot_data_end");
4868	}
4869
4870	// Put local symbols at the beginning.
4871	llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) {
4872	if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL)
4873	return true;
4874	return false;
4875	});
4876
4877	for (const ELFSymTy &Symbol : Symbols)
4878	Write(0, Symbol);
4879	}
4880
4881	template <typename ELFT>
4882	void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {
4883	const ELFFile<ELFT> &Obj = File->getELFFile();
4884	using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;
4885	using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;
4886
4887	// Compute a preview of how section indices will change after rewriting, so
4888	// we can properly update the symbol table based on new section indices.
4889	std::vector<uint32_t> NewSectionIndex;
4890	getOutputSections(File, NewSectionIndex);
4891
4892	// Set pointer at the end of the output file, so we can pwrite old symbol
4893	// tables if we need to.
4894	uint64_t NextAvailableOffset = getFileOffsetForAddress(Address: NextAvailableAddress);
4895	assert(NextAvailableOffset >= FirstNonAllocatableOffset &&
4896	"next available offset calculation failure");
4897	Out->os().seek(off: NextAvailableOffset);
4898
4899	// Update dynamic symbol table.
4900	const ELFShdrTy *DynSymSection = nullptr;
4901	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
4902	if (Section.sh_type == ELF::SHT_DYNSYM) {
4903	DynSymSection = &Section;
4904	break;
4905	}
4906	}
4907	assert((DynSymSection || BC->IsStaticExecutable) &&
4908	"dynamic symbol table expected");
4909	if (DynSymSection) {
4910	updateELFSymbolTable(
4911	File,
4912	/*IsDynSym=*/true,
4913	*DynSymSection,
4914	NewSectionIndex,
4915	[&](size_t Offset, const ELFSymTy &Sym) {
4916	Out->os().pwrite(Ptr: reinterpret_cast<const char *>(&Sym),
4917	Size: sizeof(ELFSymTy),
4918	Offset: DynSymSection->sh_offset + Offset);
4919	},
4920	[](StringRef) -> size_t { return 0; });
4921	}
4922
4923	if (opts::RemoveSymtab)
4924	return;
4925
4926	// (re)create regular symbol table.
4927	const ELFShdrTy *SymTabSection = nullptr;
4928	for (const ELFShdrTy &Section : cantFail(Obj.sections())) {
4929	if (Section.sh_type == ELF::SHT_SYMTAB) {
4930	SymTabSection = &Section;
4931	break;
4932	}
4933	}
4934	if (!SymTabSection) {
4935	BC->errs() << "BOLT-WARNING: no symbol table found\n";
4936	return;
4937	}
4938
4939	const ELFShdrTy *StrTabSection =
4940	cantFail(Obj.getSection(SymTabSection->sh_link));
4941	std::string NewContents;
4942	std::string NewStrTab = std::string(
4943	File->getData().substr(StrTabSection->sh_offset, StrTabSection->sh_size));
4944	StringRef SecName = cantFail(Obj.getSectionName(*SymTabSection));
4945	StringRef StrSecName = cantFail(Obj.getSectionName(*StrTabSection));
4946
4947	NumLocalSymbols = 0;
4948	updateELFSymbolTable(
4949	File,
4950	/*IsDynSym=*/false,
4951	*SymTabSection,
4952	NewSectionIndex,
4953	[&](size_t Offset, const ELFSymTy &Sym) {
4954	if (Sym.getBinding() == ELF::STB_LOCAL)
4955	++NumLocalSymbols;
4956	NewContents.append(s: reinterpret_cast<const char *>(&Sym),
4957	n: sizeof(ELFSymTy));
4958	},
4959	[&](StringRef Str) {
4960	size_t Idx = NewStrTab.size();
4961	NewStrTab.append(str: NameResolver::restore(Name: Str).str());
4962	NewStrTab.append(n: 1, c: '\0');
4963	return Idx;
4964	});
4965
4966	BC->registerOrUpdateNoteSection(Name: SecName,
4967	Data: copyByteArray(Buffer: NewContents),
4968	Size: NewContents.size(),
4969	/*Alignment=*/1,
4970	/*IsReadOnly=*/true,
4971	ELFType: ELF::SHT_SYMTAB);
4972
4973	BC->registerOrUpdateNoteSection(Name: StrSecName,
4974	Data: copyByteArray(Buffer: NewStrTab),
4975	Size: NewStrTab.size(),
4976	/*Alignment=*/1,
4977	/*IsReadOnly=*/true,
4978	ELFType: ELF::SHT_STRTAB);
4979	}
4980
4981	template <typename ELFT>
4982	void RewriteInstance::patchELFAllocatableRelrSection(
4983	ELFObjectFile<ELFT> *File) {
4984	if (!DynamicRelrAddress)
4985	return;
4986
4987	raw_fd_ostream &OS = Out->os();
4988	const uint8_t PSize = BC->AsmInfo->getCodePointerSize();
4989	const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;
4990
4991	auto FixAddend = [&](const BinarySection &Section, const Relocation &Rel,
4992	uint64_t FileOffset) {
4993	// Fix relocation symbol value in place if no static relocation found
4994	// on the same address. We won't check the BF relocations here since it
4995	// is rare case and no optimization is required.
4996	if (Section.getRelocationAt(Offset: Rel.Offset))
4997	return;
4998
4999	// No fixup needed if symbol address was not changed
5000	const uint64_t Addend = getNewFunctionOrDataAddress(OldAddress: Rel.Addend);
5001	if (!Addend)
5002	return;
5003
5004	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Addend), Size: PSize, Offset: FileOffset);
5005	};
5006
5007	// Fill new relative relocation offsets set
5008	std::set<uint64_t> RelOffsets;
5009	for (const BinarySection &Section : BC->allocatableSections()) {
5010	const uint64_t SectionInputAddress = Section.getAddress();
5011	uint64_t SectionAddress = Section.getOutputAddress();
5012	if (!SectionAddress)
5013	SectionAddress = SectionInputAddress;
5014
5015	for (const Relocation &Rel : Section.dynamicRelocations()) {
5016	if (!Rel.isRelative())
5017	continue;
5018
5019	uint64_t RelOffset =
5020	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5021
5022	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5023	assert((RelOffset & 1) == 0 && "Wrong relocation offset");
5024	RelOffsets.emplace(args&: RelOffset);
5025	FixAddend(Section, Rel, RelOffset);
5026	}
5027	}
5028
5029	ErrorOr<BinarySection &> Section =
5030	BC->getSectionForAddress(Address: *DynamicRelrAddress);
5031	assert(Section && "cannot get .relr.dyn section");
5032	assert(Section->isRelr() && "Expected section to be SHT_RELR type");
5033	uint64_t RelrDynOffset = Section->getInputFileOffset();
5034	const uint64_t RelrDynEndOffset = RelrDynOffset + Section->getSize();
5035
5036	auto WriteRelr = [&](uint64_t Value) {
5037	if (RelrDynOffset + DynamicRelrEntrySize > RelrDynEndOffset) {
5038	BC->errs() << "BOLT-ERROR: Offset overflow for relr.dyn section\n";
5039	exit(status: 1);
5040	}
5041
5042	OS.pwrite(Ptr: reinterpret_cast<const char *>(&Value), Size: DynamicRelrEntrySize,
5043	Offset: RelrDynOffset);
5044	RelrDynOffset += DynamicRelrEntrySize;
5045	};
5046
5047	for (auto RelIt = RelOffsets.begin(); RelIt != RelOffsets.end();) {
5048	WriteRelr(*RelIt);
5049	uint64_t Base = *RelIt++ + PSize;
5050	while (1) {
5051	uint64_t Bitmap = 0;
5052	for (; RelIt != RelOffsets.end(); ++RelIt) {
5053	const uint64_t Delta = *RelIt - Base;
5054	if (Delta >= MaxDelta || Delta % PSize)
5055	break;
5056
5057	Bitmap |= (1ULL << (Delta / PSize));
5058	}
5059
5060	if (!Bitmap)
5061	break;
5062
5063	WriteRelr((Bitmap << 1) | 1);
5064	Base += MaxDelta;
5065	}
5066	}
5067
5068	// Fill the rest of the section with empty bitmap value
5069	while (RelrDynOffset != RelrDynEndOffset)
5070	WriteRelr(1);
5071	}
5072
5073	template <typename ELFT>
5074	void
5075	RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile<ELFT> *File) {
5076	using Elf_Rela = typename ELFT::Rela;
5077	raw_fd_ostream &OS = Out->os();
5078	const ELFFile<ELFT> &EF = File->getELFFile();
5079
5080	uint64_t RelDynOffset = 0, RelDynEndOffset = 0;
5081	uint64_t RelPltOffset = 0, RelPltEndOffset = 0;
5082
5083	auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start,
5084	uint64_t &End) {
5085	ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
5086	assert(Section && "cannot get relocation section");
5087	Start = Section->getInputFileOffset();
5088	End = Start + Section->getSize();
5089	};
5090
5091	if (!DynamicRelocationsAddress && !PLTRelocationsAddress)
5092	return;
5093
5094	if (DynamicRelocationsAddress)
5095	setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset,
5096	RelDynEndOffset);
5097
5098	if (PLTRelocationsAddress)
5099	setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset,
5100	RelPltEndOffset);
5101
5102	DynamicRelativeRelocationsCount = 0;
5103
5104	auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) {
5105	OS.pwrite(Ptr: reinterpret_cast<const char *>(RelA), Size: sizeof(*RelA), Offset);
5106	Offset += sizeof(*RelA);
5107	};
5108
5109	auto writeRelocations = [&](bool PatchRelative) {
5110	for (BinarySection &Section : BC->allocatableSections()) {
5111	const uint64_t SectionInputAddress = Section.getAddress();
5112	uint64_t SectionAddress = Section.getOutputAddress();
5113	if (!SectionAddress)
5114	SectionAddress = SectionInputAddress;
5115
5116	for (const Relocation &Rel : Section.dynamicRelocations()) {
5117	const bool IsRelative = Rel.isRelative();
5118	if (PatchRelative != IsRelative)
5119	continue;
5120
5121	if (IsRelative)
5122	++DynamicRelativeRelocationsCount;
5123
5124	Elf_Rela NewRelA;
5125	MCSymbol *Symbol = Rel.Symbol;
5126	uint32_t SymbolIdx = 0;
5127	uint64_t Addend = Rel.Addend;
5128	uint64_t RelOffset =
5129	getNewFunctionOrDataAddress(OldAddress: SectionInputAddress + Rel.Offset);
5130
5131	RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;
5132	if (Rel.Symbol) {
5133	SymbolIdx = getOutputDynamicSymbolIndex(Symbol);
5134	} else {
5135	// Usually this case is used for R_*_(I)RELATIVE relocations
5136	const uint64_t Address = getNewFunctionOrDataAddress(OldAddress: Addend);
5137	if (Address)
5138	Addend = Address;
5139	}
5140
5141	NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL());
5142	NewRelA.r_offset = RelOffset;
5143	NewRelA.r_addend = Addend;
5144
5145	const bool IsJmpRel = IsJmpRelocation.contains(Val: Rel.Type);
5146	uint64_t &Offset = IsJmpRel ? RelPltOffset : RelDynOffset;
5147	const uint64_t &EndOffset =
5148	IsJmpRel ? RelPltEndOffset : RelDynEndOffset;
5149	if (!Offset || !EndOffset) {
5150	BC->errs() << "BOLT-ERROR: Invalid offsets for dynamic relocation\n";
5151	exit(status: 1);
5152	}
5153
5154	if (Offset + sizeof(NewRelA) > EndOffset) {
5155	BC->errs() << "BOLT-ERROR: Offset overflow for dynamic relocation\n";
5156	exit(status: 1);
5157	}
5158
5159	writeRela(&NewRelA, Offset);
5160	}
5161	}
5162	};
5163
5164	// Place R_*_RELATIVE relocations in RELA section if RELR is not presented.
5165	// The dynamic linker expects all R_*_RELATIVE relocations in RELA
5166	// to be emitted first.
5167	if (!DynamicRelrAddress)
5168	writeRelocations(/* PatchRelative */ true);
5169	writeRelocations(/* PatchRelative */ false);
5170
5171	auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) {
5172	if (!Offset)
5173	return;
5174
5175	typename ELFObjectFile<ELFT>::Elf_Rela RelA;
5176	RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL());
5177	RelA.r_offset = 0;
5178	RelA.r_addend = 0;
5179	while (Offset < EndOffset)
5180	writeRela(&RelA, Offset);
5181
5182	assert(Offset == EndOffset && "Unexpected section overflow");
5183	};
5184
5185	// Fill the rest of the sections with R_*_NONE relocations
5186	fillNone(RelDynOffset, RelDynEndOffset);
5187	fillNone(RelPltOffset, RelPltEndOffset);
5188	}
5189
5190	template <typename ELFT>
5191	void RewriteInstance::patchELFGOT(ELFObjectFile<ELFT> *File) {
5192	raw_fd_ostream &OS = Out->os();
5193
5194	SectionRef GOTSection;
5195	for (const SectionRef &Section : File->sections()) {
5196	StringRef SectionName = cantFail(ValOrErr: Section.getName());
5197	if (SectionName == ".got") {
5198	GOTSection = Section;
5199	break;
5200	}
5201	}
5202	if (!GOTSection.getObject()) {
5203	if (!BC->IsStaticExecutable)
5204	BC->errs() << "BOLT-INFO: no .got section found\n";
5205	return;
5206	}
5207
5208	StringRef GOTContents = cantFail(ValOrErr: GOTSection.getContents());
5209	for (const uint64_t *GOTEntry =
5210	reinterpret_cast<const uint64_t *>(GOTContents.data());
5211	GOTEntry < reinterpret_cast<const uint64_t *>(GOTContents.data() +
5212	GOTContents.size());
5213	++GOTEntry) {
5214	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: *GOTEntry)) {
5215	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x"
5216	<< Twine::utohexstr(*GOTEntry) << " with 0x"
5217	<< Twine::utohexstr(NewAddress) << '\n');
5218	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewAddress), Size: sizeof(NewAddress),
5219	Offset: reinterpret_cast<const char *>(GOTEntry) -
5220	File->getData().data());
5221	}
5222	}
5223	}
5224
5225	template <typename ELFT>
5226	void RewriteInstance::patchELFDynamic(ELFObjectFile<ELFT> *File) {
5227	if (BC->IsStaticExecutable)
5228	return;
5229
5230	const ELFFile<ELFT> &Obj = File->getELFFile();
5231	raw_fd_ostream &OS = Out->os();
5232
5233	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5234	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5235
5236	// Locate DYNAMIC by looking through program headers.
5237	uint64_t DynamicOffset = 0;
5238	const Elf_Phdr *DynamicPhdr = nullptr;
5239	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5240	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5241	DynamicOffset = Phdr.p_offset;
5242	DynamicPhdr = &Phdr;
5243	assert(Phdr.p_memsz == Phdr.p_filesz && "dynamic sizes should match");
5244	break;
5245	}
5246	}
5247	assert(DynamicPhdr && "missing dynamic in ELF binary");
5248
5249	bool ZNowSet = false;
5250
5251	// Go through all dynamic entries and patch functions addresses with
5252	// new ones.
5253	typename ELFT::DynRange DynamicEntries =
5254	cantFail(Obj.dynamicEntries(), "error accessing dynamic table");
5255	auto DTB = DynamicEntries.begin();
5256	for (const Elf_Dyn &Dyn : DynamicEntries) {
5257	Elf_Dyn NewDE = Dyn;
5258	bool ShouldPatch = true;
5259	switch (Dyn.d_tag) {
5260	default:
5261	ShouldPatch = false;
5262	break;
5263	case ELF::DT_RELACOUNT:
5264	NewDE.d_un.d_val = DynamicRelativeRelocationsCount;
5265	break;
5266	case ELF::DT_INIT:
5267	case ELF::DT_FINI: {
5268	if (BC->HasRelocations) {
5269	if (uint64_t NewAddress = getNewFunctionAddress(OldAddress: Dyn.getPtr())) {
5270	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching dynamic entry of type "
5271	<< Dyn.getTag() << '\n');
5272	NewDE.d_un.d_ptr = NewAddress;
5273	}
5274	}
5275	RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary();
5276	if (RtLibrary && Dyn.getTag() == ELF::DT_FINI) {
5277	if (uint64_t Addr = RtLibrary->getRuntimeFiniAddress())
5278	NewDE.d_un.d_ptr = Addr;
5279	}
5280	if (RtLibrary && Dyn.getTag() == ELF::DT_INIT && !BC->HasInterpHeader) {
5281	if (auto Addr = RtLibrary->getRuntimeStartAddress()) {
5282	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set DT_INIT to 0x"
5283	<< Twine::utohexstr(Addr) << '\n');
5284	NewDE.d_un.d_ptr = Addr;
5285	}
5286	}
5287	break;
5288	}
5289	case ELF::DT_FLAGS:
5290	if (BC->RequiresZNow) {
5291	NewDE.d_un.d_val |= ELF::DF_BIND_NOW;
5292	ZNowSet = true;
5293	}
5294	break;
5295	case ELF::DT_FLAGS_1:
5296	if (BC->RequiresZNow) {
5297	NewDE.d_un.d_val |= ELF::DF_1_NOW;
5298	ZNowSet = true;
5299	}
5300	break;
5301	}
5302	if (ShouldPatch)
5303	OS.pwrite(Ptr: reinterpret_cast<const char *>(&NewDE), Size: sizeof(NewDE),
5304	Offset: DynamicOffset + (&Dyn - DTB) * sizeof(Dyn));
5305	}
5306
5307	if (BC->RequiresZNow && !ZNowSet) {
5308	BC->errs()
5309	<< "BOLT-ERROR: output binary requires immediate relocation "
5310	"processing which depends on DT_FLAGS or DT_FLAGS_1 presence in "
5311	".dynamic. Please re-link the binary with -znow.\n";
5312	exit(status: 1);
5313	}
5314	}
5315
5316	template <typename ELFT>
5317	Error RewriteInstance::readELFDynamic(ELFObjectFile<ELFT> *File) {
5318	const ELFFile<ELFT> &Obj = File->getELFFile();
5319
5320	using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;
5321	using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;
5322
5323	// Locate DYNAMIC by looking through program headers.
5324	const Elf_Phdr *DynamicPhdr = nullptr;
5325	for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {
5326	if (Phdr.p_type == ELF::PT_DYNAMIC) {
5327	DynamicPhdr = &Phdr;
5328	break;
5329	}
5330	}
5331
5332	if (!DynamicPhdr) {
5333	BC->outs() << "BOLT-INFO: static input executable detected\n";
5334	// TODO: static PIE executable might have dynamic header
5335	BC->IsStaticExecutable = true;
5336	return Error::success();
5337	}
5338
5339	if (DynamicPhdr->p_memsz != DynamicPhdr->p_filesz)
5340	return createStringError(EC: errc::executable_format_error,
5341	Msg: "dynamic section sizes should match");
5342
5343	// Go through all dynamic entries to locate entries of interest.
5344	auto DynamicEntriesOrErr = Obj.dynamicEntries();
5345	if (!DynamicEntriesOrErr)
5346	return DynamicEntriesOrErr.takeError();
5347	typename ELFT::DynRange DynamicEntries = DynamicEntriesOrErr.get();
5348
5349	for (const Elf_Dyn &Dyn : DynamicEntries) {
5350	switch (Dyn.d_tag) {
5351	case ELF::DT_INIT:
5352	if (!BC->HasInterpHeader) {
5353	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Set start function address\n");
5354	BC->StartFunctionAddress = Dyn.getPtr();
5355	}
5356	break;
5357	case ELF::DT_FINI:
5358	BC->FiniAddress = Dyn.getPtr();
5359	break;
5360	case ELF::DT_FINI_ARRAY:
5361	BC->FiniArrayAddress = Dyn.getPtr();
5362	break;
5363	case ELF::DT_FINI_ARRAYSZ:
5364	BC->FiniArraySize = Dyn.getPtr();
5365	break;
5366	case ELF::DT_RELA:
5367	DynamicRelocationsAddress = Dyn.getPtr();
5368	break;
5369	case ELF::DT_RELASZ:
5370	DynamicRelocationsSize = Dyn.getVal();
5371	break;
5372	case ELF::DT_JMPREL:
5373	PLTRelocationsAddress = Dyn.getPtr();
5374	break;
5375	case ELF::DT_PLTRELSZ:
5376	PLTRelocationsSize = Dyn.getVal();
5377	break;
5378	case ELF::DT_RELACOUNT:
5379	DynamicRelativeRelocationsCount = Dyn.getVal();
5380	break;
5381	case ELF::DT_RELR:
5382	DynamicRelrAddress = Dyn.getPtr();
5383	break;
5384	case ELF::DT_RELRSZ:
5385	DynamicRelrSize = Dyn.getVal();
5386	break;
5387	case ELF::DT_RELRENT:
5388	DynamicRelrEntrySize = Dyn.getVal();
5389	break;
5390	}
5391	}
5392
5393	if (!DynamicRelocationsAddress || !DynamicRelocationsSize) {
5394	DynamicRelocationsAddress.reset();
5395	DynamicRelocationsSize = 0;
5396	}
5397
5398	if (!PLTRelocationsAddress || !PLTRelocationsSize) {
5399	PLTRelocationsAddress.reset();
5400	PLTRelocationsSize = 0;
5401	}
5402
5403	if (!DynamicRelrAddress || !DynamicRelrSize) {
5404	DynamicRelrAddress.reset();
5405	DynamicRelrSize = 0;
5406	} else if (!DynamicRelrEntrySize) {
5407	BC->errs() << "BOLT-ERROR: expected DT_RELRENT to be presented "
5408	<< "in DYNAMIC section\n";
5409	exit(status: 1);
5410	} else if (DynamicRelrSize % DynamicRelrEntrySize) {
5411	BC->errs() << "BOLT-ERROR: expected RELR table size to be divisible "
5412	<< "by RELR entry size\n";
5413	exit(status: 1);
5414	}
5415
5416	return Error::success();
5417	}
5418
5419	uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {
5420	const BinaryFunction *Function = BC->getBinaryFunctionAtAddress(Address: OldAddress);
5421	if (!Function)
5422	return 0;
5423
5424	return Function->getOutputAddress();
5425	}
5426
5427	uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) {
5428	if (uint64_t Function = getNewFunctionAddress(OldAddress))
5429	return Function;
5430
5431	const BinaryData *BD = BC->getBinaryDataAtAddress(Address: OldAddress);
5432	if (BD && BD->isMoved())
5433	return BD->getOutputAddress();
5434
5435	return 0;
5436	}
5437
5438	void RewriteInstance::rewriteFile() {
5439	std::error_code EC;
5440	Out = std::make_unique<ToolOutputFile>(args&: opts::OutputFilename, args&: EC,
5441	args: sys::fs::OF_None);
5442	check_error(EC, Message: "cannot create output executable file");
5443
5444	raw_fd_ostream &OS = Out->os();
5445
5446	// Copy allocatable part of the input.
5447	OS << InputFile->getData().substr(Start: 0, N: FirstNonAllocatableOffset);
5448
5449	auto Streamer = BC->createStreamer(OS);
5450	// Make sure output stream has enough reserved space, otherwise
5451	// pwrite() will fail.
5452	uint64_t Offset = OS.seek(off: getFileOffsetForAddress(Address: NextAvailableAddress));
5453	(void)Offset;
5454	assert(Offset == getFileOffsetForAddress(NextAvailableAddress) &&
5455	"error resizing output file");
5456
5457	// Overwrite functions with fixed output address. This is mostly used by
5458	// non-relocation mode, with one exception: injected functions are covered
5459	// here in both modes.
5460	uint64_t CountOverwrittenFunctions = 0;
5461	uint64_t OverwrittenScore = 0;
5462	for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
5463	if (Function->getImageAddress() == 0 || Function->getImageSize() == 0)
5464	continue;
5465
5466	if (Function->getImageSize() > Function->getMaxSize()) {
5467	assert(!BC->isX86() && "Unexpected large function.");
5468	if (opts::Verbosity >= 1)
5469	BC->errs() << "BOLT-WARNING: new function size (0x"
5470	<< Twine::utohexstr(Val: Function->getImageSize())
5471	<< ") is larger than maximum allowed size (0x"
5472	<< Twine::utohexstr(Val: Function->getMaxSize())
5473	<< ") for function " << *Function << '\n';
5474
5475	// Remove jump table sections that this function owns in non-reloc mode
5476	// because we don't want to write them anymore.
5477	if (!BC->HasRelocations && opts::JumpTables == JTS_BASIC) {
5478	for (auto &JTI : Function->JumpTables) {
5479	JumpTable *JT = JTI.second;
5480	BinarySection &Section = JT->getOutputSection();
5481	BC->deregisterSection(Section);
5482	}
5483	}
5484	continue;
5485	}
5486
5487	const auto HasAddress = [](const FunctionFragment &FF) {
5488	return FF.empty() ||
5489	(FF.getImageAddress() != 0 && FF.getImageSize() != 0);
5490	};
5491	const bool SplitFragmentsHaveAddress =
5492	llvm::all_of(Range: Function->getLayout().getSplitFragments(), P: HasAddress);
5493	if (Function->isSplit() && !SplitFragmentsHaveAddress) {
5494	const auto HasNoAddress = [](const FunctionFragment &FF) {
5495	return FF.getImageAddress() == 0 && FF.getImageSize() == 0;
5496	};
5497	assert(llvm::all_of(Function->getLayout().getSplitFragments(),
5498	HasNoAddress) &&
5499	"Some split fragments have an address while others do not");
5500	(void)HasNoAddress;
5501	continue;
5502	}
5503
5504	OverwrittenScore += Function->getFunctionScore();
5505	++CountOverwrittenFunctions;
5506
5507	// Overwrite function in the output file.
5508	if (opts::Verbosity >= 2)
5509	BC->outs() << "BOLT: rewriting function \"" << *Function << "\"\n";
5510
5511	OS.pwrite(Ptr: reinterpret_cast<char *>(Function->getImageAddress()),
5512	Size: Function->getImageSize(), Offset: Function->getFileOffset());
5513
5514	// Write nops at the end of the function.
5515	if (Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {
5516	uint64_t Pos = OS.tell();
5517	OS.seek(off: Function->getFileOffset() + Function->getImageSize());
5518	BC->MAB->writeNopData(
5519	OS, Count: Function->getMaxSize() - Function->getImageSize(), STI: &*BC->STI);
5520
5521	OS.seek(off: Pos);
5522	}
5523
5524	if (!Function->isSplit())
5525	continue;
5526
5527	// Write cold part
5528	if (opts::Verbosity >= 2) {
5529	BC->outs() << formatv(Fmt: "BOLT: rewriting function \"{0}\" (split parts)\n",
5530	Vals&: *Function);
5531	}
5532
5533	for (const FunctionFragment &FF :
5534	Function->getLayout().getSplitFragments()) {
5535	OS.pwrite(Ptr: reinterpret_cast<char *>(FF.getImageAddress()),
5536	Size: FF.getImageSize(), Offset: FF.getFileOffset());
5537	}
5538	}
5539
5540	// Print function statistics for non-relocation mode.
5541	if (!BC->HasRelocations) {
5542	BC->outs() << "BOLT: " << CountOverwrittenFunctions << " out of "
5543	<< BC->getBinaryFunctions().size()
5544	<< " functions were overwritten.\n";
5545	if (BC->TotalScore != 0) {
5546	double Coverage = OverwrittenScore / (double)BC->TotalScore * 100.0;
5547	BC->outs() << format(Fmt: "BOLT-INFO: rewritten functions cover %.2lf",
5548	Vals: Coverage)
5549	<< "% of the execution count of simple functions of "
5550	"this binary\n";
5551	}
5552	}
5553
5554	if (BC->HasRelocations && opts::TrapOldCode) {
5555	uint64_t SavedPos = OS.tell();
5556	// Overwrite function body to make sure we never execute these instructions.
5557	for (auto &BFI : BC->getBinaryFunctions()) {
5558	BinaryFunction &BF = BFI.second;
5559	if (!BF.getFileOffset() || !BF.isEmitted())
5560	continue;
5561	OS.seek(off: BF.getFileOffset());
5562	StringRef TrapInstr = BC->MIB->getTrapFillValue();
5563	unsigned NInstr = BF.getMaxSize() / TrapInstr.size();
5564	for (unsigned I = 0; I < NInstr; ++I)
5565	OS.write(Ptr: TrapInstr.data(), Size: TrapInstr.size());
5566	}
5567	OS.seek(off: SavedPos);
5568	}
5569
5570	// Write all allocatable sections - reloc-mode text is written here as well
5571	for (BinarySection &Section : BC->allocatableSections()) {
5572	if (!Section.isFinalized() || !Section.getOutputData())
5573	continue;
5574	if (Section.isLinkOnly())
5575	continue;
5576
5577	if (opts::Verbosity >= 1)
5578	BC->outs() << "BOLT: writing new section " << Section.getName()
5579	<< "\n data at 0x"
5580	<< Twine::utohexstr(Val: Section.getAllocAddress()) << "\n of size "
5581	<< Section.getOutputSize() << "\n at offset "
5582	<< Section.getOutputFileOffset() << '\n';
5583	OS.pwrite(Ptr: reinterpret_cast<const char *>(Section.getOutputData()),
5584	Size: Section.getOutputSize(), Offset: Section.getOutputFileOffset());
5585	}
5586
5587	for (BinarySection &Section : BC->allocatableSections())
5588	Section.flushPendingRelocations(OS, Resolver: [this](const MCSymbol *S) {
5589	return getNewValueForSymbol(Name: S->getName());
5590	});
5591
5592	// If .eh_frame is present create .eh_frame_hdr.
5593	if (EHFrameSection)
5594	writeEHFrameHeader();
5595
5596	// Add BOLT Addresses Translation maps to allow profile collection to
5597	// happen in the output binary
5598	if (opts::EnableBAT)
5599	addBATSection();
5600
5601	// Patch program header table.
5602	if (!BC->IsLinuxKernel)
5603	patchELFPHDRTable();
5604
5605	// Finalize memory image of section string table.
5606	finalizeSectionStringTable();
5607
5608	// Update symbol tables.
5609	patchELFSymTabs();
5610
5611	patchBuildID();
5612
5613	if (opts::EnableBAT)
5614	encodeBATSection();
5615
5616	// Copy non-allocatable sections once allocatable part is finished.
5617	rewriteNoteSections();
5618
5619	if (BC->HasRelocations) {
5620	patchELFAllocatableRelaSections();
5621	patchELFAllocatableRelrSection();
5622	patchELFGOT();
5623	}
5624
5625	// Patch dynamic section/segment.
5626	patchELFDynamic();
5627
5628	// Update ELF book-keeping info.
5629	patchELFSectionHeaderTable();
5630
5631	if (opts::PrintSections) {
5632	BC->outs() << "BOLT-INFO: Sections after processing:\n";
5633	BC->printSections(OS&: BC->outs());
5634	}
5635
5636	Out->keep();
5637	EC = sys::fs::setPermissions(
5638	Path: opts::OutputFilename,
5639	Permissions: static_cast<sys::fs::perms>(sys::fs::perms::all_all &
5640	~sys::fs::getUmask()));
5641	check_error(EC, Message: "cannot set permissions of output file");
5642	}
5643
5644	void RewriteInstance::writeEHFrameHeader() {
5645	BinarySection *NewEHFrameSection =
5646	getSection(Name: getNewSecPrefix() + getEHFrameSectionName());
5647
5648	// No need to update the header if no new .eh_frame was created.
5649	if (!NewEHFrameSection)
5650	return;
5651
5652	DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true,
5653	NewEHFrameSection->getOutputAddress());
5654	Error E = NewEHFrame.parse(Data: DWARFDataExtractor(
5655	NewEHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(),
5656	BC->AsmInfo->getCodePointerSize()));
5657	check_error(E: std::move(E), Message: "failed to parse EH frame");
5658
5659	uint64_t RelocatedEHFrameAddress = 0;
5660	StringRef RelocatedEHFrameContents;
5661	BinarySection *RelocatedEHFrameSection =
5662	getSection(Name: ".relocated" + getEHFrameSectionName());
5663	if (RelocatedEHFrameSection) {
5664	RelocatedEHFrameAddress = RelocatedEHFrameSection->getOutputAddress();
5665	RelocatedEHFrameContents = RelocatedEHFrameSection->getOutputContents();
5666	}
5667	DWARFDebugFrame RelocatedEHFrame(BC->TheTriple->getArch(), true,
5668	RelocatedEHFrameAddress);
5669	Error Er = RelocatedEHFrame.parse(Data: DWARFDataExtractor(
5670	RelocatedEHFrameContents, BC->AsmInfo->isLittleEndian(),
5671	BC->AsmInfo->getCodePointerSize()));
5672	check_error(E: std::move(Er), Message: "failed to parse EH frame");
5673
5674	LLVM_DEBUG(dbgs() << "BOLT: writing a new .eh_frame_hdr\n");
5675
5676	NextAvailableAddress =
5677	appendPadding(OS&: Out->os(), Offset: NextAvailableAddress, Alignment: EHFrameHdrAlign);
5678
5679	const uint64_t EHFrameHdrOutputAddress = NextAvailableAddress;
5680	const uint64_t EHFrameHdrFileOffset =
5681	getFileOffsetForAddress(Address: NextAvailableAddress);
5682
5683	std::vector<char> NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(
5684	OldEHFrame: RelocatedEHFrame, NewEHFrame, EHFrameHeaderAddress: EHFrameHdrOutputAddress, FailedAddresses);
5685
5686	assert(Out->os().tell() == EHFrameHdrFileOffset && "offset mismatch");
5687	Out->os().write(Ptr: NewEHFrameHdr.data(), Size: NewEHFrameHdr.size());
5688
5689	const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,
5690	/*IsText=*/false,
5691	/*IsAllocatable=*/true);
5692	BinarySection *OldEHFrameHdrSection = getSection(Name: ".eh_frame_hdr");
5693	if (OldEHFrameHdrSection)
5694	OldEHFrameHdrSection->setOutputName(getOrgSecPrefix() + ".eh_frame_hdr");
5695
5696	BinarySection &EHFrameHdrSec = BC->registerOrUpdateSection(
5697	Name: getNewSecPrefix() + ".eh_frame_hdr", ELFType: ELF::SHT_PROGBITS, ELFFlags: Flags, Data: nullptr,
5698	Size: NewEHFrameHdr.size(), /*Alignment=*/1);
5699	EHFrameHdrSec.setOutputFileOffset(EHFrameHdrFileOffset);
5700	EHFrameHdrSec.setOutputAddress(EHFrameHdrOutputAddress);
5701	EHFrameHdrSec.setOutputName(".eh_frame_hdr");
5702
5703	NextAvailableAddress += EHFrameHdrSec.getOutputSize();
5704
5705	// Merge new .eh_frame with the relocated original so that gdb can locate all
5706	// FDEs.
5707	if (RelocatedEHFrameSection) {
5708	const uint64_t NewEHFrameSectionSize =
5709	RelocatedEHFrameSection->getOutputAddress() +
5710	RelocatedEHFrameSection->getOutputSize() -
5711	NewEHFrameSection->getOutputAddress();
5712	NewEHFrameSection->updateContents(NewData: NewEHFrameSection->getOutputData(),
5713	NewSize: NewEHFrameSectionSize);
5714	BC->deregisterSection(Section&: *RelocatedEHFrameSection);
5715	}
5716
5717	LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of .eh_frame after merge is "
5718	<< NewEHFrameSection->getOutputSize() << '\n');
5719	}
5720
5721	uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) {
5722	auto Value = Linker->lookupSymbol(Name);
5723	if (Value)
5724	return *Value;
5725
5726	// Return the original value if we haven't emitted the symbol.
5727	BinaryData *BD = BC->getBinaryDataByName(Name);
5728	if (!BD)
5729	return 0;
5730
5731	return BD->getAddress();
5732	}
5733
5734	uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const {
5735	// Check if it's possibly part of the new segment.
5736	if (Address >= NewTextSegmentAddress)
5737	return Address - NewTextSegmentAddress + NewTextSegmentOffset;
5738
5739	// Find an existing segment that matches the address.
5740	const auto SegmentInfoI = BC->SegmentMapInfo.upper_bound(x: Address);
5741	if (SegmentInfoI == BC->SegmentMapInfo.begin())
5742	return 0;
5743
5744	const SegmentInfo &SegmentInfo = std::prev(x: SegmentInfoI)->second;
5745	if (Address < SegmentInfo.Address ||
5746	Address >= SegmentInfo.Address + SegmentInfo.FileSize)
5747	return 0;
5748
5749	return SegmentInfo.FileOffset + Address - SegmentInfo.Address;
5750	}
5751
5752	bool RewriteInstance::willOverwriteSection(StringRef SectionName) {
5753	if (llvm::is_contained(Range: SectionsToOverwrite, Element: SectionName))
5754	return true;
5755	if (llvm::is_contained(Range&: DebugSectionsToOverwrite, Element: SectionName))
5756	return true;
5757
5758	ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
5759	return Section && Section->isAllocatable() && Section->isFinalized();
5760	}
5761
5762	bool RewriteInstance::isDebugSection(StringRef SectionName) {
5763	if (SectionName.starts_with(Prefix: ".debug_") ||
5764	SectionName.starts_with(Prefix: ".zdebug_") || SectionName == ".gdb_index" ||
5765	SectionName == ".stab" || SectionName == ".stabstr")
5766	return true;
5767
5768	return false;
5769	}
5770