1//===- bolt/Rewrite/ExecutableFileMemoryManager.cpp -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "bolt/Rewrite/ExecutableFileMemoryManager.h"
10#include "bolt/Rewrite/JITLinkLinker.h"
11#include "bolt/Rewrite/RewriteInstance.h"
12#include "llvm/ExecutionEngine/JITLink/JITLink.h"
13#include "llvm/Support/MemAlloc.h"
14
15#undef DEBUG_TYPE
16#define DEBUG_TYPE "efmm"
17
18using namespace llvm;
19using namespace object;
20using namespace bolt;
21
22namespace llvm {
23
24namespace bolt {
25
26namespace {
27
28SmallVector<jitlink::Section *> orderedSections(jitlink::LinkGraph &G) {
29 SmallVector<jitlink::Section *> Sections(
30 llvm::map_range(C: G.sections(), F: [](auto &S) { return &S; }));
31 llvm::sort(C&: Sections, Comp: [](const auto *LHS, const auto *RHS) {
32 return LHS->getOrdinal() < RHS->getOrdinal();
33 });
34 return Sections;
35}
36
37size_t sectionAlignment(const jitlink::Section &Section) {
38 assert(!Section.empty() && "Cannot get alignment for empty section");
39 return JITLinkLinker::orderedBlocks(Section).front()->getAlignment();
40}
41
42StringRef sectionName(const jitlink::Section &Section,
43 const BinaryContext &BC) {
44 auto Name = Section.getName();
45
46 if (BC.isMachO()) {
47 // JITLink "normalizes" section names as "SegmentName,SectionName" on
48 // Mach-O. BOLT internally refers to sections just by the section name so
49 // strip-off the segment name.
50 auto SegmentEnd = Name.find(C: ',');
51 assert(SegmentEnd != StringRef::npos && "Mach-O segment not found");
52 Name = Name.substr(Start: SegmentEnd + 1);
53 }
54
55 return Name;
56}
57
58struct SectionAllocInfo {
59 void *Address;
60 size_t Size;
61 size_t Alignment;
62};
63
64struct AllocInfo {
65 SmallVector<SectionAllocInfo, 8> AllocatedSections;
66
67 ~AllocInfo() {
68 for (auto &Section : AllocatedSections)
69 deallocate_buffer(Ptr: Section.Address, Size: Section.Size, Alignment: Section.Alignment);
70 }
71
72 SectionAllocInfo allocateSection(const jitlink::Section &Section) {
73 auto Size = JITLinkLinker::sectionSize(Section);
74 auto Alignment = sectionAlignment(Section);
75 auto *Buf = allocate_buffer(Size, Alignment);
76 SectionAllocInfo Alloc{.Address: Buf, .Size: Size, .Alignment: Alignment};
77 AllocatedSections.push_back(Elt: Alloc);
78 return Alloc;
79 }
80};
81
82struct BOLTInFlightAlloc : ExecutableFileMemoryManager::InFlightAlloc {
83 // Even though this is passed using a raw pointer in FinalizedAlloc, we keep
84 // it in a unique_ptr as long as possible to enjoy automatic cleanup when
85 // something goes wrong.
86 std::unique_ptr<AllocInfo> Alloc;
87
88public:
89 BOLTInFlightAlloc(std::unique_ptr<AllocInfo> Alloc)
90 : Alloc(std::move(Alloc)) {}
91
92 virtual void abandon(OnAbandonedFunction OnAbandoned) override {
93 OnAbandoned(Error::success());
94 }
95
96 virtual void finalize(OnFinalizedFunction OnFinalized) override {
97 OnFinalized(ExecutableFileMemoryManager::FinalizedAlloc(
98 orc::ExecutorAddr::fromPtr(Ptr: Alloc.release())));
99 }
100};
101
102} // anonymous namespace
103
104void ExecutableFileMemoryManager::updateSection(
105 const jitlink::Section &JLSection, uint8_t *Contents, size_t Size,
106 size_t Alignment) {
107 auto SectionID = JLSection.getName();
108 auto SectionName = sectionName(Section: JLSection, BC);
109 auto Prot = JLSection.getMemProt();
110 auto IsCode = (Prot & orc::MemProt::Exec) != orc::MemProt::None;
111 auto IsReadOnly = (Prot & orc::MemProt::Write) == orc::MemProt::None;
112
113 // Register a debug section as a note section.
114 if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) {
115 BinarySection &Section =
116 BC.registerOrUpdateNoteSection(Name: SectionName, Data: Contents, Size, Alignment);
117 Section.setSectionID(SectionID);
118 assert(!Section.isAllocatable() && "note sections cannot be allocatable");
119 return;
120 }
121
122 if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" ||
123 SectionName == "" || SectionName.starts_with(Prefix: ".rela.")))
124 return;
125
126 SmallVector<char, 256> Buf;
127 if (ObjectsLoaded > 0) {
128 if (BC.isELF()) {
129 SectionName = (Twine(SectionName) + ".bolt.extra." + Twine(ObjectsLoaded))
130 .toStringRef(Out&: Buf);
131 } else if (BC.isMachO()) {
132 assert((SectionName == "__text" || SectionName == "__data" ||
133 SectionName == "__fini" || SectionName == "__setup" ||
134 SectionName == "__cstring" || SectionName == "__literal16") &&
135 "Unexpected section in the instrumentation library");
136 // Sections coming from the instrumentation runtime are prefixed with "I".
137 SectionName = ("I" + Twine(SectionName)).toStringRef(Out&: Buf);
138 }
139 }
140
141 BinarySection *Section = nullptr;
142 if (!OrgSecPrefix.empty() && SectionName.starts_with(Prefix: OrgSecPrefix)) {
143 // Update the original section contents.
144 ErrorOr<BinarySection &> OrgSection =
145 BC.getUniqueSectionByName(SectionName: SectionName.substr(Start: OrgSecPrefix.length()));
146 assert(OrgSection && OrgSection->isAllocatable() &&
147 "Original section must exist and be allocatable.");
148
149 Section = &OrgSection.get();
150 Section->updateContents(NewData: Contents, NewSize: Size);
151 } else {
152 // If the input contains a section with the section name, rename it in the
153 // output file to avoid the section name conflict and emit the new section
154 // under a unique internal name.
155 ErrorOr<BinarySection &> OrgSection =
156 BC.getUniqueSectionByName(SectionName);
157 bool UsePrefix = false;
158 if (OrgSection && OrgSection->hasSectionRef()) {
159 OrgSection->setOutputName(OrgSecPrefix + SectionName);
160 UsePrefix = true;
161 }
162
163 // Register the new section under a unique name to avoid name collision with
164 // sections in the input file.
165 BinarySection &NewSection = BC.registerOrUpdateSection(
166 Name: UsePrefix ? NewSecPrefix + SectionName : SectionName, ELFType: ELF::SHT_PROGBITS,
167 ELFFlags: BinarySection::getFlags(IsReadOnly, IsText: IsCode, IsAllocatable: true), Data: Contents, Size,
168 Alignment);
169 if (UsePrefix)
170 NewSection.setOutputName(SectionName);
171 Section = &NewSection;
172 }
173
174 LLVM_DEBUG({
175 dbgs() << "BOLT: allocating "
176 << (IsCode ? "code" : (IsReadOnly ? "read-only data" : "data"))
177 << " section : " << Section->getOutputName() << " ("
178 << Section->getName() << ")"
179 << " with size " << Size << ", alignment " << Alignment << " at "
180 << Contents << ", ID = " << SectionID << "\n";
181 });
182
183 Section->setSectionID(SectionID);
184}
185
186void ExecutableFileMemoryManager::allocate(const jitlink::JITLinkDylib *JD,
187 jitlink::LinkGraph &G,
188 OnAllocatedFunction OnAllocated) {
189 auto Alloc = std::make_unique<AllocInfo>();
190
191 for (auto *Section : orderedSections(G)) {
192 if (Section->empty())
193 continue;
194
195 auto SectionAlloc = Alloc->allocateSection(Section: *Section);
196 updateSection(JLSection: *Section, Contents: static_cast<uint8_t *>(SectionAlloc.Address),
197 Size: SectionAlloc.Size, Alignment: SectionAlloc.Alignment);
198
199 size_t CurrentOffset = 0;
200 auto *Buf = static_cast<char *>(SectionAlloc.Address);
201 for (auto *Block : JITLinkLinker::orderedBlocks(Section: *Section)) {
202 CurrentOffset = jitlink::alignToBlock(Addr: CurrentOffset, B: *Block);
203 auto BlockSize = Block->getSize();
204 auto *BlockBuf = Buf + CurrentOffset;
205
206 if (Block->isZeroFill())
207 std::memset(s: BlockBuf, c: 0, n: BlockSize);
208 else
209 std::memcpy(dest: BlockBuf, src: Block->getContent().data(), n: BlockSize);
210
211 Block->setMutableContent({BlockBuf, Block->getSize()});
212 CurrentOffset += BlockSize;
213 }
214 }
215
216 OnAllocated(std::make_unique<BOLTInFlightAlloc>(args: std::move(Alloc)));
217}
218
219void ExecutableFileMemoryManager::deallocate(
220 std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {
221 for (auto &Alloc : Allocs)
222 delete Alloc.release().toPtr<AllocInfo *>();
223
224 OnDeallocated(Error::success());
225}
226
227} // namespace bolt
228
229} // namespace llvm
230

source code of bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp