1//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Contains utilities for adding indirections and breaking up modules.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15
16#include "llvm/ADT/StringMap.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ExecutionEngine/JITSymbol.h"
19#include "llvm/ExecutionEngine/Orc/Core.h"
20#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/Memory.h"
23#include "llvm/Support/Process.h"
24#include "llvm/Transforms/Utils/ValueMapper.h"
25#include <algorithm>
26#include <cassert>
27#include <cstdint>
28#include <functional>
29#include <future>
30#include <map>
31#include <memory>
32#include <system_error>
33#include <utility>
34#include <vector>
35
36namespace llvm {
37
38class Constant;
39class Function;
40class FunctionType;
41class GlobalAlias;
42class GlobalVariable;
43class Module;
44class PointerType;
45class Triple;
46class Twine;
47class Value;
48class MCDisassembler;
49class MCInstrAnalysis;
50
51namespace jitlink {
52class LinkGraph;
53class Symbol;
54} // namespace jitlink
55
56namespace orc {
57
58/// Base class for pools of compiler re-entry trampolines.
59/// These trampolines are callable addresses that save all register state
60/// before calling a supplied function to return the trampoline landing
61/// address, then restore all state before jumping to that address. They
62/// are used by various ORC APIs to support lazy compilation
63class TrampolinePool {
64public:
65 using NotifyLandingResolvedFunction =
66 unique_function<void(ExecutorAddr) const>;
67
68 using ResolveLandingFunction = unique_function<void(
69 ExecutorAddr TrampolineAddr,
70 NotifyLandingResolvedFunction OnLandingResolved) const>;
71
72 virtual ~TrampolinePool();
73
74 /// Get an available trampoline address.
75 /// Returns an error if no trampoline can be created.
76 Expected<ExecutorAddr> getTrampoline() {
77 std::lock_guard<std::mutex> Lock(TPMutex);
78 if (AvailableTrampolines.empty()) {
79 if (auto Err = grow())
80 return std::move(Err);
81 }
82 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
83 auto TrampolineAddr = AvailableTrampolines.back();
84 AvailableTrampolines.pop_back();
85 return TrampolineAddr;
86 }
87
88 /// Returns the given trampoline to the pool for re-use.
89 void releaseTrampoline(ExecutorAddr TrampolineAddr) {
90 std::lock_guard<std::mutex> Lock(TPMutex);
91 AvailableTrampolines.push_back(x: TrampolineAddr);
92 }
93
94protected:
95 virtual Error grow() = 0;
96
97 std::mutex TPMutex;
98 std::vector<ExecutorAddr> AvailableTrampolines;
99};
100
101/// A trampoline pool for trampolines within the current process.
102template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
103public:
104 /// Creates a LocalTrampolinePool with the given RunCallback function.
105 /// Returns an error if this function is unable to correctly allocate, write
106 /// and protect the resolver code block.
107 static Expected<std::unique_ptr<LocalTrampolinePool>>
108 Create(ResolveLandingFunction ResolveLanding) {
109 Error Err = Error::success();
110
111 auto LTP = std::unique_ptr<LocalTrampolinePool>(
112 new LocalTrampolinePool(std::move(ResolveLanding), Err));
113
114 if (Err)
115 return std::move(Err);
116 return std::move(LTP);
117 }
118
119private:
120 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
121 LocalTrampolinePool<ORCABI> *TrampolinePool =
122 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
123
124 std::promise<ExecutorAddr> LandingAddressP;
125 auto LandingAddressF = LandingAddressP.get_future();
126
127 TrampolinePool->ResolveLanding(ExecutorAddr::fromPtr(Ptr: TrampolineId),
128 [&](ExecutorAddr LandingAddress) {
129 LandingAddressP.set_value(LandingAddress);
130 });
131 return LandingAddressF.get().getValue();
132 }
133
134 LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
135 : ResolveLanding(std::move(ResolveLanding)) {
136
137 ErrorAsOutParameter _(&Err);
138
139 /// Try to set up the resolver block.
140 std::error_code EC;
141 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
142 NumBytes: ORCABI::ResolverCodeSize, NearBlock: nullptr,
143 Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
144 if (EC) {
145 Err = errorCodeToError(EC);
146 return;
147 }
148
149 ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),
150 ExecutorAddr::fromPtr(Ptr: ResolverBlock.base()),
151 ExecutorAddr::fromPtr(Ptr: &reenter),
152 ExecutorAddr::fromPtr(this));
153
154 EC = sys::Memory::protectMappedMemory(Block: ResolverBlock.getMemoryBlock(),
155 Flags: sys::Memory::MF_READ |
156 sys::Memory::MF_EXEC);
157 if (EC) {
158 Err = errorCodeToError(EC);
159 return;
160 }
161 }
162
163 Error grow() override {
164 assert(AvailableTrampolines.empty() && "Growing prematurely?");
165
166 std::error_code EC;
167 auto TrampolineBlock =
168 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
169 NumBytes: sys::Process::getPageSizeEstimate(), NearBlock: nullptr,
170 Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
171 if (EC)
172 return errorCodeToError(EC);
173
174 unsigned NumTrampolines =
175 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
176 ORCABI::TrampolineSize;
177
178 char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());
179 ORCABI::writeTrampolines(
180 TrampolineMem, ExecutorAddr::fromPtr(Ptr: TrampolineMem),
181 ExecutorAddr::fromPtr(Ptr: ResolverBlock.base()), NumTrampolines);
182
183 for (unsigned I = 0; I < NumTrampolines; ++I)
184 AvailableTrampolines.push_back(
185 ExecutorAddr::fromPtr(TrampolineMem + (I * ORCABI::TrampolineSize)));
186
187 if (auto EC = sys::Memory::protectMappedMemory(
188 Block: TrampolineBlock.getMemoryBlock(),
189 Flags: sys::Memory::MF_READ | sys::Memory::MF_EXEC))
190 return errorCodeToError(EC);
191
192 TrampolineBlocks.push_back(x: std::move(TrampolineBlock));
193 return Error::success();
194 }
195
196 ResolveLandingFunction ResolveLanding;
197
198 sys::OwningMemoryBlock ResolverBlock;
199 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
200};
201
202/// Target-independent base class for compile callback management.
203class JITCompileCallbackManager {
204public:
205 using CompileFunction = std::function<ExecutorAddr()>;
206
207 virtual ~JITCompileCallbackManager() = default;
208
209 /// Reserve a compile callback.
210 Expected<ExecutorAddr> getCompileCallback(CompileFunction Compile);
211
212 /// Execute the callback for the given trampoline id. Called by the JIT
213 /// to compile functions on demand.
214 ExecutorAddr executeCompileCallback(ExecutorAddr TrampolineAddr);
215
216protected:
217 /// Construct a JITCompileCallbackManager.
218 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
219 ExecutionSession &ES,
220 ExecutorAddr ErrorHandlerAddress)
221 : TP(std::move(TP)), ES(ES),
222 CallbacksJD(ES.createBareJITDylib(Name: "<Callbacks>")),
223 ErrorHandlerAddress(ErrorHandlerAddress) {}
224
225 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
226 this->TP = std::move(TP);
227 }
228
229private:
230 std::mutex CCMgrMutex;
231 std::unique_ptr<TrampolinePool> TP;
232 ExecutionSession &ES;
233 JITDylib &CallbacksJD;
234 ExecutorAddr ErrorHandlerAddress;
235 std::map<ExecutorAddr, SymbolStringPtr> AddrToSymbol;
236 size_t NextCallbackId = 0;
237};
238
239/// Manage compile callbacks for in-process JITs.
240template <typename ORCABI>
241class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
242public:
243 /// Create a new LocalJITCompileCallbackManager.
244 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
245 Create(ExecutionSession &ES, ExecutorAddr ErrorHandlerAddress) {
246 Error Err = Error::success();
247 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
248 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
249 if (Err)
250 return std::move(Err);
251 return std::move(CCMgr);
252 }
253
254private:
255 /// Construct a InProcessJITCompileCallbackManager.
256 /// @param ErrorHandlerAddress The address of an error handler in the target
257 /// process to be used if a compile callback fails.
258 LocalJITCompileCallbackManager(ExecutionSession &ES,
259 ExecutorAddr ErrorHandlerAddress, Error &Err)
260 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
261 using NotifyLandingResolvedFunction =
262 TrampolinePool::NotifyLandingResolvedFunction;
263
264 ErrorAsOutParameter _(&Err);
265 auto TP = LocalTrampolinePool<ORCABI>::Create(
266 [this](ExecutorAddr TrampolineAddr,
267 NotifyLandingResolvedFunction NotifyLandingResolved) {
268 NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
269 });
270
271 if (!TP) {
272 Err = TP.takeError();
273 return;
274 }
275
276 setTrampolinePool(std::move(*TP));
277 }
278};
279
280/// Base class for managing collections of named indirect stubs.
281class IndirectStubsManager {
282public:
283 /// Map type for initializing the manager. See init.
284 using StubInitsMap = StringMap<std::pair<ExecutorAddr, JITSymbolFlags>>;
285
286 virtual ~IndirectStubsManager() = default;
287
288 /// Create a single stub with the given name, target address and flags.
289 virtual Error createStub(StringRef StubName, ExecutorAddr StubAddr,
290 JITSymbolFlags StubFlags) = 0;
291
292 /// Create StubInits.size() stubs with the given names, target
293 /// addresses, and flags.
294 virtual Error createStubs(const StubInitsMap &StubInits) = 0;
295
296 /// Find the stub with the given name. If ExportedStubsOnly is true,
297 /// this will only return a result if the stub's flags indicate that it
298 /// is exported.
299 virtual ExecutorSymbolDef findStub(StringRef Name,
300 bool ExportedStubsOnly) = 0;
301
302 /// Find the implementation-pointer for the stub.
303 virtual ExecutorSymbolDef findPointer(StringRef Name) = 0;
304
305 /// Change the value of the implementation pointer for the stub.
306 virtual Error updatePointer(StringRef Name, ExecutorAddr NewAddr) = 0;
307
308private:
309 virtual void anchor();
310};
311
312template <typename ORCABI> class LocalIndirectStubsInfo {
313public:
314 LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
315 : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}
316
317 static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,
318 unsigned PageSize) {
319 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);
320
321 assert((ISAS.StubBytes % PageSize == 0) &&
322 "StubBytes is not a page size multiple");
323 uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);
324
325 // Allocate memory for stubs and pointers in one call.
326 std::error_code EC;
327 auto StubsAndPtrsMem =
328 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
329 NumBytes: ISAS.StubBytes + PointerAlloc, NearBlock: nullptr,
330 Flags: sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
331 if (EC)
332 return errorCodeToError(EC);
333
334 sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);
335 auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());
336 auto PtrBlockAddress =
337 ExecutorAddr::fromPtr(Ptr: StubsBlockMem) + ISAS.StubBytes;
338
339 ORCABI::writeIndirectStubsBlock(StubsBlockMem,
340 ExecutorAddr::fromPtr(Ptr: StubsBlockMem),
341 PtrBlockAddress, ISAS.NumStubs);
342
343 if (auto EC = sys::Memory::protectMappedMemory(
344 Block: StubsBlock, Flags: sys::Memory::MF_READ | sys::Memory::MF_EXEC))
345 return errorCodeToError(EC);
346
347 return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));
348 }
349
350 unsigned getNumStubs() const { return NumStubs; }
351
352 void *getStub(unsigned Idx) const {
353 return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;
354 }
355
356 void **getPtr(unsigned Idx) const {
357 char *PtrsBase =
358 static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;
359 return reinterpret_cast<void **>(PtrsBase) + Idx;
360 }
361
362private:
363 unsigned NumStubs = 0;
364 sys::OwningMemoryBlock StubsMem;
365};
366
367/// IndirectStubsManager implementation for the host architecture, e.g.
368/// OrcX86_64. (See OrcArchitectureSupport.h).
369template <typename TargetT>
370class LocalIndirectStubsManager : public IndirectStubsManager {
371public:
372 Error createStub(StringRef StubName, ExecutorAddr StubAddr,
373 JITSymbolFlags StubFlags) override {
374 std::lock_guard<std::mutex> Lock(StubsMutex);
375 if (auto Err = reserveStubs(NumStubs: 1))
376 return Err;
377
378 createStubInternal(StubName, InitAddr: StubAddr, StubFlags);
379
380 return Error::success();
381 }
382
383 Error createStubs(const StubInitsMap &StubInits) override {
384 std::lock_guard<std::mutex> Lock(StubsMutex);
385 if (auto Err = reserveStubs(NumStubs: StubInits.size()))
386 return Err;
387
388 for (const auto &Entry : StubInits)
389 createStubInternal(StubName: Entry.first(), InitAddr: Entry.second.first,
390 StubFlags: Entry.second.second);
391
392 return Error::success();
393 }
394
395 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override {
396 std::lock_guard<std::mutex> Lock(StubsMutex);
397 auto I = StubIndexes.find(Key: Name);
398 if (I == StubIndexes.end())
399 return ExecutorSymbolDef();
400 auto Key = I->second.first;
401 void *StubPtr = IndirectStubsInfos[Key.first].getStub(Key.second);
402 assert(StubPtr && "Missing stub address");
403 auto StubAddr = ExecutorAddr::fromPtr(Ptr: StubPtr);
404 auto StubSymbol = ExecutorSymbolDef(StubAddr, I->second.second);
405 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
406 return ExecutorSymbolDef();
407 return StubSymbol;
408 }
409
410 ExecutorSymbolDef findPointer(StringRef Name) override {
411 std::lock_guard<std::mutex> Lock(StubsMutex);
412 auto I = StubIndexes.find(Key: Name);
413 if (I == StubIndexes.end())
414 return ExecutorSymbolDef();
415 auto Key = I->second.first;
416 void *PtrPtr = IndirectStubsInfos[Key.first].getPtr(Key.second);
417 assert(PtrPtr && "Missing pointer address");
418 auto PtrAddr = ExecutorAddr::fromPtr(Ptr: PtrPtr);
419 return ExecutorSymbolDef(PtrAddr, I->second.second);
420 }
421
422 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override {
423 using AtomicIntPtr = std::atomic<uintptr_t>;
424
425 std::lock_guard<std::mutex> Lock(StubsMutex);
426 auto I = StubIndexes.find(Key: Name);
427 assert(I != StubIndexes.end() && "No stub pointer for symbol");
428 auto Key = I->second.first;
429 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
430 IndirectStubsInfos[Key.first].getPtr(Key.second));
431 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr.getValue());
432 return Error::success();
433 }
434
435private:
436 Error reserveStubs(unsigned NumStubs) {
437 if (NumStubs <= FreeStubs.size())
438 return Error::success();
439
440 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
441 unsigned NewBlockId = IndirectStubsInfos.size();
442 auto ISI =
443 LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);
444 if (!ISI)
445 return ISI.takeError();
446 for (unsigned I = 0; I < ISI->getNumStubs(); ++I)
447 FreeStubs.push_back(x: std::make_pair(x&: NewBlockId, y&: I));
448 IndirectStubsInfos.push_back(std::move(*ISI));
449 return Error::success();
450 }
451
452 void createStubInternal(StringRef StubName, ExecutorAddr InitAddr,
453 JITSymbolFlags StubFlags) {
454 auto Key = FreeStubs.back();
455 FreeStubs.pop_back();
456 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
457 InitAddr.toPtr<void *>();
458 StubIndexes[StubName] = std::make_pair(x&: Key, y&: StubFlags);
459 }
460
461 unsigned PageSize = sys::Process::getPageSizeEstimate();
462 std::mutex StubsMutex;
463 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
464 using StubKey = std::pair<uint16_t, uint16_t>;
465 std::vector<StubKey> FreeStubs;
466 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
467};
468
469/// Create a local compile callback manager.
470///
471/// The given target triple will determine the ABI, and the given
472/// ErrorHandlerAddress will be used by the resulting compile callback
473/// manager if a compile callback fails.
474Expected<std::unique_ptr<JITCompileCallbackManager>>
475createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
476 ExecutorAddr ErrorHandlerAddress);
477
478/// Create a local indirect stubs manager builder.
479///
480/// The given target triple will determine the ABI.
481std::function<std::unique_ptr<IndirectStubsManager>()>
482createLocalIndirectStubsManagerBuilder(const Triple &T);
483
484/// Build a function pointer of FunctionType with the given constant
485/// address.
486///
487/// Usage example: Turn a trampoline address into a function pointer constant
488/// for use in a stub.
489Constant *createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr);
490
491/// Create a function pointer with the given type, name, and initializer
492/// in the given Module.
493GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
494 Constant *Initializer);
495
496/// Turn a function declaration into a stub function that makes an
497/// indirect call using the given function pointer.
498void makeStub(Function &F, Value &ImplPointer);
499
500/// Promotes private symbols to global hidden, and renames to prevent clashes
501/// with other promoted symbols. The same SymbolPromoter instance should be
502/// used for all symbols to be added to a single JITDylib.
503class SymbolLinkagePromoter {
504public:
505 /// Promote symbols in the given module. Returns the set of global values
506 /// that have been renamed/promoted.
507 std::vector<GlobalValue *> operator()(Module &M);
508
509private:
510 unsigned NextId = 0;
511};
512
513/// Clone a function declaration into a new module.
514///
515/// This function can be used as the first step towards creating a callback
516/// stub (see makeStub).
517///
518/// If the VMap argument is non-null, a mapping will be added between F and
519/// the new declaration, and between each of F's arguments and the new
520/// declaration's arguments. This map can then be passed in to moveFunction to
521/// move the function body if required. Note: When moving functions between
522/// modules with these utilities, all decls should be cloned (and added to a
523/// single VMap) before any bodies are moved. This will ensure that references
524/// between functions all refer to the versions in the new module.
525Function *cloneFunctionDecl(Module &Dst, const Function &F,
526 ValueToValueMapTy *VMap = nullptr);
527
528/// Clone a global variable declaration into a new module.
529GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
530 ValueToValueMapTy *VMap = nullptr);
531
532/// Clone a global alias declaration into a new module.
533GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
534 ValueToValueMapTy &VMap);
535
536/// Introduce relocations to \p Sym in its own definition if there are any
537/// pointers formed via PC-relative address that do not already have a
538/// relocation.
539///
540/// This is useful when introducing indirection via a stub function at link time
541/// without compiler support. If a function pointer is formed without a
542/// relocation, e.g. in the definition of \c foo
543///
544/// \code
545/// _foo:
546/// leaq -7(%rip), rax # form pointer to _foo without relocation
547/// _bar:
548/// leaq (%rip), %rax # uses X86_64_RELOC_SIGNED to '_foo'
549/// \endcode
550///
551/// the pointer to \c _foo computed by \c _foo and \c _bar may differ if we
552/// introduce a stub for _foo. If the pointer is used as a key, this may be
553/// observable to the program. This pass will attempt to introduce the missing
554/// "self-relocation" on the leaq instruction.
555///
556/// This is based on disassembly and should be considered "best effort". It may
557/// silently fail to add relocations.
558Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
559 jitlink::LinkGraph &G,
560 MCDisassembler &Disassembler,
561 MCInstrAnalysis &MIA);
562
563} // end namespace orc
564
565} // end namespace llvm
566
567#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
568

source code of llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h