1//===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Contains utilities for adding indirections and breaking up modules.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15
16#include "llvm/ADT/StringMap.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/ExecutionEngine/JITSymbol.h"
19#include "llvm/ExecutionEngine/Orc/Core.h"
20#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
21#include "llvm/Support/Error.h"
22#include "llvm/Support/Memory.h"
23#include "llvm/Support/Process.h"
24#include "llvm/Transforms/Utils/ValueMapper.h"
25#include <algorithm>
26#include <cassert>
27#include <cstdint>
28#include <functional>
29#include <future>
30#include <map>
31#include <memory>
32#include <system_error>
33#include <utility>
34#include <vector>
35
36namespace llvm {
37
38class Constant;
39class Function;
40class FunctionType;
41class GlobalAlias;
42class GlobalVariable;
43class Module;
44class PointerType;
45class Triple;
46class Twine;
47class Value;
48class MCDisassembler;
49class MCInstrAnalysis;
50
51namespace jitlink {
52class LinkGraph;
53class Symbol;
54} // namespace jitlink
55
56namespace orc {
57
58/// Base class for pools of compiler re-entry trampolines.
59/// These trampolines are callable addresses that save all register state
60/// before calling a supplied function to return the trampoline landing
61/// address, then restore all state before jumping to that address. They
62/// are used by various ORC APIs to support lazy compilation
63class TrampolinePool {
64public:
65 using NotifyLandingResolvedFunction =
66 unique_function<void(JITTargetAddress) const>;
67
68 using ResolveLandingFunction = unique_function<void(
69 JITTargetAddress TrampolineAddr,
70 NotifyLandingResolvedFunction OnLandingResolved) const>;
71
72 virtual ~TrampolinePool();
73
74 /// Get an available trampoline address.
75 /// Returns an error if no trampoline can be created.
76 Expected<JITTargetAddress> getTrampoline() {
77 std::lock_guard<std::mutex> Lock(TPMutex);
78 if (AvailableTrampolines.empty()) {
79 if (auto Err = grow())
80 return std::move(Err);
81 }
82 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
83 auto TrampolineAddr = AvailableTrampolines.back();
84 AvailableTrampolines.pop_back();
85 return TrampolineAddr;
86 }
87
88 /// Returns the given trampoline to the pool for re-use.
89 void releaseTrampoline(JITTargetAddress TrampolineAddr) {
90 std::lock_guard<std::mutex> Lock(TPMutex);
91 AvailableTrampolines.push_back(TrampolineAddr);
92 }
93
94protected:
95 virtual Error grow() = 0;
96
97 std::mutex TPMutex;
98 std::vector<JITTargetAddress> AvailableTrampolines;
99};
100
101/// A trampoline pool for trampolines within the current process.
102template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
103public:
104 /// Creates a LocalTrampolinePool with the given RunCallback function.
105 /// Returns an error if this function is unable to correctly allocate, write
106 /// and protect the resolver code block.
107 static Expected<std::unique_ptr<LocalTrampolinePool>>
108 Create(ResolveLandingFunction ResolveLanding) {
109 Error Err = Error::success();
110
111 auto LTP = std::unique_ptr<LocalTrampolinePool>(
112 new LocalTrampolinePool(std::move(ResolveLanding), Err));
113
114 if (Err)
115 return std::move(Err);
116 return std::move(LTP);
117 }
118
119private:
120 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
121 LocalTrampolinePool<ORCABI> *TrampolinePool =
122 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
123
124 std::promise<JITTargetAddress> LandingAddressP;
125 auto LandingAddressF = LandingAddressP.get_future();
126
127 TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId),
128 [&](JITTargetAddress LandingAddress) {
129 LandingAddressP.set_value(LandingAddress);
130 });
131 return LandingAddressF.get();
132 }
133
134 LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err)
135 : ResolveLanding(std::move(ResolveLanding)) {
136
137 ErrorAsOutParameter _(&Err);
138
139 /// Try to set up the resolver block.
140 std::error_code EC;
141 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
142 ORCABI::ResolverCodeSize, nullptr,
143 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
144 if (EC) {
145 Err = errorCodeToError(EC);
146 return;
147 }
148
149 ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()),
150 pointerToJITTargetAddress(ResolverBlock.base()),
151 pointerToJITTargetAddress(&reenter),
152 pointerToJITTargetAddress(this));
153
154 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
155 sys::Memory::MF_READ |
156 sys::Memory::MF_EXEC);
157 if (EC) {
158 Err = errorCodeToError(EC);
159 return;
160 }
161 }
162
163 Error grow() override {
164 assert(AvailableTrampolines.empty() && "Growing prematurely?");
165
166 std::error_code EC;
167 auto TrampolineBlock =
168 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
169 sys::Process::getPageSizeEstimate(), nullptr,
170 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
171 if (EC)
172 return errorCodeToError(EC);
173
174 unsigned NumTrampolines =
175 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
176 ORCABI::TrampolineSize;
177
178 char *TrampolineMem = static_cast<char *>(TrampolineBlock.base());
179 ORCABI::writeTrampolines(
180 TrampolineMem, pointerToJITTargetAddress(TrampolineMem),
181 pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines);
182
183 for (unsigned I = 0; I < NumTrampolines; ++I)
184 AvailableTrampolines.push_back(pointerToJITTargetAddress(
185 TrampolineMem + (I * ORCABI::TrampolineSize)));
186
187 if (auto EC = sys::Memory::protectMappedMemory(
188 TrampolineBlock.getMemoryBlock(),
189 sys::Memory::MF_READ | sys::Memory::MF_EXEC))
190 return errorCodeToError(EC);
191
192 TrampolineBlocks.push_back(std::move(TrampolineBlock));
193 return Error::success();
194 }
195
196 ResolveLandingFunction ResolveLanding;
197
198 sys::OwningMemoryBlock ResolverBlock;
199 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
200};
201
202/// Target-independent base class for compile callback management.
203class JITCompileCallbackManager {
204public:
205 using CompileFunction = std::function<JITTargetAddress()>;
206
207 virtual ~JITCompileCallbackManager() = default;
208
209 /// Reserve a compile callback.
210 Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
211
212 /// Execute the callback for the given trampoline id. Called by the JIT
213 /// to compile functions on demand.
214 JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
215
216protected:
217 /// Construct a JITCompileCallbackManager.
218 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
219 ExecutionSession &ES,
220 JITTargetAddress ErrorHandlerAddress)
221 : TP(std::move(TP)), ES(ES),
222 CallbacksJD(ES.createBareJITDylib("<Callbacks>")),
223 ErrorHandlerAddress(ErrorHandlerAddress) {}
224
225 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
226 this->TP = std::move(TP);
227 }
228
229private:
230 std::mutex CCMgrMutex;
231 std::unique_ptr<TrampolinePool> TP;
232 ExecutionSession &ES;
233 JITDylib &CallbacksJD;
234 JITTargetAddress ErrorHandlerAddress;
235 std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
236 size_t NextCallbackId = 0;
237};
238
239/// Manage compile callbacks for in-process JITs.
240template <typename ORCABI>
241class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
242public:
243 /// Create a new LocalJITCompileCallbackManager.
244 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
245 Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
246 Error Err = Error::success();
247 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
248 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
249 if (Err)
250 return std::move(Err);
251 return std::move(CCMgr);
252 }
253
254private:
255 /// Construct a InProcessJITCompileCallbackManager.
256 /// @param ErrorHandlerAddress The address of an error handler in the target
257 /// process to be used if a compile callback fails.
258 LocalJITCompileCallbackManager(ExecutionSession &ES,
259 JITTargetAddress ErrorHandlerAddress,
260 Error &Err)
261 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
262 using NotifyLandingResolvedFunction =
263 TrampolinePool::NotifyLandingResolvedFunction;
264
265 ErrorAsOutParameter _(&Err);
266 auto TP = LocalTrampolinePool<ORCABI>::Create(
267 [this](JITTargetAddress TrampolineAddr,
268 NotifyLandingResolvedFunction NotifyLandingResolved) {
269 NotifyLandingResolved(executeCompileCallback(TrampolineAddr));
270 });
271
272 if (!TP) {
273 Err = TP.takeError();
274 return;
275 }
276
277 setTrampolinePool(std::move(*TP));
278 }
279};
280
281/// Base class for managing collections of named indirect stubs.
282class IndirectStubsManager {
283public:
284 /// Map type for initializing the manager. See init.
285 using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
286
287 virtual ~IndirectStubsManager() = default;
288
289 /// Create a single stub with the given name, target address and flags.
290 virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
291 JITSymbolFlags StubFlags) = 0;
292
293 /// Create StubInits.size() stubs with the given names, target
294 /// addresses, and flags.
295 virtual Error createStubs(const StubInitsMap &StubInits) = 0;
296
297 /// Find the stub with the given name. If ExportedStubsOnly is true,
298 /// this will only return a result if the stub's flags indicate that it
299 /// is exported.
300 virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
301
302 /// Find the implementation-pointer for the stub.
303 virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
304
305 /// Change the value of the implementation pointer for the stub.
306 virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
307
308private:
309 virtual void anchor();
310};
311
312template <typename ORCABI> class LocalIndirectStubsInfo {
313public:
314 LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
315 : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {}
316
317 static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs,
318 unsigned PageSize) {
319 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize);
320
321 assert((ISAS.StubBytes % PageSize == 0) &&
322 "StubBytes is not a page size multiple");
323 uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize);
324
325 // Allocate memory for stubs and pointers in one call.
326 std::error_code EC;
327 auto StubsAndPtrsMem =
328 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
329 ISAS.StubBytes + PointerAlloc, nullptr,
330 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
331 if (EC)
332 return errorCodeToError(EC);
333
334 sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes);
335 auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base());
336 auto PtrBlockAddress =
337 pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes;
338
339 ORCABI::writeIndirectStubsBlock(StubsBlockMem,
340 pointerToJITTargetAddress(StubsBlockMem),
341 PtrBlockAddress, ISAS.NumStubs);
342
343 if (auto EC = sys::Memory::protectMappedMemory(
344 StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC))
345 return errorCodeToError(EC);
346
347 return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem));
348 }
349
350 unsigned getNumStubs() const { return NumStubs; }
351
352 void *getStub(unsigned Idx) const {
353 return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize;
354 }
355
356 void **getPtr(unsigned Idx) const {
357 char *PtrsBase =
358 static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize;
359 return reinterpret_cast<void **>(PtrsBase) + Idx;
360 }
361
362private:
363 unsigned NumStubs = 0;
364 sys::OwningMemoryBlock StubsMem;
365};
366
367/// IndirectStubsManager implementation for the host architecture, e.g.
368/// OrcX86_64. (See OrcArchitectureSupport.h).
369template <typename TargetT>
370class LocalIndirectStubsManager : public IndirectStubsManager {
371public:
372 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
373 JITSymbolFlags StubFlags) override {
374 std::lock_guard<std::mutex> Lock(StubsMutex);
375 if (auto Err = reserveStubs(1))
376 return Err;
377
378 createStubInternal(StubName, StubAddr, StubFlags);
379
380 return Error::success();
381 }
382
383 Error createStubs(const StubInitsMap &StubInits) override {
384 std::lock_guard<std::mutex> Lock(StubsMutex);
385 if (auto Err = reserveStubs(StubInits.size()))
386 return Err;
387
388 for (const auto &Entry : StubInits)
389 createStubInternal(Entry.first(), Entry.second.first,
390 Entry.second.second);
391
392 return Error::success();
393 }
394
395 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
396 std::lock_guard<std::mutex> Lock(StubsMutex);
397 auto I = StubIndexes.find(Name);
398 if (I == StubIndexes.end())
399 return nullptr;
400 auto Key = I->second.first;
401 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
402 assert(StubAddr && "Missing stub address");
403 auto StubTargetAddr =
404 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
405 auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
406 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
407 return nullptr;
408 return StubSymbol;
409 }
410
411 JITEvaluatedSymbol findPointer(StringRef Name) override {
412 std::lock_guard<std::mutex> Lock(StubsMutex);
413 auto I = StubIndexes.find(Name);
414 if (I == StubIndexes.end())
415 return nullptr;
416 auto Key = I->second.first;
417 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
418 assert(PtrAddr && "Missing pointer address");
419 auto PtrTargetAddr =
420 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
421 return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
422 }
423
424 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
425 using AtomicIntPtr = std::atomic<uintptr_t>;
426
427 std::lock_guard<std::mutex> Lock(StubsMutex);
428 auto I = StubIndexes.find(Name);
429 assert(I != StubIndexes.end() && "No stub pointer for symbol");
430 auto Key = I->second.first;
431 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
432 IndirectStubsInfos[Key.first].getPtr(Key.second));
433 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
434 return Error::success();
435 }
436
437private:
438 Error reserveStubs(unsigned NumStubs) {
439 if (NumStubs <= FreeStubs.size())
440 return Error::success();
441
442 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
443 unsigned NewBlockId = IndirectStubsInfos.size();
444 auto ISI =
445 LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize);
446 if (!ISI)
447 return ISI.takeError();
448 for (unsigned I = 0; I < ISI->getNumStubs(); ++I)
449 FreeStubs.push_back(std::make_pair(NewBlockId, I));
450 IndirectStubsInfos.push_back(std::move(*ISI));
451 return Error::success();
452 }
453
454 void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
455 JITSymbolFlags StubFlags) {
456 auto Key = FreeStubs.back();
457 FreeStubs.pop_back();
458 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
459 jitTargetAddressToPointer<void *>(InitAddr);
460 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
461 }
462
463 unsigned PageSize = sys::Process::getPageSizeEstimate();
464 std::mutex StubsMutex;
465 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
466 using StubKey = std::pair<uint16_t, uint16_t>;
467 std::vector<StubKey> FreeStubs;
468 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
469};
470
471/// Create a local compile callback manager.
472///
473/// The given target triple will determine the ABI, and the given
474/// ErrorHandlerAddress will be used by the resulting compile callback
475/// manager if a compile callback fails.
476Expected<std::unique_ptr<JITCompileCallbackManager>>
477createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
478 JITTargetAddress ErrorHandlerAddress);
479
480/// Create a local indriect stubs manager builder.
481///
482/// The given target triple will determine the ABI.
483std::function<std::unique_ptr<IndirectStubsManager>()>
484createLocalIndirectStubsManagerBuilder(const Triple &T);
485
486/// Build a function pointer of FunctionType with the given constant
487/// address.
488///
489/// Usage example: Turn a trampoline address into a function pointer constant
490/// for use in a stub.
491Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
492
493/// Create a function pointer with the given type, name, and initializer
494/// in the given Module.
495GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
496 Constant *Initializer);
497
498/// Turn a function declaration into a stub function that makes an
499/// indirect call using the given function pointer.
500void makeStub(Function &F, Value &ImplPointer);
501
502/// Promotes private symbols to global hidden, and renames to prevent clashes
503/// with other promoted symbols. The same SymbolPromoter instance should be
504/// used for all symbols to be added to a single JITDylib.
505class SymbolLinkagePromoter {
506public:
507 /// Promote symbols in the given module. Returns the set of global values
508 /// that have been renamed/promoted.
509 std::vector<GlobalValue *> operator()(Module &M);
510
511private:
512 unsigned NextId = 0;
513};
514
515/// Clone a function declaration into a new module.
516///
517/// This function can be used as the first step towards creating a callback
518/// stub (see makeStub), or moving a function body (see moveFunctionBody).
519///
520/// If the VMap argument is non-null, a mapping will be added between F and
521/// the new declaration, and between each of F's arguments and the new
522/// declaration's arguments. This map can then be passed in to moveFunction to
523/// move the function body if required. Note: When moving functions between
524/// modules with these utilities, all decls should be cloned (and added to a
525/// single VMap) before any bodies are moved. This will ensure that references
526/// between functions all refer to the versions in the new module.
527Function *cloneFunctionDecl(Module &Dst, const Function &F,
528 ValueToValueMapTy *VMap = nullptr);
529
530/// Move the body of function 'F' to a cloned function declaration in a
531/// different module (See related cloneFunctionDecl).
532///
533/// If the target function declaration is not supplied via the NewF parameter
534/// then it will be looked up via the VMap.
535///
536/// This will delete the body of function 'F' from its original parent module,
537/// but leave its declaration.
538void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
539 ValueMaterializer *Materializer = nullptr,
540 Function *NewF = nullptr);
541
542/// Clone a global variable declaration into a new module.
543GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
544 ValueToValueMapTy *VMap = nullptr);
545
546/// Move global variable GV from its parent module to cloned global
547/// declaration in a different module.
548///
549/// If the target global declaration is not supplied via the NewGV parameter
550/// then it will be looked up via the VMap.
551///
552/// This will delete the initializer of GV from its original parent module,
553/// but leave its declaration.
554void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
555 ValueToValueMapTy &VMap,
556 ValueMaterializer *Materializer = nullptr,
557 GlobalVariable *NewGV = nullptr);
558
559/// Clone a global alias declaration into a new module.
560GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
561 ValueToValueMapTy &VMap);
562
563/// Clone module flags metadata into the destination module.
564void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
565 ValueToValueMapTy &VMap);
566
567/// Introduce relocations to \p Sym in its own definition if there are any
568/// pointers formed via PC-relative address that do not already have a
569/// relocation.
570///
571/// This is useful when introducing indirection via a stub function at link time
572/// without compiler support. If a function pointer is formed without a
573/// relocation, e.g. in the definition of \c foo
574///
575/// \code
576/// _foo:
577/// leaq -7(%rip), rax # form pointer to _foo without relocation
578/// _bar:
579/// leaq (%rip), %rax # uses X86_64_RELOC_SIGNED to '_foo'
580/// \endcode
581///
582/// the pointer to \c _foo computed by \c _foo and \c _bar may differ if we
583/// introduce a stub for _foo. If the pointer is used as a key, this may be
584/// observable to the program. This pass will attempt to introduce the missing
585/// "self-relocation" on the leaq instruction.
586///
587/// This is based on disassembly and should be considered "best effort". It may
588/// silently fail to add relocations.
589Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
590 jitlink::LinkGraph &G,
591 MCDisassembler &Disassembler,
592 MCInstrAnalysis &MIA);
593
594} // end namespace orc
595
596} // end namespace llvm
597
598#endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
599

source code of llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h