1 | //===- elfnix_platform.cpp ------------------------------------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains code required to load the rest of the ELF-on-*IX runtime. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "elfnix_platform.h" |
14 | #include "common.h" |
15 | #include "compiler.h" |
16 | #include "error.h" |
17 | #include "jit_dispatch.h" |
18 | #include "record_section_tracker.h" |
19 | #include "wrapper_function_utils.h" |
20 | |
21 | #include <algorithm> |
22 | #include <map> |
23 | #include <mutex> |
24 | #include <sstream> |
25 | #include <string_view> |
26 | #include <unordered_map> |
27 | #include <vector> |
28 | |
29 | using namespace orc_rt; |
30 | using namespace orc_rt::elfnix; |
31 | |
32 | // Declare function tags for functions in the JIT process. |
33 | ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag) |
34 | ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag) |
35 | ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag) |
36 | |
37 | // eh-frame registration functions, made available via aliases |
38 | // installed by the Platform |
39 | extern "C"void __register_frame(const void *); |
40 | extern "C"void __deregister_frame(const void *); |
41 | |
42 | extern "C"void |
43 | __unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT; |
44 | extern "C"void |
45 | __unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT; |
46 | |
47 | namespace { |
48 | |
49 | struct TLSInfoEntry { |
50 | unsigned long Key = 0; |
51 | unsigned long DataAddress = 0; |
52 | }; |
53 | |
54 | struct TLSDescriptor { |
55 | void (*Resolver)(void *); |
56 | TLSInfoEntry *InfoEntry; |
57 | }; |
58 | |
59 | class ELFNixPlatformRuntimeState { |
60 | private: |
61 | struct AtExitEntry { |
62 | void (*Func)(void *); |
63 | void *Arg; |
64 | }; |
65 | |
66 | using AtExitsVector = std::vector<AtExitEntry>; |
67 | |
68 | struct PerJITDylibState { |
69 | std::string Name; |
70 | void *Header = nullptr; |
71 | size_t RefCount = 0; |
72 | size_t LinkedAgainstRefCount = 0; |
73 | bool AllowReinitialization = false; |
74 | AtExitsVector AtExits; |
75 | std::vector<PerJITDylibState *> Deps; |
76 | RecordSectionsTracker<void (*)()> RecordedInits; |
77 | |
78 | bool referenced() const { |
79 | return LinkedAgainstRefCount != 0 || RefCount != 0; |
80 | } |
81 | }; |
82 | |
83 | public: |
84 | static void initialize(void *DSOHandle); |
85 | static ELFNixPlatformRuntimeState &get(); |
86 | static void destroy(); |
87 | |
88 | ELFNixPlatformRuntimeState(void *DSOHandle); |
89 | |
90 | // Delete copy and move constructors. |
91 | ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete; |
92 | ELFNixPlatformRuntimeState & |
93 | operator=(const ELFNixPlatformRuntimeState &) = delete; |
94 | ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete; |
95 | ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete; |
96 | |
97 | Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR); |
98 | Error registerJITDylib(std::string &Name, void *Handle); |
99 | Error deregisterJITDylib(void *Handle); |
100 | Error registerInits(ExecutorAddr HeaderAddr, |
101 | std::vector<ExecutorAddrRange> Inits); |
102 | Error deregisterInits(ExecutorAddr HeaderAddr, |
103 | std::vector<ExecutorAddrRange> Inits); |
104 | Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR); |
105 | |
106 | const char *dlerror(); |
107 | void *dlopen(std::string_view Name, int Mode); |
108 | int dlupdate(void *DSOHandle); |
109 | int dlclose(void *DSOHandle); |
110 | void *dlsym(void *DSOHandle, std::string_view Symbol); |
111 | |
112 | int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); |
113 | void runAtExits(void *DSOHandle); |
114 | void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock, |
115 | PerJITDylibState &JDS); |
116 | |
117 | /// Returns the base address of the section containing ThreadData. |
118 | Expected<std::pair<const char *, size_t>> |
119 | getThreadDataSectionFor(const char *ThreadData); |
120 | |
121 | void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; } |
122 | |
123 | private: |
124 | PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); |
125 | PerJITDylibState *getJITDylibStateByName(std::string_view Path); |
126 | |
127 | Error registerThreadDataSection(span<const char> ThreadDataSection); |
128 | |
129 | Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, |
130 | std::string_view Symbol); |
131 | |
132 | Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock, |
133 | PerJITDylibState &JDS); |
134 | Expected<void *> dlopenImpl(std::string_view Path, int Mode); |
135 | Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock, |
136 | PerJITDylibState &JDS); |
137 | Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock, |
138 | PerJITDylibState &JDS, |
139 | ELFNixJITDylibDepInfoMap &DepInfo); |
140 | Error dlupdateImpl(void *DSOHandle); |
141 | Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock, |
142 | PerJITDylibState &JDS); |
143 | |
144 | Error dlcloseImpl(void *DSOHandle); |
145 | Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock, |
146 | PerJITDylibState &JDS); |
147 | |
148 | static ELFNixPlatformRuntimeState *MOPS; |
149 | |
150 | void *PlatformJDDSOHandle; |
151 | |
152 | // Frame registration functions: |
153 | void (*registerEHFrameSection)(const void *) = nullptr; |
154 | void (*deregisterEHFrameSection)(const void *) = nullptr; |
155 | |
156 | // FIXME: Move to thread-state. |
157 | std::string DLFcnError; |
158 | |
159 | std::recursive_mutex JDStatesMutex; |
160 | std::unordered_map<void *, PerJITDylibState> JDStates; |
161 | std::unordered_map<std::string, void *> JDNameToHeader; |
162 | |
163 | std::mutex ThreadDataSectionsMutex; |
164 | std::map<const char *, size_t> ThreadDataSections; |
165 | }; |
166 | |
167 | ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr; |
168 | |
169 | void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) { |
170 | assert(!MOPS && "ELFNixPlatformRuntimeState should be null"); |
171 | MOPS = new ELFNixPlatformRuntimeState(DSOHandle); |
172 | } |
173 | |
174 | ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() { |
175 | assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); |
176 | return *MOPS; |
177 | } |
178 | |
179 | void ELFNixPlatformRuntimeState::destroy() { |
180 | assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); |
181 | delete MOPS; |
182 | } |
183 | |
184 | ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle) |
185 | : PlatformJDDSOHandle(DSOHandle) { |
186 | if (__unw_add_dynamic_eh_frame_section && |
187 | __unw_remove_dynamic_eh_frame_section) { |
188 | registerEHFrameSection = __unw_add_dynamic_eh_frame_section; |
189 | deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section; |
190 | } else { |
191 | registerEHFrameSection = __register_frame; |
192 | deregisterEHFrameSection = __deregister_frame; |
193 | } |
194 | } |
195 | |
196 | Error ELFNixPlatformRuntimeState::registerObjectSections( |
197 | ELFNixPerObjectSectionsToRegister POSR) { |
198 | if (POSR.EHFrameSection.Start) |
199 | registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>()); |
200 | |
201 | if (POSR.ThreadDataSection.Start) { |
202 | if (auto Err = registerThreadDataSection( |
203 | ThreadDataSection: POSR.ThreadDataSection.toSpan<const char>())) |
204 | return Err; |
205 | } |
206 | |
207 | return Error::success(); |
208 | } |
209 | |
210 | Error ELFNixPlatformRuntimeState::deregisterObjectSections( |
211 | ELFNixPerObjectSectionsToRegister POSR) { |
212 | if (POSR.EHFrameSection.Start) |
213 | deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>()); |
214 | |
215 | return Error::success(); |
216 | } |
217 | |
218 | Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name, |
219 | void *Handle) { |
220 | std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); |
221 | |
222 | if (JDStates.count(x: Handle)) { |
223 | std::ostringstream ErrStream; |
224 | ErrStream << "Duplicate JITDylib registration for header "<< Handle |
225 | << " (name = "<< Name << ")"; |
226 | return make_error<StringError>(Args: ErrStream.str()); |
227 | } |
228 | |
229 | if (JDNameToHeader.count(x: Name)) { |
230 | std::ostringstream ErrStream; |
231 | ErrStream << "Duplicate JITDylib registration for header "<< Handle |
232 | << " (header = "<< Handle << ")"; |
233 | return make_error<StringError>(Args: ErrStream.str()); |
234 | } |
235 | |
236 | auto &JD = JDStates[Handle]; |
237 | JD.Header = Handle; |
238 | JD.Name = std::move(t&: Name); |
239 | JDNameToHeader[JD.Name] = Handle; |
240 | return Error::success(); |
241 | } |
242 | |
243 | Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) { |
244 | std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); |
245 | |
246 | auto I = JDStates.find(x: Handle); |
247 | if (I == JDStates.end()) { |
248 | std::ostringstream ErrStream; |
249 | ErrStream << "Attempted to deregister unrecognized header "<< Handle; |
250 | return make_error<StringError>(Args: ErrStream.str()); |
251 | } |
252 | |
253 | auto J = JDNameToHeader.find( |
254 | x: std::string(I->second.Name.data(), I->second.Name.size())); |
255 | assert(J != JDNameToHeader.end() && |
256 | "Missing JDNameToHeader entry for JITDylib"); |
257 | JDNameToHeader.erase(position: J); |
258 | JDStates.erase(position: I); |
259 | return Error::success(); |
260 | } |
261 | |
262 | Error ELFNixPlatformRuntimeState::registerInits( |
263 | ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) { |
264 | std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); |
265 | PerJITDylibState *JDS = |
266 | getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>()); |
267 | |
268 | if (!JDS) { |
269 | std::ostringstream ErrStream; |
270 | ErrStream << "Could not register object platform sections for " |
271 | "unrecognized header " |
272 | << HeaderAddr.toPtr<void *>(); |
273 | return make_error<StringError>(Args: ErrStream.str()); |
274 | } |
275 | |
276 | for (auto &I : Inits) { |
277 | JDS->RecordedInits.add(Sec: I.toSpan<void (*)()>()); |
278 | } |
279 | |
280 | return Error::success(); |
281 | } |
282 | |
283 | Error ELFNixPlatformRuntimeState::deregisterInits( |
284 | ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) { |
285 | |
286 | std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); |
287 | PerJITDylibState *JDS = |
288 | getJITDylibStateByHeaderAddr(DSOHandle: HeaderAddr.toPtr<void *>()); |
289 | |
290 | if (!JDS) { |
291 | std::ostringstream ErrStream; |
292 | ErrStream << "Could not register object platform sections for unrecognized " |
293 | "header " |
294 | << HeaderAddr.toPtr<void *>(); |
295 | return make_error<StringError>(Args: ErrStream.str()); |
296 | } |
297 | |
298 | for (auto &I : Inits) { |
299 | JDS->RecordedInits.removeIfPresent(R: I); |
300 | } |
301 | |
302 | return Error::success(); |
303 | } |
304 | |
305 | const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } |
306 | |
307 | void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { |
308 | if (auto H = dlopenImpl(Path, Mode)) |
309 | return *H; |
310 | else { |
311 | // FIXME: Make dlerror thread safe. |
312 | DLFcnError = toString(Err: H.takeError()); |
313 | return nullptr; |
314 | } |
315 | } |
316 | |
317 | int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) { |
318 | if (auto Err = dlupdateImpl(DSOHandle)) { |
319 | // FIXME: Make dlerror thread safe. |
320 | DLFcnError = toString(Err: std::move(t&: Err)); |
321 | return -1; |
322 | } |
323 | return 0; |
324 | } |
325 | |
326 | int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) { |
327 | if (auto Err = dlcloseImpl(DSOHandle)) { |
328 | DLFcnError = toString(Err: std::move(t&: Err)); |
329 | return -1; |
330 | } |
331 | return 0; |
332 | } |
333 | |
334 | void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, |
335 | std::string_view Symbol) { |
336 | auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); |
337 | if (!Addr) { |
338 | DLFcnError = toString(Err: Addr.takeError()); |
339 | return 0; |
340 | } |
341 | |
342 | return Addr->toPtr<void *>(); |
343 | } |
344 | |
345 | int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, |
346 | void *DSOHandle) { |
347 | // FIXME: Handle out-of-memory errors, returning -1 if OOM. |
348 | std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); |
349 | auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); |
350 | assert(JDS && "JITDylib state not initialized"); |
351 | JDS->AtExits.push_back(x: {.Func: F, .Arg: Arg}); |
352 | return 0; |
353 | } |
354 | |
355 | void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) { |
356 | std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex); |
357 | PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle); |
358 | |
359 | if (JDS) |
360 | runAtExits(JDStateLock&: Lock, JDS&: *JDS); |
361 | } |
362 | |
363 | void ELFNixPlatformRuntimeState::runAtExits( |
364 | std::unique_lock<std::recursive_mutex> &JDStateLock, |
365 | PerJITDylibState &JDS) { |
366 | AtExitsVector V = std::move(t&: JDS.AtExits); |
367 | |
368 | while (!V.empty()) { |
369 | auto &AE = V.back(); |
370 | AE.Func(AE.Arg); |
371 | V.pop_back(); |
372 | } |
373 | } |
374 | |
375 | Expected<std::pair<const char *, size_t>> |
376 | ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { |
377 | std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); |
378 | auto I = ThreadDataSections.upper_bound(x: ThreadData); |
379 | // Check that we have a valid entry conovering this address. |
380 | if (I == ThreadDataSections.begin()) |
381 | return make_error<StringError>(Args: "No thread local data section for key"); |
382 | I = std::prev(x: I); |
383 | if (ThreadData >= I->first + I->second) |
384 | return make_error<StringError>(Args: "No thread local data section for key"); |
385 | return *I; |
386 | } |
387 | |
388 | ELFNixPlatformRuntimeState::PerJITDylibState * |
389 | ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { |
390 | auto I = JDStates.find(x: DSOHandle); |
391 | if (I == JDStates.end()) |
392 | return nullptr; |
393 | |
394 | return &I->second; |
395 | } |
396 | |
397 | ELFNixPlatformRuntimeState::PerJITDylibState * |
398 | ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { |
399 | // FIXME: Avoid creating string copy here. |
400 | auto I = JDNameToHeader.find(x: std::string(Name.data(), Name.size())); |
401 | if (I == JDNameToHeader.end()) |
402 | return nullptr; |
403 | void *H = I->second; |
404 | auto J = JDStates.find(x: H); |
405 | assert(J != JDStates.end() && |
406 | "JITDylib has name map entry but no header map entry"); |
407 | return &J->second; |
408 | } |
409 | |
410 | Error ELFNixPlatformRuntimeState::registerThreadDataSection( |
411 | span<const char> ThreadDataSection) { |
412 | std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); |
413 | auto I = ThreadDataSections.upper_bound(x: ThreadDataSection.data()); |
414 | if (I != ThreadDataSections.begin()) { |
415 | auto J = std::prev(x: I); |
416 | if (J->first + J->second > ThreadDataSection.data()) |
417 | return make_error<StringError>(Args: "Overlapping .tdata sections"); |
418 | } |
419 | ThreadDataSections.insert( |
420 | position: I, x: std::make_pair(x: ThreadDataSection.data(), y: ThreadDataSection.size())); |
421 | return Error::success(); |
422 | } |
423 | |
424 | Expected<ExecutorAddr> |
425 | ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, |
426 | std::string_view Sym) { |
427 | Expected<ExecutorAddr> Result((ExecutorAddr())); |
428 | if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( |
429 | SPSExecutorAddr, |
430 | SPSString)>::call(Dispatch: JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag), |
431 | Result, Args: ExecutorAddr::fromPtr(Ptr: DSOHandle), Args: Sym)) |
432 | return std::move(t&: Err); |
433 | return Result; |
434 | } |
435 | |
436 | Error ELFNixPlatformRuntimeState::runInits( |
437 | std::unique_lock<std::recursive_mutex> &JDStatesLock, |
438 | PerJITDylibState &JDS) { |
439 | std::vector<span<void (*)()>> InitSections; |
440 | InitSections.reserve(n: JDS.RecordedInits.numNewSections()); |
441 | |
442 | JDS.RecordedInits.processNewSections( |
443 | ProcessSection: [&](span<void (*)()> Inits) { InitSections.push_back(x: Inits); }); |
444 | |
445 | JDStatesLock.unlock(); |
446 | for (auto Sec : InitSections) |
447 | for (auto *Init : Sec) |
448 | Init(); |
449 | |
450 | JDStatesLock.lock(); |
451 | |
452 | return Error::success(); |
453 | } |
454 | |
455 | Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path, |
456 | int Mode) { |
457 | std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex); |
458 | PerJITDylibState *JDS = getJITDylibStateByName(Name: Path); |
459 | |
460 | if (!JDS) |
461 | return make_error<StringError>(Args: "No registered JTIDylib for path "+ |
462 | std::string(Path.data(), Path.size())); |
463 | |
464 | if (auto Err = dlopenFull(JDStatesLock&: Lock, JDS&: *JDS)) |
465 | return std::move(t&: Err); |
466 | |
467 | ++JDS->RefCount; |
468 | |
469 | return JDS->Header; |
470 | } |
471 | |
472 | Error ELFNixPlatformRuntimeState::dlopenFull( |
473 | std::unique_lock<std::recursive_mutex> &JDStateLock, |
474 | PerJITDylibState &JDS) { |
475 | Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap())); |
476 | JDStateLock.unlock(); |
477 | if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>( |
478 | SPSExecutorAddr)>:: |
479 | call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo, |
480 | Args: ExecutorAddr::fromPtr(Ptr: JDS.Header))) |
481 | return Err; |
482 | JDStateLock.lock(); |
483 | |
484 | if (!DepInfo) |
485 | return DepInfo.takeError(); |
486 | |
487 | if (auto Err = dlopenInitialize(JDStatesLock&: JDStateLock, JDS, DepInfo&: *DepInfo)) |
488 | return Err; |
489 | |
490 | if (!DepInfo->empty()) { |
491 | std::ostringstream ErrStream; |
492 | ErrStream << "Encountered unrecognized dep-info key headers " |
493 | "while processing dlopen of " |
494 | << JDS.Name; |
495 | return make_error<StringError>(Args: ErrStream.str()); |
496 | } |
497 | |
498 | return Error::success(); |
499 | } |
500 | |
501 | Error ELFNixPlatformRuntimeState::dlopenInitialize( |
502 | std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS, |
503 | ELFNixJITDylibDepInfoMap &DepInfo) { |
504 | |
505 | auto I = DepInfo.find(x: ExecutorAddr::fromPtr(Ptr: JDS.Header)); |
506 | if (I == DepInfo.end()) |
507 | return Error::success(); |
508 | |
509 | auto Deps = std::move(t&: I->second); |
510 | DepInfo.erase(position: I); |
511 | |
512 | std::vector<PerJITDylibState *> OldDeps; |
513 | std::swap(x&: JDS.Deps, y&: OldDeps); |
514 | JDS.Deps.reserve(n: Deps.size()); |
515 | for (auto H : Deps) { |
516 | PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(DSOHandle: H.toPtr<void *>()); |
517 | if (!DepJDS) { |
518 | std::ostringstream ErrStream; |
519 | ErrStream << "Encountered unrecognized dep header "<< H.toPtr<void *>() |
520 | << " while initializing "<< JDS.Name; |
521 | return make_error<StringError>(Args: ErrStream.str()); |
522 | } |
523 | ++DepJDS->LinkedAgainstRefCount; |
524 | if (auto Err = dlopenInitialize(JDStatesLock, JDS&: *DepJDS, DepInfo)) |
525 | return Err; |
526 | } |
527 | |
528 | if (auto Err = runInits(JDStatesLock, JDS)) |
529 | return Err; |
530 | |
531 | for (auto *DepJDS : OldDeps) { |
532 | --DepJDS->LinkedAgainstRefCount; |
533 | if (!DepJDS->referenced()) |
534 | if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS)) |
535 | return Err; |
536 | } |
537 | return Error::success(); |
538 | } |
539 | |
540 | Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) { |
541 | std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex); |
542 | |
543 | // Try to find JITDylib state by name. |
544 | auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); |
545 | |
546 | if (!JDS) { |
547 | std::ostringstream ErrStream; |
548 | ErrStream << "No registered JITDylib for "<< DSOHandle; |
549 | return make_error<StringError>(Args: ErrStream.str()); |
550 | } |
551 | |
552 | if (!JDS->referenced()) |
553 | return make_error<StringError>(Args: "dlupdate failed, JITDylib must be open."); |
554 | |
555 | if (auto Err = dlupdateFull(JDStatesLock&: Lock, JDS&: *JDS)) |
556 | return Err; |
557 | |
558 | return Error::success(); |
559 | } |
560 | |
561 | Error ELFNixPlatformRuntimeState::dlupdateFull( |
562 | std::unique_lock<std::recursive_mutex> &JDStatesLock, |
563 | PerJITDylibState &JDS) { |
564 | // Call back to the JIT to push the initializers. |
565 | Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap())); |
566 | // Unlock so that we can accept the initializer update. |
567 | JDStatesLock.unlock(); |
568 | if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>( |
569 | SPSExecutorAddr)>:: |
570 | call(Dispatch: JITDispatch(&__orc_rt_elfnix_push_initializers_tag), Result&: DepInfo, |
571 | Args: ExecutorAddr::fromPtr(Ptr: JDS.Header))) |
572 | return Err; |
573 | JDStatesLock.lock(); |
574 | |
575 | if (!DepInfo) |
576 | return DepInfo.takeError(); |
577 | |
578 | if (auto Err = runInits(JDStatesLock, JDS)) |
579 | return Err; |
580 | |
581 | return Error::success(); |
582 | } |
583 | |
584 | Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { |
585 | |
586 | std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex); |
587 | PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle); |
588 | |
589 | if (!JDS) { |
590 | std::ostringstream ErrStream; |
591 | ErrStream << "No registered JITDylib for "<< DSOHandle; |
592 | return make_error<StringError>(Args: ErrStream.str()); |
593 | } |
594 | |
595 | --JDS->RefCount; |
596 | |
597 | if (!JDS->referenced()) |
598 | return dlcloseInitialize(JDStatesLock&: Lock, JDS&: *JDS); |
599 | |
600 | return Error::success(); |
601 | } |
602 | |
603 | Error ELFNixPlatformRuntimeState::dlcloseInitialize( |
604 | std::unique_lock<std::recursive_mutex> &JDStatesLock, |
605 | PerJITDylibState &JDS) { |
606 | runAtExits(JDStateLock&: JDStatesLock, JDS); |
607 | JDS.RecordedInits.reset(); |
608 | for (auto *DepJDS : JDS.Deps) |
609 | if (!JDS.referenced()) |
610 | if (auto Err = dlcloseInitialize(JDStatesLock, JDS&: *DepJDS)) |
611 | return Err; |
612 | |
613 | return Error::success(); |
614 | } |
615 | |
616 | class ELFNixPlatformRuntimeTLVManager { |
617 | public: |
618 | void *getInstance(const char *ThreadData); |
619 | |
620 | private: |
621 | std::unordered_map<const char *, char *> Instances; |
622 | std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; |
623 | }; |
624 | |
625 | void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { |
626 | auto I = Instances.find(x: ThreadData); |
627 | if (I != Instances.end()) |
628 | return I->second; |
629 | auto TDS = |
630 | ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); |
631 | if (!TDS) { |
632 | __orc_rt_log_error(ErrMsg: toString(Err: TDS.takeError()).c_str()); |
633 | return nullptr; |
634 | } |
635 | |
636 | auto &Allocated = AllocatedSections[TDS->first]; |
637 | if (!Allocated) { |
638 | Allocated = std::make_unique<char[]>(num: TDS->second); |
639 | memcpy(dest: Allocated.get(), src: TDS->first, n: TDS->second); |
640 | } |
641 | size_t ThreadDataDelta = ThreadData - TDS->first; |
642 | assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); |
643 | |
644 | char *Instance = Allocated.get() + ThreadDataDelta; |
645 | Instances[ThreadData] = Instance; |
646 | return Instance; |
647 | } |
648 | |
649 | void destroyELFNixTLVMgr(void *ELFNixTLVMgr) { |
650 | delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr); |
651 | } |
652 | |
653 | } // end anonymous namespace |
654 | |
655 | //------------------------------------------------------------------------------ |
656 | // JIT entry points |
657 | //------------------------------------------------------------------------------ |
658 | |
659 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
660 | __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) { |
661 | return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( |
662 | ArgData, ArgSize, |
663 | Handler: [](ExecutorAddr DSOHandle) { |
664 | ELFNixPlatformRuntimeState::initialize( |
665 | DSOHandle: DSOHandle.toPtr<void *>()); |
666 | return Error::success(); |
667 | }) |
668 | .release(); |
669 | } |
670 | |
671 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
672 | __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) { |
673 | return WrapperFunction<SPSError()>::handle( |
674 | ArgData, ArgSize, |
675 | Handler: []() { |
676 | ELFNixPlatformRuntimeState::destroy(); |
677 | return Error::success(); |
678 | }) |
679 | .release(); |
680 | } |
681 | |
682 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
683 | __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) { |
684 | return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle( |
685 | ArgData, ArgSize, |
686 | Handler: [](std::string &JDName, ExecutorAddr HeaderAddr) { |
687 | return ELFNixPlatformRuntimeState::get().registerJITDylib( |
688 | Name&: JDName, Handle: HeaderAddr.toPtr<void *>()); |
689 | }) |
690 | .release(); |
691 | } |
692 | |
693 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
694 | __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) { |
695 | return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( |
696 | ArgData, ArgSize, |
697 | Handler: [](ExecutorAddr HeaderAddr) { |
698 | return ELFNixPlatformRuntimeState::get().deregisterJITDylib( |
699 | Handle: HeaderAddr.toPtr<void *>()); |
700 | }) |
701 | .release(); |
702 | } |
703 | |
704 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
705 | __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) { |
706 | return WrapperFunction<SPSError(SPSExecutorAddr, |
707 | SPSSequence<SPSExecutorAddrRange>)>:: |
708 | handle(ArgData, ArgSize, |
709 | Handler: [](ExecutorAddr HeaderAddr, |
710 | std::vector<ExecutorAddrRange> &Inits) { |
711 | return ELFNixPlatformRuntimeState::get().registerInits( |
712 | HeaderAddr, Inits: std::move(t&: Inits)); |
713 | }) |
714 | .release(); |
715 | } |
716 | |
717 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
718 | __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) { |
719 | return WrapperFunction<SPSError(SPSExecutorAddr, |
720 | SPSSequence<SPSExecutorAddrRange>)>:: |
721 | handle(ArgData, ArgSize, |
722 | Handler: [](ExecutorAddr HeaderAddr, |
723 | std::vector<ExecutorAddrRange> &Inits) { |
724 | return ELFNixPlatformRuntimeState::get().deregisterInits( |
725 | HeaderAddr, Inits: std::move(t&: Inits)); |
726 | }) |
727 | .release(); |
728 | } |
729 | |
730 | /// Wrapper function for registering metadata on a per-object basis. |
731 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
732 | __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) { |
733 | return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: |
734 | handle(ArgData, ArgSize, |
735 | Handler: [](ELFNixPerObjectSectionsToRegister &POSR) { |
736 | return ELFNixPlatformRuntimeState::get().registerObjectSections( |
737 | POSR: std::move(t&: POSR)); |
738 | }) |
739 | .release(); |
740 | } |
741 | |
742 | /// Wrapper for releasing per-object metadat. |
743 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
744 | __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) { |
745 | return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: |
746 | handle(ArgData, ArgSize, |
747 | Handler: [](ELFNixPerObjectSectionsToRegister &POSR) { |
748 | return ELFNixPlatformRuntimeState::get() |
749 | .deregisterObjectSections(POSR: std::move(t&: POSR)); |
750 | }) |
751 | .release(); |
752 | } |
753 | |
754 | //------------------------------------------------------------------------------ |
755 | // TLV support |
756 | //------------------------------------------------------------------------------ |
757 | |
758 | ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) { |
759 | auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>( |
760 | pthread_getspecific(key: D->Key)); |
761 | if (!TLVMgr) |
762 | TLVMgr = new ELFNixPlatformRuntimeTLVManager(); |
763 | if (pthread_setspecific(key: D->Key, pointer: TLVMgr)) { |
764 | __orc_rt_log_error(ErrMsg: "Call to pthread_setspecific failed"); |
765 | return nullptr; |
766 | } |
767 | |
768 | return TLVMgr->getInstance( |
769 | ThreadData: reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); |
770 | } |
771 | |
772 | ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl( |
773 | TLSDescriptor *D, const char *ThreadPointer) { |
774 | const char *TLVPtr = reinterpret_cast<const char *>( |
775 | __orc_rt_elfnix_tls_get_addr_impl(D: D->InfoEntry)); |
776 | return TLVPtr - ThreadPointer; |
777 | } |
778 | |
779 | ORC_RT_INTERFACE orc_rt_WrapperFunctionResult |
780 | __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) { |
781 | return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( |
782 | ArgData, ArgSize, |
783 | Handler: []() -> Expected<uint64_t> { |
784 | pthread_key_t Key; |
785 | if (int Err = pthread_key_create(key: &Key, destr_function: destroyELFNixTLVMgr)) { |
786 | __orc_rt_log_error(ErrMsg: "Call to pthread_key_create failed"); |
787 | return make_error<StringError>(Args: strerror(errnum: Err)); |
788 | } |
789 | return static_cast<uint64_t>(Key); |
790 | }) |
791 | .release(); |
792 | } |
793 | |
794 | //------------------------------------------------------------------------------ |
795 | // cxa_atexit support |
796 | //------------------------------------------------------------------------------ |
797 | |
798 | int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg, |
799 | void *dso_handle) { |
800 | return ELFNixPlatformRuntimeState::get().registerAtExit(F: func, Arg: arg, |
801 | DSOHandle: dso_handle); |
802 | } |
803 | |
804 | int __orc_rt_elfnix_atexit(void (*func)(void *)) { |
805 | auto &PlatformRTState = ELFNixPlatformRuntimeState::get(); |
806 | return ELFNixPlatformRuntimeState::get().registerAtExit( |
807 | F: func, NULL, DSOHandle: PlatformRTState.getPlatformJDDSOHandle()); |
808 | } |
809 | |
810 | void __orc_rt_elfnix_cxa_finalize(void *dso_handle) { |
811 | ELFNixPlatformRuntimeState::get().runAtExits(DSOHandle: dso_handle); |
812 | } |
813 | |
814 | //------------------------------------------------------------------------------ |
815 | // JIT'd dlfcn alternatives. |
816 | //------------------------------------------------------------------------------ |
817 | |
818 | const char *__orc_rt_elfnix_jit_dlerror() { |
819 | return ELFNixPlatformRuntimeState::get().dlerror(); |
820 | } |
821 | |
822 | void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) { |
823 | return ELFNixPlatformRuntimeState::get().dlopen(Path: path, Mode: mode); |
824 | } |
825 | |
826 | int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) { |
827 | return ELFNixPlatformRuntimeState::get().dlupdate(DSOHandle: dso_handle); |
828 | } |
829 | |
830 | int __orc_rt_elfnix_jit_dlclose(void *dso_handle) { |
831 | return ELFNixPlatformRuntimeState::get().dlclose(DSOHandle: dso_handle); |
832 | } |
833 | |
834 | void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) { |
835 | return ELFNixPlatformRuntimeState::get().dlsym(DSOHandle: dso_handle, Symbol: symbol); |
836 | } |
837 | |
838 | //------------------------------------------------------------------------------ |
839 | // ELFNix Run Program |
840 | //------------------------------------------------------------------------------ |
841 | |
842 | ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program( |
843 | const char *JITDylibName, const char *EntrySymbolName, int argc, |
844 | char *argv[]) { |
845 | using MainTy = int (*)(int, char *[]); |
846 | |
847 | void *H = __orc_rt_elfnix_jit_dlopen(path: JITDylibName, |
848 | mode: orc_rt::elfnix::ORC_RT_RTLD_LAZY); |
849 | if (!H) { |
850 | __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror()); |
851 | return -1; |
852 | } |
853 | |
854 | auto *Main = |
855 | reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(dso_handle: H, symbol: EntrySymbolName)); |
856 | |
857 | if (!Main) { |
858 | __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror()); |
859 | return -1; |
860 | } |
861 | |
862 | int Result = Main(argc, argv); |
863 | |
864 | if (__orc_rt_elfnix_jit_dlclose(dso_handle: H) == -1) |
865 | __orc_rt_log_error(ErrMsg: __orc_rt_elfnix_jit_dlerror()); |
866 | |
867 | return Result; |
868 | } |
869 |
Definitions
- __orc_rt_reoptimize_tag
- __orc_rt_elfnix_push_initializers_tag
- __orc_rt_elfnix_symbol_lookup_tag
- TLSInfoEntry
- TLSDescriptor
- ELFNixPlatformRuntimeState
- AtExitEntry
- PerJITDylibState
- referenced
- ELFNixPlatformRuntimeState
- operator=
- ELFNixPlatformRuntimeState
- operator=
- getPlatformJDDSOHandle
- MOPS
- initialize
- get
- destroy
- ELFNixPlatformRuntimeState
- registerObjectSections
- deregisterObjectSections
- registerJITDylib
- deregisterJITDylib
- registerInits
- deregisterInits
- dlerror
- dlopen
- dlupdate
- dlclose
- dlsym
- registerAtExit
- runAtExits
- runAtExits
- getThreadDataSectionFor
- getJITDylibStateByHeaderAddr
- getJITDylibStateByName
- registerThreadDataSection
- lookupSymbolInJITDylib
- runInits
- dlopenImpl
- dlopenFull
- dlopenInitialize
- dlupdateImpl
- dlupdateFull
- dlcloseImpl
- dlcloseInitialize
- ELFNixPlatformRuntimeTLVManager
- getInstance
- destroyELFNixTLVMgr
- __orc_rt_elfnix_platform_bootstrap
- __orc_rt_elfnix_platform_shutdown
- __orc_rt_elfnix_register_jitdylib
- __orc_rt_elfnix_deregister_jitdylib
- __orc_rt_elfnix_register_init_sections
- __orc_rt_elfnix_deregister_init_sections
- __orc_rt_elfnix_register_object_sections
- __orc_rt_elfnix_deregister_object_sections
- __orc_rt_elfnix_tls_get_addr_impl
- ___orc_rt_elfnix_tlsdesc_resolver_impl
- __orc_rt_elfnix_create_pthread_key
- __orc_rt_elfnix_cxa_atexit
- __orc_rt_elfnix_atexit
- __orc_rt_elfnix_cxa_finalize
- __orc_rt_elfnix_jit_dlerror
- __orc_rt_elfnix_jit_dlopen
- __orc_rt_elfnix_jit_dlupdate
- __orc_rt_elfnix_jit_dlclose
- __orc_rt_elfnix_jit_dlsym
Update your C++ knowledge – Modern C++11/14/17 Training
Find out more