1//===- macho_platform.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code required to load the rest of the MachO runtime.
10//
11//===----------------------------------------------------------------------===//
12
13#include "macho_platform.h"
14#include "bitmask_enum.h"
15#include "common.h"
16#include "debug.h"
17#include "error.h"
18#include "interval_map.h"
19#include "wrapper_function_utils.h"
20
21#include <algorithm>
22#include <ios>
23#include <map>
24#include <mutex>
25#include <sstream>
26#include <string_view>
27#include <unordered_map>
28#include <unordered_set>
29#include <vector>
30
31#define DEBUG_TYPE "macho_platform"
32
33using namespace __orc_rt;
34using namespace __orc_rt::macho;
35
36// Declare function tags for functions in the JIT process.
37ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
38ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag)
39
40struct objc_image_info;
41struct mach_header;
42
43// Objective-C registration functions.
44// These are weakly imported. If the Objective-C runtime has not been loaded
45// then code containing Objective-C sections will generate an error.
46extern "C" void
47_objc_map_images(unsigned count, const char *const paths[],
48 const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;
49
50extern "C" void _objc_load_image(const char *path,
51 const mach_header *mh) ORC_RT_WEAK_IMPORT;
52
53// Libunwind prototypes.
54struct unw_dynamic_unwind_sections {
55 uintptr_t dso_base;
56 uintptr_t dwarf_section;
57 size_t dwarf_section_length;
58 uintptr_t compact_unwind_section;
59 size_t compact_unwind_section_length;
60};
61
62typedef int (*unw_find_dynamic_unwind_sections)(
63 uintptr_t addr, struct unw_dynamic_unwind_sections *info);
64
65extern "C" int __unw_add_find_dynamic_unwind_sections(
66 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
67 ORC_RT_WEAK_IMPORT;
68
69extern "C" int __unw_remove_find_dynamic_unwind_sections(
70 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
71 ORC_RT_WEAK_IMPORT;
72
73namespace {
74
75struct MachOJITDylibDepInfo {
76 bool Sealed = false;
77 std::vector<ExecutorAddr> DepHeaders;
78};
79
80using MachOJITDylibDepInfoMap =
81 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
82
83} // anonymous namespace
84
85namespace __orc_rt {
86
87using SPSMachOObjectPlatformSectionsMap =
88 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
89
90using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
91
92using SPSMachOJITDylibDepInfoMap =
93 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
94
95template <>
96class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
97public:
98 static size_t size(const MachOJITDylibDepInfo &JDI) {
99 return SPSMachOJITDylibDepInfo::AsArgList::size(Arg: JDI.Sealed, Args: JDI.DepHeaders);
100 }
101
102 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
103 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, Arg: JDI.Sealed,
104 Args: JDI.DepHeaders);
105 }
106
107 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
108 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, Arg&: JDI.Sealed,
109 Args&: JDI.DepHeaders);
110 }
111};
112
113struct UnwindSectionInfo {
114 std::vector<ExecutorAddrRange> CodeRanges;
115 ExecutorAddrRange DwarfSection;
116 ExecutorAddrRange CompactUnwindSection;
117};
118
119using SPSUnwindSectionInfo =
120 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
121 SPSExecutorAddrRange>;
122
123template <>
124class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
125public:
126 static size_t size(const UnwindSectionInfo &USI) {
127 return SPSUnwindSectionInfo::AsArgList::size(
128 Arg: USI.CodeRanges, Args: USI.DwarfSection, Args: USI.CompactUnwindSection);
129 }
130
131 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
132 return SPSUnwindSectionInfo::AsArgList::serialize(
133 OB, Arg: USI.CodeRanges, Args: USI.DwarfSection, Args: USI.CompactUnwindSection);
134 }
135
136 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
137 return SPSUnwindSectionInfo::AsArgList::deserialize(
138 IB, Arg&: USI.CodeRanges, Args&: USI.DwarfSection, Args&: USI.CompactUnwindSection);
139 }
140};
141
142} // namespace __orc_rt
143
144namespace {
145struct TLVDescriptor {
146 void *(*Thunk)(TLVDescriptor *) = nullptr;
147 unsigned long Key = 0;
148 unsigned long DataAddress = 0;
149};
150
151class MachOPlatformRuntimeState {
152public:
153 // Used internally by MachOPlatformRuntimeState, but made public to enable
154 // serialization.
155 enum class MachOExecutorSymbolFlags : uint8_t {
156 None = 0,
157 Weak = 1U << 0,
158 Callable = 1U << 1,
159 ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)
160 };
161
162private:
163 struct AtExitEntry {
164 void (*Func)(void *);
165 void *Arg;
166 };
167
168 using AtExitsVector = std::vector<AtExitEntry>;
169
170 /// Used to manage sections of fixed-sized metadata records (e.g. pointer
171 /// sections, selector refs, etc.)
172 template <typename RecordElement> class RecordSectionsTracker {
173 public:
174 /// Add a section to the "new" list.
175 void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); }
176
177 /// Returns true if there are new sections to process.
178 bool hasNewSections() const { return !New.empty(); }
179
180 /// Returns the number of new sections to process.
181 size_t numNewSections() const { return New.size(); }
182
183 /// Process all new sections.
184 template <typename ProcessSectionFunc>
185 std::enable_if_t<std::is_void_v<
186 std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>>
187 processNewSections(ProcessSectionFunc &&ProcessSection) {
188 for (auto &Sec : New)
189 ProcessSection(Sec);
190 moveNewToProcessed();
191 }
192
193 /// Proces all new sections with a fallible handler.
194 ///
195 /// Successfully handled sections will be moved to the Processed
196 /// list.
197 template <typename ProcessSectionFunc>
198 std::enable_if_t<
199 std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc,
200 span<RecordElement>>>,
201 Error>
202 processNewSections(ProcessSectionFunc &&ProcessSection) {
203 for (size_t I = 0; I != New.size(); ++I) {
204 if (auto Err = ProcessSection(New[I])) {
205 for (size_t J = 0; J != I; ++J)
206 Processed.push_back(New[J]);
207 New.erase(New.begin(), New.begin() + I);
208 return Err;
209 }
210 }
211 moveNewToProcessed();
212 return Error::success();
213 }
214
215 /// Move all sections back to New for reprocessing.
216 void reset() {
217 moveNewToProcessed();
218 New = std::move(Processed);
219 }
220
221 /// Remove the section with the given range.
222 bool removeIfPresent(ExecutorAddrRange R) {
223 if (removeIfPresent(New, R))
224 return true;
225 return removeIfPresent(Processed, R);
226 }
227
228 private:
229 void moveNewToProcessed() {
230 if (Processed.empty())
231 Processed = std::move(New);
232 else {
233 Processed.reserve(Processed.size() + New.size());
234 std::copy(New.begin(), New.end(), std::back_inserter(Processed));
235 New.clear();
236 }
237 }
238
239 bool removeIfPresent(std::vector<span<RecordElement>> &V,
240 ExecutorAddrRange R) {
241 auto RI = std::find_if(
242 V.rbegin(), V.rend(),
243 [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) {
244 return E.data() == RS.data();
245 });
246 if (RI != V.rend()) {
247 V.erase(std::next(RI).base());
248 return true;
249 }
250 return false;
251 }
252
253 std::vector<span<RecordElement>> Processed;
254 std::vector<span<RecordElement>> New;
255 };
256
257 struct UnwindSections {
258 UnwindSections(const UnwindSectionInfo &USI)
259 : DwarfSection(USI.DwarfSection.toSpan<char>()),
260 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
261
262 span<char> DwarfSection;
263 span<char> CompactUnwindSection;
264 };
265
266 using UnwindSectionsMap =
267 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
268
269 struct JITDylibState {
270
271 using SymbolTableMap =
272 std::unordered_map<std::string_view,
273 std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>;
274
275 std::string Name;
276 void *Header = nullptr;
277 bool Sealed = false;
278 size_t LinkedAgainstRefCount = 0;
279 size_t DlRefCount = 0;
280 SymbolTableMap SymbolTable;
281 std::vector<JITDylibState *> Deps;
282 AtExitsVector AtExits;
283 const objc_image_info *ObjCImageInfo = nullptr;
284 std::unordered_map<void *, std::vector<char>> DataSectionContent;
285 std::unordered_map<void *, size_t> ZeroInitRanges;
286 UnwindSectionsMap UnwindSections;
287 RecordSectionsTracker<void (*)()> ModInitsSections;
288 RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;
289
290 bool referenced() const {
291 return LinkedAgainstRefCount != 0 || DlRefCount != 0;
292 }
293 };
294
295public:
296 static Error create();
297 static MachOPlatformRuntimeState &get();
298 static Error destroy();
299
300 MachOPlatformRuntimeState() = default;
301
302 // Delete copy and move constructors.
303 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
304 MachOPlatformRuntimeState &
305 operator=(const MachOPlatformRuntimeState &) = delete;
306 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
307 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
308
309 Error initialize();
310 Error shutdown();
311
312 Error registerJITDylib(std::string Name, void *Header);
313 Error deregisterJITDylib(void *Header);
314 Error registerThreadDataSection(span<const char> ThreadDataSection);
315 Error deregisterThreadDataSection(span<const char> ThreadDataSection);
316 Error registerObjectSymbolTable(
317 ExecutorAddr HeaderAddr,
318 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
319 MachOExecutorSymbolFlags>> &Entries);
320 Error deregisterObjectSymbolTable(
321 ExecutorAddr HeaderAddr,
322 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
323 MachOExecutorSymbolFlags>> &Entries);
324 Error registerObjectPlatformSections(
325 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
326 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
327 Error deregisterObjectPlatformSections(
328 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
329 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
330
331 const char *dlerror();
332 void *dlopen(std::string_view Name, int Mode);
333 int dlclose(void *DSOHandle);
334 void *dlsym(void *DSOHandle, const char *Symbol);
335
336 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
337 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
338 JITDylibState &JDS);
339 void runAtExits(void *DSOHandle);
340
341 /// Returns the base address of the section containing ThreadData.
342 Expected<std::pair<const char *, size_t>>
343 getThreadDataSectionFor(const char *ThreadData);
344
345private:
346 JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
347 JITDylibState *getJITDylibStateByName(std::string_view Path);
348
349 /// Requests materialization of the given symbols. For each pair, the bool
350 /// element indicates whether the symbol is required (true) or weakly
351 /// referenced (false).
352 Error requestPushSymbols(JITDylibState &JDS,
353 span<std::pair<std::string_view, bool>> Symbols);
354
355 /// Attempts to look up the given symbols locally, requesting a push from the
356 /// remote if they're not found. Results are written to the Result span, which
357 /// must have the same size as the Symbols span.
358 Error
359 lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
360 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
361 span<std::pair<std::string_view, bool>> Symbols);
362
363 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
364
365 static int findDynamicUnwindSections(uintptr_t addr,
366 unw_dynamic_unwind_sections *info);
367 static Error registerEHFrames(span<const char> EHFrameSection);
368 static Error deregisterEHFrames(span<const char> EHFrameSection);
369
370 static Error registerObjCRegistrationObjects(JITDylibState &JDS);
371 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
372 JITDylibState &JDS);
373
374 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
375 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
376 JITDylibState &JDS);
377 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
378 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
379
380 Error dlcloseImpl(void *DSOHandle);
381 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
382 JITDylibState &JDS);
383
384 static MachOPlatformRuntimeState *MOPS;
385
386 bool UseCallbackStyleUnwindInfo = false;
387
388 // FIXME: Move to thread-state.
389 std::string DLFcnError;
390
391 // APIMutex guards against concurrent entry into key "dyld" API functions
392 // (e.g. dlopen, dlclose).
393 std::recursive_mutex DyldAPIMutex;
394
395 // JDStatesMutex guards the data structures that hold JITDylib state.
396 std::mutex JDStatesMutex;
397 std::unordered_map<void *, JITDylibState> JDStates;
398 std::unordered_map<std::string_view, void *> JDNameToHeader;
399
400 // ThreadDataSectionsMutex guards thread local data section state.
401 std::mutex ThreadDataSectionsMutex;
402 std::map<const char *, size_t> ThreadDataSections;
403};
404
405} // anonymous namespace
406
407namespace __orc_rt {
408
409class SPSMachOExecutorSymbolFlags;
410
411template <>
412class SPSSerializationTraits<
413 SPSMachOExecutorSymbolFlags,
414 MachOPlatformRuntimeState::MachOExecutorSymbolFlags> {
415private:
416 using UT = std::underlying_type_t<
417 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>;
418
419public:
420 static size_t
421 size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
422 return sizeof(UT);
423 }
424
425 static bool
426 serialize(SPSOutputBuffer &OB,
427 const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
428 return SPSArgList<UT>::serialize(OB, Arg: static_cast<UT>(SF));
429 }
430
431 static bool
432 deserialize(SPSInputBuffer &IB,
433 MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
434 UT Tmp;
435 if (!SPSArgList<UT>::deserialize(IB, Arg&: Tmp))
436 return false;
437 SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp);
438 return true;
439 }
440};
441
442} // namespace __orc_rt
443
444namespace {
445
446MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
447
448Error MachOPlatformRuntimeState::create() {
449 assert(!MOPS && "MachOPlatformRuntimeState should be null");
450 MOPS = new MachOPlatformRuntimeState();
451 return MOPS->initialize();
452}
453
454MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
455 assert(MOPS && "MachOPlatformRuntimeState not initialized");
456 return *MOPS;
457}
458
459Error MachOPlatformRuntimeState::destroy() {
460 assert(MOPS && "MachOPlatformRuntimeState not initialized");
461 auto Err = MOPS->shutdown();
462 delete MOPS;
463 return Err;
464}
465
466Error MachOPlatformRuntimeState::initialize() {
467 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
468 __unw_remove_find_dynamic_unwind_sections;
469 if (UseCallbackStyleUnwindInfo) {
470 ORC_RT_DEBUG({
471 printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
472 " Using callback-based frame info lookup.\n");
473 });
474 if (__unw_add_find_dynamic_unwind_sections(find_dynamic_unwind_sections: &findDynamicUnwindSections))
475 return make_error<StringError>(
476 Args: "Could not register findDynamicUnwindSections");
477 } else {
478 ORC_RT_DEBUG({
479 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
480 " Using classic frame info registration.\n");
481 });
482 }
483 return Error::success();
484}
485
486Error MachOPlatformRuntimeState::shutdown() {
487 if (UseCallbackStyleUnwindInfo) {
488 if (__unw_remove_find_dynamic_unwind_sections(find_dynamic_unwind_sections: &findDynamicUnwindSections)) {
489 ORC_RT_DEBUG(
490 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
491 }
492 }
493 return Error::success();
494}
495
496Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
497 void *Header) {
498 ORC_RT_DEBUG({
499 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
500 });
501 std::lock_guard<std::mutex> Lock(JDStatesMutex);
502 if (JDStates.count(x: Header)) {
503 std::ostringstream ErrStream;
504 ErrStream << "Duplicate JITDylib registration for header " << Header
505 << " (name = " << Name << ")";
506 return make_error<StringError>(Args: ErrStream.str());
507 }
508 if (JDNameToHeader.count(x: Name)) {
509 std::ostringstream ErrStream;
510 ErrStream << "Duplicate JITDylib registration for header " << Header
511 << " (header = " << Header << ")";
512 return make_error<StringError>(Args: ErrStream.str());
513 }
514
515 auto &JDS = JDStates[Header];
516 JDS.Name = std::move(t&: Name);
517 JDS.Header = Header;
518 JDNameToHeader[JDS.Name] = Header;
519 return Error::success();
520}
521
522Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
523 std::lock_guard<std::mutex> Lock(JDStatesMutex);
524 auto I = JDStates.find(x: Header);
525 if (I == JDStates.end()) {
526 std::ostringstream ErrStream;
527 ErrStream << "Attempted to deregister unrecognized header " << Header;
528 return make_error<StringError>(Args: ErrStream.str());
529 }
530
531 // Remove std::string construction once we can use C++20.
532 auto J = JDNameToHeader.find(
533 x: std::string(I->second.Name.data(), I->second.Name.size()));
534 assert(J != JDNameToHeader.end() &&
535 "Missing JDNameToHeader entry for JITDylib");
536
537 ORC_RT_DEBUG({
538 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
539 Header);
540 });
541
542 JDNameToHeader.erase(position: J);
543 JDStates.erase(position: I);
544 return Error::success();
545}
546
547Error MachOPlatformRuntimeState::registerThreadDataSection(
548 span<const char> ThreadDataSection) {
549 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
550 auto I = ThreadDataSections.upper_bound(x: ThreadDataSection.data());
551 if (I != ThreadDataSections.begin()) {
552 auto J = std::prev(x: I);
553 if (J->first + J->second > ThreadDataSection.data())
554 return make_error<StringError>(Args: "Overlapping __thread_data sections");
555 }
556 ThreadDataSections.insert(
557 position: I, x: std::make_pair(x: ThreadDataSection.data(), y: ThreadDataSection.size()));
558 return Error::success();
559}
560
561Error MachOPlatformRuntimeState::deregisterThreadDataSection(
562 span<const char> ThreadDataSection) {
563 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
564 auto I = ThreadDataSections.find(x: ThreadDataSection.data());
565 if (I == ThreadDataSections.end())
566 return make_error<StringError>(Args: "Attempt to deregister unknown thread data "
567 "section");
568 ThreadDataSections.erase(position: I);
569 return Error::success();
570}
571
572Error MachOPlatformRuntimeState::registerObjectSymbolTable(
573 ExecutorAddr HeaderAddr,
574 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
575 MachOExecutorSymbolFlags>> &Entries) {
576
577 std::lock_guard<std::mutex> Lock(JDStatesMutex);
578 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
579 if (!JDS) {
580 std::ostringstream ErrStream;
581 ErrStream << "Could not register object platform sections for "
582 "unrecognized header "
583 << HeaderAddr.toPtr<void *>();
584 return make_error<StringError>(Args: ErrStream.str());
585 }
586
587 for (auto &[NameAddr, SymAddr, Flags] : Entries)
588 JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags};
589
590 return Error::success();
591}
592
593Error MachOPlatformRuntimeState::deregisterObjectSymbolTable(
594 ExecutorAddr HeaderAddr,
595 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
596 MachOExecutorSymbolFlags>> &Entries) {
597
598 std::lock_guard<std::mutex> Lock(JDStatesMutex);
599 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
600 if (!JDS) {
601 std::ostringstream ErrStream;
602 ErrStream << "Could not register object platform sections for "
603 "unrecognized header "
604 << HeaderAddr.toPtr<void *>();
605 return make_error<StringError>(Args: ErrStream.str());
606 }
607
608 for (auto &[NameAddr, SymAddr, Flags] : Entries)
609 JDS->SymbolTable.erase(x: NameAddr.toPtr<const char *>());
610
611 return Error::success();
612}
613
614Error MachOPlatformRuntimeState::registerObjectPlatformSections(
615 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
616 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
617
618 // FIXME: Reject platform section registration after the JITDylib is
619 // sealed?
620
621 ORC_RT_DEBUG({
622 printdbg("MachOPlatform: Registering object sections for %p.\n",
623 HeaderAddr.toPtr<void *>());
624 });
625
626 std::lock_guard<std::mutex> Lock(JDStatesMutex);
627 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
628 if (!JDS) {
629 std::ostringstream ErrStream;
630 ErrStream << "Could not register object platform sections for "
631 "unrecognized header "
632 << HeaderAddr.toPtr<void *>();
633 return make_error<StringError>(Args: ErrStream.str());
634 }
635
636 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
637 ORC_RT_DEBUG({
638 printdbg(" Registering new-style unwind info for:\n"
639 " DWARF: %p -- %p\n"
640 " Compact-unwind: %p -- %p\n"
641 " for:\n",
642 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
643 UnwindInfo->DwarfSection.End.toPtr<void *>(),
644 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
645 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
646 });
647 for (auto &CodeRange : UnwindInfo->CodeRanges) {
648 JDS->UnwindSections.insert(KS: CodeRange.Start.toPtr<char *>(),
649 KE: CodeRange.End.toPtr<char *>(), V: *UnwindInfo);
650 ORC_RT_DEBUG({
651 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
652 CodeRange.End.toPtr<void *>());
653 });
654 }
655 }
656
657 for (auto &KV : Secs) {
658 // FIXME: Validate section ranges?
659 if (KV.first == "__TEXT,__eh_frame") {
660 if (!UseCallbackStyleUnwindInfo) {
661 // Use classic libunwind registration.
662 if (auto Err = registerEHFrames(EHFrameSection: KV.second.toSpan<const char>()))
663 return Err;
664 }
665 } else if (KV.first == "__DATA,__data") {
666 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
667 "Address already registered.");
668 auto S = KV.second.toSpan<char>();
669 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
670 std::vector<char>(S.begin(), S.end());
671 } else if (KV.first == "__DATA,__common") {
672 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
673 } else if (KV.first == "__DATA,__thread_data") {
674 if (auto Err = registerThreadDataSection(ThreadDataSection: KV.second.toSpan<const char>()))
675 return Err;
676 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
677 JDS->ObjCRuntimeRegistrationObjects.add(Sec: KV.second.toSpan<char>());
678 else if (KV.first == "__DATA,__mod_init_func")
679 JDS->ModInitsSections.add(Sec: KV.second.toSpan<void (*)()>());
680 else {
681 // Should this be a warning instead?
682 return make_error<StringError>(
683 Args: "Encountered unexpected section " +
684 std::string(KV.first.data(), KV.first.size()) +
685 " while registering object platform sections");
686 }
687 }
688
689 return Error::success();
690}
691
692Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
693 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
694 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
695 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
696 // TODO: Add a JITDylib prepare-for-teardown operation that clears all
697 // registered sections, causing this function to take the fast-path.
698 ORC_RT_DEBUG({
699 printdbg("MachOPlatform: Deregistering object sections for %p.\n",
700 HeaderAddr.toPtr<void *>());
701 });
702
703 std::lock_guard<std::mutex> Lock(JDStatesMutex);
704 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
705 if (!JDS) {
706 std::ostringstream ErrStream;
707 ErrStream << "Could not register object platform sections for unrecognized "
708 "header "
709 << HeaderAddr.toPtr<void *>();
710 return make_error<StringError>(Args: ErrStream.str());
711 }
712
713 // FIXME: Implement faster-path by returning immediately if JDS is being
714 // torn down entirely?
715
716 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
717 // any Swift or ObjC. Once this happens we can clear (and no longer record)
718 // data section content, as the library could never be re-initialized.
719
720 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
721 ORC_RT_DEBUG({
722 printdbg(" Deregistering new-style unwind info for:\n"
723 " DWARF: %p -- %p\n"
724 " Compact-unwind: %p -- %p\n"
725 " for:\n",
726 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
727 UnwindInfo->DwarfSection.End.toPtr<void *>(),
728 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
729 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
730 });
731 for (auto &CodeRange : UnwindInfo->CodeRanges) {
732 JDS->UnwindSections.erase(KS: CodeRange.Start.toPtr<char *>(),
733 KE: CodeRange.End.toPtr<char *>());
734 ORC_RT_DEBUG({
735 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
736 CodeRange.End.toPtr<void *>());
737 });
738 }
739 }
740
741 for (auto &KV : Secs) {
742 // FIXME: Validate section ranges?
743 if (KV.first == "__TEXT,__eh_frame") {
744 if (!UseCallbackStyleUnwindInfo) {
745 // Use classic libunwind registration.
746 if (auto Err = deregisterEHFrames(EHFrameSection: KV.second.toSpan<const char>()))
747 return Err;
748 }
749 } else if (KV.first == "__DATA,__data") {
750 JDS->DataSectionContent.erase(x: KV.second.Start.toPtr<char *>());
751 } else if (KV.first == "__DATA,__common") {
752 JDS->ZeroInitRanges.erase(x: KV.second.Start.toPtr<char *>());
753 } else if (KV.first == "__DATA,__thread_data") {
754 if (auto Err =
755 deregisterThreadDataSection(ThreadDataSection: KV.second.toSpan<const char>()))
756 return Err;
757 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
758 JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(R: KV.second);
759 else if (KV.first == "__DATA,__mod_init_func")
760 JDS->ModInitsSections.removeIfPresent(R: KV.second);
761 else {
762 // Should this be a warning instead?
763 return make_error<StringError>(
764 Args: "Encountered unexpected section " +
765 std::string(KV.first.data(), KV.first.size()) +
766 " while deregistering object platform sections");
767 }
768 }
769 return Error::success();
770}
771
772const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
773
774void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
775 ORC_RT_DEBUG({
776 std::string S(Path.data(), Path.size());
777 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
778 });
779 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
780 if (auto H = dlopenImpl(Path, Mode))
781 return *H;
782 else {
783 // FIXME: Make dlerror thread safe.
784 DLFcnError = toString(Err: H.takeError());
785 return nullptr;
786 }
787}
788
789int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
790 ORC_RT_DEBUG({
791 auto *JDS = getJITDylibStateByHeader(DSOHandle);
792 std::string DylibName;
793 if (JDS) {
794 std::string S;
795 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
796 } else
797 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
798 "invalid handle");
799 });
800 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
801 if (auto Err = dlcloseImpl(DSOHandle)) {
802 // FIXME: Make dlerror thread safe.
803 DLFcnError = toString(Err: std::move(t&: Err));
804 return -1;
805 }
806 return 0;
807}
808
809void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) {
810 std::unique_lock<std::mutex> Lock(JDStatesMutex);
811 auto *JDS = getJITDylibStateByHeader(DSOHandle);
812 if (!JDS) {
813 std::ostringstream ErrStream;
814 ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle;
815 DLFcnError = ErrStream.str();
816 return nullptr;
817 }
818
819 std::string MangledName = std::string("_") + Symbol;
820 std::pair<std::string_view, bool> Lookup(MangledName, false);
821 std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result;
822
823 if (auto Err = lookupSymbols(JDS&: *JDS, JDStatesLock&: Lock, Result: {&Result, 1}, Symbols: {&Lookup, 1})) {
824 DLFcnError = toString(Err: std::move(t&: Err));
825 return nullptr;
826 }
827
828 // Sign callable symbols as functions, to match dyld.
829 if ((Result.second & MachOExecutorSymbolFlags::Callable) ==
830 MachOExecutorSymbolFlags::Callable)
831 return reinterpret_cast<void *>(Result.first.toPtr<void(void)>());
832 return Result.first.toPtr<void *>();
833}
834
835int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
836 void *DSOHandle) {
837 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
838 std::lock_guard<std::mutex> Lock(JDStatesMutex);
839 auto *JDS = getJITDylibStateByHeader(DSOHandle);
840 if (!JDS) {
841 ORC_RT_DEBUG({
842 printdbg("MachOPlatformRuntimeState::registerAtExit called with "
843 "unrecognized dso handle %p\n",
844 DSOHandle);
845 });
846 return -1;
847 }
848 JDS->AtExits.push_back(x: {.Func: F, .Arg: Arg});
849 return 0;
850}
851
852void MachOPlatformRuntimeState::runAtExits(
853 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
854 auto AtExits = std::move(t&: JDS.AtExits);
855
856 // Unlock while running atexits, as they may trigger operations that modify
857 // JDStates.
858 JDStatesLock.unlock();
859 while (!AtExits.empty()) {
860 auto &AE = AtExits.back();
861 AE.Func(AE.Arg);
862 AtExits.pop_back();
863 }
864 JDStatesLock.lock();
865}
866
867void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
868 std::unique_lock<std::mutex> Lock(JDStatesMutex);
869 auto *JDS = getJITDylibStateByHeader(DSOHandle);
870 ORC_RT_DEBUG({
871 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
872 "dso_handle %p\n",
873 DSOHandle);
874 });
875 if (JDS)
876 runAtExits(JDStatesLock&: Lock, JDS&: *JDS);
877}
878
879Expected<std::pair<const char *, size_t>>
880MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
881 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
882 auto I = ThreadDataSections.upper_bound(x: ThreadData);
883 // Check that we have a valid entry covering this address.
884 if (I == ThreadDataSections.begin())
885 return make_error<StringError>(Args: "No thread local data section for key");
886 I = std::prev(x: I);
887 if (ThreadData >= I->first + I->second)
888 return make_error<StringError>(Args: "No thread local data section for key");
889 return *I;
890}
891
892MachOPlatformRuntimeState::JITDylibState *
893MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
894 auto I = JDStates.find(x: DSOHandle);
895 if (I == JDStates.end()) {
896 I = JDStates.insert(x: std::make_pair(x&: DSOHandle, y: JITDylibState())).first;
897 I->second.Header = DSOHandle;
898 }
899 return &I->second;
900}
901
902MachOPlatformRuntimeState::JITDylibState *
903MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
904 // FIXME: Avoid creating string once we have C++20.
905 auto I = JDNameToHeader.find(x: std::string(Name.data(), Name.size()));
906 if (I != JDNameToHeader.end())
907 return getJITDylibStateByHeader(DSOHandle: I->second);
908 return nullptr;
909}
910
911Error MachOPlatformRuntimeState::requestPushSymbols(
912 JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) {
913 Error OpErr = Error::success();
914 if (auto Err = WrapperFunction<SPSError(
915 SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>::
916 call(FnTag: &__orc_rt_macho_push_symbols_tag, Result&: OpErr,
917 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header), Args: Symbols)) {
918 cantFail(Err: std::move(t&: OpErr));
919 return std::move(t&: Err);
920 }
921 return OpErr;
922}
923
924Error MachOPlatformRuntimeState::lookupSymbols(
925 JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
926 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
927 span<std::pair<std::string_view, bool>> Symbols) {
928 assert(JDStatesLock.owns_lock() &&
929 "JDStatesLock should be locked at call-site");
930 assert(Result.size() == Symbols.size() &&
931 "Results and Symbols span sizes should match");
932
933 // Make an initial pass over the local symbol table.
934 std::vector<size_t> MissingSymbolIndexes;
935 for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) {
936 auto I = JDS.SymbolTable.find(x: Symbols[Idx].first);
937 if (I != JDS.SymbolTable.end())
938 Result[Idx] = I->second;
939 else
940 MissingSymbolIndexes.push_back(x: Idx);
941 }
942
943 // If everything has been resolved already then bail out early.
944 if (MissingSymbolIndexes.empty())
945 return Error::success();
946
947 // Otherwise call back to the controller to try to request that the symbol
948 // be materialized.
949 std::vector<std::pair<std::string_view, bool>> MissingSymbols;
950 MissingSymbols.reserve(n: MissingSymbolIndexes.size());
951 ORC_RT_DEBUG({
952 printdbg("requesting push of %i missing symbols...\n",
953 MissingSymbolIndexes.size());
954 });
955 for (auto MissingIdx : MissingSymbolIndexes)
956 MissingSymbols.push_back(x: Symbols[MissingIdx]);
957
958 JDStatesLock.unlock();
959 if (auto Err = requestPushSymbols(
960 JDS, Symbols: {MissingSymbols.data(), MissingSymbols.size()}))
961 return Err;
962 JDStatesLock.lock();
963
964 // Try to resolve the previously missing symbols locally.
965 std::vector<size_t> MissingRequiredSymbols;
966 for (auto MissingIdx : MissingSymbolIndexes) {
967 auto I = JDS.SymbolTable.find(x: Symbols[MissingIdx].first);
968 if (I != JDS.SymbolTable.end())
969 Result[MissingIdx] = I->second;
970 else {
971 if (Symbols[MissingIdx].second)
972 MissingRequiredSymbols.push_back(x: MissingIdx);
973 else
974 Result[MissingIdx] = {ExecutorAddr(), {}};
975 }
976 }
977
978 // Error out if any missing symbols could not be resolved.
979 if (!MissingRequiredSymbols.empty()) {
980 std::ostringstream ErrStream;
981 ErrStream << "Lookup could not find required symbols: [ ";
982 for (auto MissingIdx : MissingRequiredSymbols)
983 ErrStream << "\"" << Symbols[MissingIdx].first << "\" ";
984 ErrStream << "]";
985 return make_error<StringError>(Args: ErrStream.str());
986 }
987
988 return Error::success();
989}
990
991// eh-frame registration functions.
992// We expect these to be available for all processes.
993extern "C" void __register_frame(const void *);
994extern "C" void __deregister_frame(const void *);
995
996template <typename HandleFDEFn>
997void walkEHFrameSection(span<const char> EHFrameSection,
998 HandleFDEFn HandleFDE) {
999 const char *CurCFIRecord = EHFrameSection.data();
1000 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
1001
1002 while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
1003 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
1004 if (Size == 0xffffffff)
1005 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
1006 else
1007 Size += 4;
1008 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
1009
1010 if (Offset != 0)
1011 HandleFDE(CurCFIRecord);
1012
1013 CurCFIRecord += Size;
1014 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
1015 }
1016}
1017
1018bool MachOPlatformRuntimeState::lookupUnwindSections(
1019 void *Addr, unw_dynamic_unwind_sections &Info) {
1020 ORC_RT_DEBUG(
1021 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
1022 std::lock_guard<std::mutex> Lock(JDStatesMutex);
1023 for (auto &KV : JDStates) {
1024 auto &JD = KV.second;
1025 auto I = JD.UnwindSections.find(K: reinterpret_cast<char *>(Addr));
1026 if (I != JD.UnwindSections.end()) {
1027 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
1028 Info.dwarf_section =
1029 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
1030 Info.dwarf_section_length = I->second.DwarfSection.size();
1031 Info.compact_unwind_section =
1032 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
1033 Info.compact_unwind_section_length =
1034 I->second.CompactUnwindSection.size();
1035 return true;
1036 }
1037 }
1038 return false;
1039}
1040
1041int MachOPlatformRuntimeState::findDynamicUnwindSections(
1042 uintptr_t addr, unw_dynamic_unwind_sections *info) {
1043 if (!info)
1044 return 0;
1045 return MachOPlatformRuntimeState::get().lookupUnwindSections(Addr: (void *)addr,
1046 Info&: *info);
1047}
1048
1049Error MachOPlatformRuntimeState::registerEHFrames(
1050 span<const char> EHFrameSection) {
1051 walkEHFrameSection(EHFrameSection, HandleFDE: __register_frame);
1052 return Error::success();
1053}
1054
1055Error MachOPlatformRuntimeState::deregisterEHFrames(
1056 span<const char> EHFrameSection) {
1057 walkEHFrameSection(EHFrameSection, HandleFDE: __deregister_frame);
1058 return Error::success();
1059}
1060
1061Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(
1062 JITDylibState &JDS) {
1063 ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));
1064
1065 std::vector<char *> RegObjBases;
1066 JDS.ObjCRuntimeRegistrationObjects.processNewSections(
1067 ProcessSection: [&](span<char> RegObj) { RegObjBases.push_back(x: RegObj.data()); });
1068
1069 if (RegObjBases.empty())
1070 return Error::success();
1071
1072 if (!_objc_map_images || !_objc_load_image)
1073 return make_error<StringError>(
1074 Args: "Could not register Objective-C / Swift metadata: _objc_map_images / "
1075 "_objc_load_image not found");
1076
1077 std::vector<char *> Paths;
1078 Paths.resize(new_size: RegObjBases.size());
1079 _objc_map_images(count: RegObjBases.size(), paths: Paths.data(),
1080 mhdrs: reinterpret_cast<mach_header **>(RegObjBases.data()));
1081
1082 for (void *RegObjBase : RegObjBases)
1083 _objc_load_image(path: nullptr, mh: reinterpret_cast<mach_header *>(RegObjBase));
1084
1085 return Error::success();
1086}
1087
1088Error MachOPlatformRuntimeState::runModInits(
1089 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1090 std::vector<span<void (*)()>> InitSections;
1091 InitSections.reserve(n: JDS.ModInitsSections.numNewSections());
1092
1093 // Copy initializer sections: If the JITDylib is unsealed then the
1094 // initializers could reach back into the JIT and cause more initializers to
1095 // be added.
1096 // FIXME: Skip unlock and run in-place on sealed JITDylibs?
1097 JDS.ModInitsSections.processNewSections(
1098 ProcessSection: [&](span<void (*)()> Inits) { InitSections.push_back(x: Inits); });
1099
1100 JDStatesLock.unlock();
1101 for (auto InitSec : InitSections)
1102 for (auto *Init : InitSec)
1103 Init();
1104 JDStatesLock.lock();
1105
1106 return Error::success();
1107}
1108
1109Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
1110 int Mode) {
1111 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1112
1113 // Try to find JITDylib state by name.
1114 auto *JDS = getJITDylibStateByName(Name: Path);
1115
1116 if (!JDS)
1117 return make_error<StringError>(Args: "No registered JTIDylib for path " +
1118 std::string(Path.data(), Path.size()));
1119
1120 // If this JITDylib is unsealed, or this is the first dlopen then run
1121 // full dlopen path (update deps, push and run initializers, update ref
1122 // counts on all JITDylibs in the dep tree).
1123 if (!JDS->referenced() || !JDS->Sealed) {
1124 if (auto Err = dlopenFull(JDStatesLock&: Lock, JDS&: *JDS))
1125 return std::move(t&: Err);
1126 }
1127
1128 // Bump the ref-count on this dylib.
1129 ++JDS->DlRefCount;
1130
1131 // Return the header address.
1132 return JDS->Header;
1133}
1134
1135Error MachOPlatformRuntimeState::dlopenFull(
1136 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1137 // Call back to the JIT to push the initializers.
1138 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1139 // Unlock so that we can accept the initializer update.
1140 JDStatesLock.unlock();
1141 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1142 SPSExecutorAddr)>::call(FnTag: &__orc_rt_macho_push_initializers_tag,
1143 Result&: DepInfo, Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
1144 return Err;
1145 JDStatesLock.lock();
1146
1147 if (!DepInfo)
1148 return DepInfo.takeError();
1149
1150 if (auto Err = dlopenInitialize(JDStatesLock, JDS, DepInfo&: *DepInfo))
1151 return Err;
1152
1153 if (!DepInfo->empty()) {
1154 ORC_RT_DEBUG({
1155 printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
1156 JDS.Name.c_str());
1157 });
1158 std::ostringstream ErrStream;
1159 ErrStream << "Encountered unrecognized dep-info key headers "
1160 "while processing dlopen of "
1161 << JDS.Name;
1162 return make_error<StringError>(Args: ErrStream.str());
1163 }
1164
1165 return Error::success();
1166}
1167
1168Error MachOPlatformRuntimeState::dlopenInitialize(
1169 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
1170 MachOJITDylibDepInfoMap &DepInfo) {
1171 ORC_RT_DEBUG({
1172 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
1173 JDS.Name.c_str());
1174 });
1175
1176 // If the header is not present in the dep map then assume that we
1177 // already processed it earlier in the dlopenInitialize traversal and
1178 // return.
1179 // TODO: Keep a visited set instead so that we can error out on missing
1180 // entries?
1181 auto I = DepInfo.find(x: ExecutorAddr::fromPtr(Ptr: JDS.Header));
1182 if (I == DepInfo.end())
1183 return Error::success();
1184
1185 auto DI = std::move(t&: I->second);
1186 DepInfo.erase(position: I);
1187
1188 // We don't need to re-initialize sealed JITDylibs that have already been
1189 // initialized. Just check that their dep-map entry is empty as expected.
1190 if (JDS.Sealed) {
1191 if (!DI.DepHeaders.empty()) {
1192 std::ostringstream ErrStream;
1193 ErrStream << "Sealed JITDylib " << JDS.Header
1194 << " already has registered dependencies";
1195 return make_error<StringError>(Args: ErrStream.str());
1196 }
1197 if (JDS.referenced())
1198 return Error::success();
1199 } else
1200 JDS.Sealed = DI.Sealed;
1201
1202 // This is an unsealed or newly sealed JITDylib. Run initializers.
1203 std::vector<JITDylibState *> OldDeps;
1204 std::swap(x&: JDS.Deps, y&: OldDeps);
1205 JDS.Deps.reserve(n: DI.DepHeaders.size());
1206 for (auto DepHeaderAddr : DI.DepHeaders) {
1207 auto *DepJDS = getJITDylibStateByHeader(DSOHandle: DepHeaderAddr.toPtr<void *>());
1208 if (!DepJDS) {
1209 std::ostringstream ErrStream;
1210 ErrStream << "Encountered unrecognized dep header "
1211 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1212 << JDS.Name;
1213 return make_error<StringError>(Args: ErrStream.str());
1214 }
1215 ++DepJDS->LinkedAgainstRefCount;
1216 if (auto Err = dlopenInitialize(JDStatesLock, JDS&: *DepJDS, DepInfo))
1217 return Err;
1218 }
1219
1220 // Initialize this JITDylib.
1221 if (auto Err = registerObjCRegistrationObjects(JDS))
1222 return Err;
1223 if (auto Err = runModInits(JDStatesLock, JDS))
1224 return Err;
1225
1226 // Decrement old deps.
1227 // FIXME: We should probably continue and just report deinitialize errors
1228 // here.
1229 for (auto *DepJDS : OldDeps) {
1230 --DepJDS->LinkedAgainstRefCount;
1231 if (!DepJDS->referenced())
1232 if (auto Err = dlcloseDeinitialize(JDStatesLock, JDS&: *DepJDS))
1233 return Err;
1234 }
1235
1236 return Error::success();
1237}
1238
1239Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1240 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1241
1242 // Try to find JITDylib state by header.
1243 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1244
1245 if (!JDS) {
1246 std::ostringstream ErrStream;
1247 ErrStream << "No registered JITDylib for " << DSOHandle;
1248 return make_error<StringError>(Args: ErrStream.str());
1249 }
1250
1251 // Bump the ref-count.
1252 --JDS->DlRefCount;
1253
1254 if (!JDS->referenced())
1255 return dlcloseDeinitialize(JDStatesLock&: Lock, JDS&: *JDS);
1256
1257 return Error::success();
1258}
1259
1260Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1261 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1262
1263 ORC_RT_DEBUG({
1264 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1265 JDS.Name.c_str());
1266 });
1267
1268 runAtExits(JDStatesLock, JDS);
1269
1270 // Reset mod-inits
1271 JDS.ModInitsSections.reset();
1272
1273 // Reset data section contents.
1274 for (auto &KV : JDS.DataSectionContent)
1275 memcpy(dest: KV.first, src: KV.second.data(), n: KV.second.size());
1276 for (auto &KV : JDS.ZeroInitRanges)
1277 memset(s: KV.first, c: 0, n: KV.second);
1278
1279 // Deinitialize any dependencies.
1280 for (auto *DepJDS : JDS.Deps) {
1281 --DepJDS->LinkedAgainstRefCount;
1282 if (!DepJDS->referenced())
1283 if (auto Err = dlcloseDeinitialize(JDStatesLock, JDS&: *DepJDS))
1284 return Err;
1285 }
1286
1287 return Error::success();
1288}
1289
1290class MachOPlatformRuntimeTLVManager {
1291public:
1292 void *getInstance(const char *ThreadData);
1293
1294private:
1295 std::unordered_map<const char *, char *> Instances;
1296 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1297};
1298
1299void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1300 auto I = Instances.find(x: ThreadData);
1301 if (I != Instances.end())
1302 return I->second;
1303
1304 auto TDS =
1305 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1306 if (!TDS) {
1307 __orc_rt_log_error(ErrMsg: toString(Err: TDS.takeError()).c_str());
1308 return nullptr;
1309 }
1310
1311 auto &Allocated = AllocatedSections[TDS->first];
1312 if (!Allocated) {
1313 Allocated = std::make_unique<char[]>(num: TDS->second);
1314 memcpy(dest: Allocated.get(), src: TDS->first, n: TDS->second);
1315 }
1316
1317 size_t ThreadDataDelta = ThreadData - TDS->first;
1318 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1319
1320 char *Instance = Allocated.get() + ThreadDataDelta;
1321 Instances[ThreadData] = Instance;
1322 return Instance;
1323}
1324
1325void destroyMachOTLVMgr(void *MachOTLVMgr) {
1326 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1327}
1328
1329Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1330 for (auto &WFC : WFCs)
1331 if (auto Err = WFC.runWithSPSRet<void>())
1332 return Err;
1333 return Error::success();
1334}
1335
1336} // end anonymous namespace
1337
1338//------------------------------------------------------------------------------
1339// JIT entry points
1340//------------------------------------------------------------------------------
1341
1342ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1343__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1344 return WrapperFunction<SPSError()>::handle(
1345 ArgData, ArgSize,
1346 Handler: []() { return MachOPlatformRuntimeState::create(); })
1347 .release();
1348}
1349
1350ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1351__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1352 return WrapperFunction<SPSError()>::handle(
1353 ArgData, ArgSize,
1354 Handler: []() { return MachOPlatformRuntimeState::destroy(); })
1355 .release();
1356}
1357
1358ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1359__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1360 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1361 ArgData, ArgSize,
1362 Handler: [](std::string &Name, ExecutorAddr HeaderAddr) {
1363 return MachOPlatformRuntimeState::get().registerJITDylib(
1364 Name: std::move(t&: Name), Header: HeaderAddr.toPtr<void *>());
1365 })
1366 .release();
1367}
1368
1369ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1370__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1371 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1372 ArgData, ArgSize,
1373 Handler: [](ExecutorAddr HeaderAddr) {
1374 return MachOPlatformRuntimeState::get().deregisterJITDylib(
1375 Header: HeaderAddr.toPtr<void *>());
1376 })
1377 .release();
1378}
1379
1380ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1381__orc_rt_macho_register_object_platform_sections(char *ArgData,
1382 size_t ArgSize) {
1383 return WrapperFunction<SPSError(SPSExecutorAddr,
1384 SPSOptional<SPSUnwindSectionInfo>,
1385 SPSMachOObjectPlatformSectionsMap)>::
1386 handle(ArgData, ArgSize,
1387 Handler: [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1388 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1389 &Secs) {
1390 return MachOPlatformRuntimeState::get()
1391 .registerObjectPlatformSections(HeaderAddr, UnwindInfo: std::move(t&: USI),
1392 Secs: std::move(t&: Secs));
1393 })
1394 .release();
1395}
1396
1397ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1398__orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
1399 using SymtabContainer = std::vector<
1400 std::tuple<ExecutorAddr, ExecutorAddr,
1401 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1402 return WrapperFunction<SPSError(
1403 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1404 SPSMachOExecutorSymbolFlags>>)>::
1405 handle(ArgData, ArgSize,
1406 Handler: [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1407 return MachOPlatformRuntimeState::get()
1408 .registerObjectSymbolTable(HeaderAddr, Entries: Symbols);
1409 })
1410 .release();
1411}
1412
1413ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1414__orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
1415 using SymtabContainer = std::vector<
1416 std::tuple<ExecutorAddr, ExecutorAddr,
1417 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1418 return WrapperFunction<SPSError(
1419 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1420 SPSMachOExecutorSymbolFlags>>)>::
1421 handle(ArgData, ArgSize,
1422 Handler: [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1423 return MachOPlatformRuntimeState::get()
1424 .deregisterObjectSymbolTable(HeaderAddr, Entries: Symbols);
1425 })
1426 .release();
1427}
1428
1429ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1430__orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1431 size_t ArgSize) {
1432 return WrapperFunction<SPSError(SPSExecutorAddr,
1433 SPSOptional<SPSUnwindSectionInfo>,
1434 SPSMachOObjectPlatformSectionsMap)>::
1435 handle(ArgData, ArgSize,
1436 Handler: [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1437 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1438 &Secs) {
1439 return MachOPlatformRuntimeState::get()
1440 .deregisterObjectPlatformSections(HeaderAddr, UnwindInfo: std::move(t&: USI),
1441 Secs: std::move(t&: Secs));
1442 })
1443 .release();
1444}
1445
1446ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1447__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1448 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1449 ArgData, ArgSize, Handler&: runWrapperFunctionCalls)
1450 .release();
1451}
1452
1453//------------------------------------------------------------------------------
1454// TLV support
1455//------------------------------------------------------------------------------
1456
1457ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1458 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1459 pthread_getspecific(key: D->Key));
1460 if (!TLVMgr) {
1461 TLVMgr = new MachOPlatformRuntimeTLVManager();
1462 if (pthread_setspecific(key: D->Key, pointer: TLVMgr)) {
1463 __orc_rt_log_error(ErrMsg: "Call to pthread_setspecific failed");
1464 return nullptr;
1465 }
1466 }
1467
1468 return TLVMgr->getInstance(
1469 ThreadData: reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1470}
1471
1472ORC_RT_INTERFACE orc_rt_CWrapperFunctionResult
1473__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1474 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1475 ArgData, ArgSize,
1476 Handler: []() -> Expected<uint64_t> {
1477 pthread_key_t Key;
1478 if (int Err = pthread_key_create(key: &Key, destr_function: destroyMachOTLVMgr)) {
1479 __orc_rt_log_error(ErrMsg: "Call to pthread_key_create failed");
1480 return make_error<StringError>(Args: strerror(errnum: Err));
1481 }
1482 return static_cast<uint64_t>(Key);
1483 })
1484 .release();
1485}
1486
1487//------------------------------------------------------------------------------
1488// cxa_atexit support
1489//------------------------------------------------------------------------------
1490
1491int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1492 void *dso_handle) {
1493 return MachOPlatformRuntimeState::get().registerAtExit(F: func, Arg: arg, DSOHandle: dso_handle);
1494}
1495
1496void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1497 MachOPlatformRuntimeState::get().runAtExits(DSOHandle: dso_handle);
1498}
1499
1500//------------------------------------------------------------------------------
1501// JIT'd dlfcn alternatives.
1502//------------------------------------------------------------------------------
1503
1504const char *__orc_rt_macho_jit_dlerror() {
1505 return MachOPlatformRuntimeState::get().dlerror();
1506}
1507
1508void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1509 return MachOPlatformRuntimeState::get().dlopen(Path: path, Mode: mode);
1510}
1511
1512int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1513 return MachOPlatformRuntimeState::get().dlclose(DSOHandle: dso_handle);
1514}
1515
1516void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1517 return MachOPlatformRuntimeState::get().dlsym(DSOHandle: dso_handle, Symbol: symbol);
1518}
1519
1520//------------------------------------------------------------------------------
1521// MachO Run Program
1522//------------------------------------------------------------------------------
1523
1524ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1525 const char *EntrySymbolName,
1526 int argc, char *argv[]) {
1527 using MainTy = int (*)(int, char *[]);
1528
1529 void *H = __orc_rt_macho_jit_dlopen(path: JITDylibName,
1530 mode: __orc_rt::macho::ORC_RT_RTLD_LAZY);
1531 if (!H) {
1532 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1533 return -1;
1534 }
1535
1536 auto *Main =
1537 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(dso_handle: H, symbol: EntrySymbolName));
1538
1539 if (!Main) {
1540 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1541 return -1;
1542 }
1543
1544 int Result = Main(argc, argv);
1545
1546 if (__orc_rt_macho_jit_dlclose(dso_handle: H) == -1)
1547 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1548
1549 return Result;
1550}
1551

source code of compiler-rt/lib/orc/macho_platform.cpp