1//===- macho_platform.cpp -------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code required to load the rest of the MachO runtime.
10//
11//===----------------------------------------------------------------------===//
12
13#include "macho_platform.h"
14#include "bitmask_enum.h"
15#include "common.h"
16#include "debug.h"
17#include "error.h"
18#include "interval_map.h"
19#include "jit_dispatch.h"
20#include "record_section_tracker.h"
21#include "wrapper_function_utils.h"
22
23#include <algorithm>
24#include <ios>
25#include <map>
26#include <mutex>
27#include <sstream>
28#include <string_view>
29#include <unordered_map>
30#include <unordered_set>
31#include <vector>
32
33#define DEBUG_TYPE "macho_platform"
34
35using namespace orc_rt;
36using namespace orc_rt::macho;
37
38// Declare function tags for functions in the JIT process.
39ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
40ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag)
41
42struct objc_image_info;
43struct mach_header;
44
45// Objective-C registration functions.
46// These are weakly imported. If the Objective-C runtime has not been loaded
47// then code containing Objective-C sections will generate an error.
48extern "C" void
49_objc_map_images(unsigned count, const char *const paths[],
50 const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;
51
52extern "C" void _objc_load_image(const char *path,
53 const mach_header *mh) ORC_RT_WEAK_IMPORT;
54
55// Libunwind prototypes.
56struct unw_dynamic_unwind_sections {
57 uintptr_t dso_base;
58 uintptr_t dwarf_section;
59 size_t dwarf_section_length;
60 uintptr_t compact_unwind_section;
61 size_t compact_unwind_section_length;
62};
63
64typedef int (*unw_find_dynamic_unwind_sections)(
65 uintptr_t addr, struct unw_dynamic_unwind_sections *info);
66
67extern "C" int __unw_add_find_dynamic_unwind_sections(
68 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
69 ORC_RT_WEAK_IMPORT;
70
71extern "C" int __unw_remove_find_dynamic_unwind_sections(
72 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
73 ORC_RT_WEAK_IMPORT;
74
75namespace {
76
77struct MachOJITDylibDepInfo {
78 bool Sealed = false;
79 std::vector<ExecutorAddr> DepHeaders;
80};
81
82using MachOJITDylibDepInfoMap =
83 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
84
85} // anonymous namespace
86
87namespace orc_rt {
88
89using SPSMachOObjectPlatformSectionsMap =
90 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
91
92using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
93
94using SPSMachOJITDylibDepInfoMap =
95 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
96
97template <>
98class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
99public:
100 static size_t size(const MachOJITDylibDepInfo &JDI) {
101 return SPSMachOJITDylibDepInfo::AsArgList::size(Arg: JDI.Sealed, Args: JDI.DepHeaders);
102 }
103
104 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
105 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, Arg: JDI.Sealed,
106 Args: JDI.DepHeaders);
107 }
108
109 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
110 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, Arg&: JDI.Sealed,
111 Args&: JDI.DepHeaders);
112 }
113};
114
115struct UnwindSectionInfo {
116 std::vector<ExecutorAddrRange> CodeRanges;
117 ExecutorAddrRange DwarfSection;
118 ExecutorAddrRange CompactUnwindSection;
119};
120
121using SPSUnwindSectionInfo =
122 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
123 SPSExecutorAddrRange>;
124
125template <>
126class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
127public:
128 static size_t size(const UnwindSectionInfo &USI) {
129 return SPSUnwindSectionInfo::AsArgList::size(
130 Arg: USI.CodeRanges, Args: USI.DwarfSection, Args: USI.CompactUnwindSection);
131 }
132
133 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
134 return SPSUnwindSectionInfo::AsArgList::serialize(
135 OB, Arg: USI.CodeRanges, Args: USI.DwarfSection, Args: USI.CompactUnwindSection);
136 }
137
138 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
139 return SPSUnwindSectionInfo::AsArgList::deserialize(
140 IB, Arg&: USI.CodeRanges, Args&: USI.DwarfSection, Args&: USI.CompactUnwindSection);
141 }
142};
143
144} // namespace orc_rt
145
146namespace {
147struct TLVDescriptor {
148 void *(*Thunk)(TLVDescriptor *) = nullptr;
149 unsigned long Key = 0;
150 unsigned long DataAddress = 0;
151};
152
153class MachOPlatformRuntimeState {
154public:
155 // Used internally by MachOPlatformRuntimeState, but made public to enable
156 // serialization.
157 enum class MachOExecutorSymbolFlags : uint8_t {
158 None = 0,
159 Weak = 1U << 0,
160 Callable = 1U << 1,
161 ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)
162 };
163
164private:
165 struct AtExitEntry {
166 void (*Func)(void *);
167 void *Arg;
168 };
169
170 using AtExitsVector = std::vector<AtExitEntry>;
171
172 struct UnwindSections {
173 UnwindSections(const UnwindSectionInfo &USI)
174 : DwarfSection(USI.DwarfSection.toSpan<char>()),
175 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
176
177 span<char> DwarfSection;
178 span<char> CompactUnwindSection;
179 };
180
181 using UnwindSectionsMap =
182 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
183
184 struct JITDylibState {
185
186 using SymbolTableMap =
187 std::unordered_map<std::string_view,
188 std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>;
189
190 std::string Name;
191 void *Header = nullptr;
192 bool Sealed = false;
193 size_t LinkedAgainstRefCount = 0;
194 size_t DlRefCount = 0;
195 SymbolTableMap SymbolTable;
196 std::vector<JITDylibState *> Deps;
197 AtExitsVector AtExits;
198 const objc_image_info *ObjCImageInfo = nullptr;
199 std::unordered_map<void *, std::vector<char>> DataSectionContent;
200 std::unordered_map<void *, size_t> ZeroInitRanges;
201 UnwindSectionsMap UnwindSections;
202 RecordSectionsTracker<void (*)()> ModInitsSections;
203 RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;
204
205 bool referenced() const {
206 return LinkedAgainstRefCount != 0 || DlRefCount != 0;
207 }
208 };
209
210public:
211 static Error create();
212 static MachOPlatformRuntimeState &get();
213 static Error destroy();
214
215 MachOPlatformRuntimeState() = default;
216
217 // Delete copy and move constructors.
218 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
219 MachOPlatformRuntimeState &
220 operator=(const MachOPlatformRuntimeState &) = delete;
221 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
222 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
223
224 Error initialize();
225 Error shutdown();
226
227 Error registerJITDylib(std::string Name, void *Header);
228 Error deregisterJITDylib(void *Header);
229 Error registerThreadDataSection(span<const char> ThreadDataSection);
230 Error deregisterThreadDataSection(span<const char> ThreadDataSection);
231 Error registerObjectSymbolTable(
232 ExecutorAddr HeaderAddr,
233 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
234 MachOExecutorSymbolFlags>> &Entries);
235 Error deregisterObjectSymbolTable(
236 ExecutorAddr HeaderAddr,
237 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
238 MachOExecutorSymbolFlags>> &Entries);
239 Error registerObjectPlatformSections(
240 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
241 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
242 Error deregisterObjectPlatformSections(
243 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
244 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
245
246 const char *dlerror();
247 void *dlopen(std::string_view Name, int Mode);
248 int dlupdate(void *DSOHandle);
249 int dlclose(void *DSOHandle);
250 void *dlsym(void *DSOHandle, const char *Symbol);
251
252 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
253 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
254 JITDylibState &JDS);
255 void runAtExits(void *DSOHandle);
256
257 /// Returns the base address of the section containing ThreadData.
258 Expected<std::pair<const char *, size_t>>
259 getThreadDataSectionFor(const char *ThreadData);
260
261private:
262 JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
263 JITDylibState *getJITDylibStateByName(std::string_view Path);
264
265 /// Requests materialization of the given symbols. For each pair, the bool
266 /// element indicates whether the symbol is required (true) or weakly
267 /// referenced (false).
268 Error requestPushSymbols(JITDylibState &JDS,
269 span<std::pair<std::string_view, bool>> Symbols);
270
271 /// Attempts to look up the given symbols locally, requesting a push from the
272 /// remote if they're not found. Results are written to the Result span, which
273 /// must have the same size as the Symbols span.
274 Error
275 lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
276 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
277 span<std::pair<std::string_view, bool>> Symbols);
278
279 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
280
281 static int findDynamicUnwindSections(uintptr_t addr,
282 unw_dynamic_unwind_sections *info);
283 static Error registerEHFrames(span<const char> EHFrameSection);
284 static Error deregisterEHFrames(span<const char> EHFrameSection);
285
286 static Error
287 registerObjCRegistrationObjects(std::unique_lock<std::mutex> &JDStatesLock,
288 JITDylibState &JDS);
289 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
290 JITDylibState &JDS);
291
292 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
293 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
294 JITDylibState &JDS);
295 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
296 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
297
298 Error dlupdateImpl(void *DSOHandle);
299 Error dlupdateFull(std::unique_lock<std::mutex> &JDStatesLock,
300 JITDylibState &JDS);
301 Error dlupdateInitialize(std::unique_lock<std::mutex> &JDStatesLock,
302 JITDylibState &JDS);
303
304 Error dlcloseImpl(void *DSOHandle);
305 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
306 JITDylibState &JDS);
307
308 static MachOPlatformRuntimeState *MOPS;
309
310 bool UseCallbackStyleUnwindInfo = false;
311
312 // FIXME: Move to thread-state.
313 std::string DLFcnError;
314
315 // APIMutex guards against concurrent entry into key "dyld" API functions
316 // (e.g. dlopen, dlclose).
317 std::recursive_mutex DyldAPIMutex;
318
319 // JDStatesMutex guards the data structures that hold JITDylib state.
320 std::mutex JDStatesMutex;
321 std::unordered_map<void *, JITDylibState> JDStates;
322 std::unordered_map<std::string_view, void *> JDNameToHeader;
323
324 // ThreadDataSectionsMutex guards thread local data section state.
325 std::mutex ThreadDataSectionsMutex;
326 std::map<const char *, size_t> ThreadDataSections;
327};
328
329} // anonymous namespace
330
331namespace orc_rt {
332
333class SPSMachOExecutorSymbolFlags;
334
335template <>
336class SPSSerializationTraits<
337 SPSMachOExecutorSymbolFlags,
338 MachOPlatformRuntimeState::MachOExecutorSymbolFlags> {
339private:
340 using UT = std::underlying_type_t<
341 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>;
342
343public:
344 static size_t
345 size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
346 return sizeof(UT);
347 }
348
349 static bool
350 serialize(SPSOutputBuffer &OB,
351 const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
352 return SPSArgList<UT>::serialize(OB, Arg: static_cast<UT>(SF));
353 }
354
355 static bool
356 deserialize(SPSInputBuffer &IB,
357 MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
358 UT Tmp;
359 if (!SPSArgList<UT>::deserialize(IB, Arg&: Tmp))
360 return false;
361 SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp);
362 return true;
363 }
364};
365
366} // namespace orc_rt
367
368namespace {
369
370MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
371
372Error MachOPlatformRuntimeState::create() {
373 assert(!MOPS && "MachOPlatformRuntimeState should be null");
374 MOPS = new MachOPlatformRuntimeState();
375 return MOPS->initialize();
376}
377
378MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
379 assert(MOPS && "MachOPlatformRuntimeState not initialized");
380 return *MOPS;
381}
382
383Error MachOPlatformRuntimeState::destroy() {
384 assert(MOPS && "MachOPlatformRuntimeState not initialized");
385 auto Err = MOPS->shutdown();
386 delete MOPS;
387 return Err;
388}
389
390Error MachOPlatformRuntimeState::initialize() {
391 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
392 __unw_remove_find_dynamic_unwind_sections;
393 if (UseCallbackStyleUnwindInfo) {
394 ORC_RT_DEBUG({
395 printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
396 " Using callback-based frame info lookup.\n");
397 });
398 if (__unw_add_find_dynamic_unwind_sections(find_dynamic_unwind_sections: &findDynamicUnwindSections))
399 return make_error<StringError>(
400 Args: "Could not register findDynamicUnwindSections");
401 } else {
402 ORC_RT_DEBUG({
403 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
404 " Using classic frame info registration.\n");
405 });
406 }
407 return Error::success();
408}
409
410Error MachOPlatformRuntimeState::shutdown() {
411 if (UseCallbackStyleUnwindInfo) {
412 if (__unw_remove_find_dynamic_unwind_sections(find_dynamic_unwind_sections: &findDynamicUnwindSections)) {
413 ORC_RT_DEBUG(
414 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
415 }
416 }
417 return Error::success();
418}
419
420Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
421 void *Header) {
422 ORC_RT_DEBUG({
423 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
424 });
425 std::lock_guard<std::mutex> Lock(JDStatesMutex);
426 if (JDStates.count(x: Header)) {
427 std::ostringstream ErrStream;
428 ErrStream << "Duplicate JITDylib registration for header " << Header
429 << " (name = " << Name << ")";
430 return make_error<StringError>(Args: ErrStream.str());
431 }
432 if (JDNameToHeader.count(x: Name)) {
433 std::ostringstream ErrStream;
434 ErrStream << "Duplicate JITDylib registration for header " << Header
435 << " (header = " << Header << ")";
436 return make_error<StringError>(Args: ErrStream.str());
437 }
438
439 auto &JDS = JDStates[Header];
440 JDS.Name = std::move(t&: Name);
441 JDS.Header = Header;
442 JDNameToHeader[JDS.Name] = Header;
443 return Error::success();
444}
445
446Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
447 std::lock_guard<std::mutex> Lock(JDStatesMutex);
448 auto I = JDStates.find(x: Header);
449 if (I == JDStates.end()) {
450 std::ostringstream ErrStream;
451 ErrStream << "Attempted to deregister unrecognized header " << Header;
452 return make_error<StringError>(Args: ErrStream.str());
453 }
454
455 // Remove std::string construction once we can use C++20.
456 auto J = JDNameToHeader.find(
457 x: std::string(I->second.Name.data(), I->second.Name.size()));
458 assert(J != JDNameToHeader.end() &&
459 "Missing JDNameToHeader entry for JITDylib");
460
461 ORC_RT_DEBUG({
462 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
463 Header);
464 });
465
466 JDNameToHeader.erase(position: J);
467 JDStates.erase(position: I);
468 return Error::success();
469}
470
471Error MachOPlatformRuntimeState::registerThreadDataSection(
472 span<const char> ThreadDataSection) {
473 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
474 auto I = ThreadDataSections.upper_bound(x: ThreadDataSection.data());
475 if (I != ThreadDataSections.begin()) {
476 auto J = std::prev(x: I);
477 if (J->first + J->second > ThreadDataSection.data())
478 return make_error<StringError>(Args: "Overlapping __thread_data sections");
479 }
480 ThreadDataSections.insert(
481 position: I, x: std::make_pair(x: ThreadDataSection.data(), y: ThreadDataSection.size()));
482 return Error::success();
483}
484
485Error MachOPlatformRuntimeState::deregisterThreadDataSection(
486 span<const char> ThreadDataSection) {
487 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
488 auto I = ThreadDataSections.find(x: ThreadDataSection.data());
489 if (I == ThreadDataSections.end())
490 return make_error<StringError>(Args: "Attempt to deregister unknown thread data "
491 "section");
492 ThreadDataSections.erase(position: I);
493 return Error::success();
494}
495
496Error MachOPlatformRuntimeState::registerObjectSymbolTable(
497 ExecutorAddr HeaderAddr,
498 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
499 MachOExecutorSymbolFlags>> &Entries) {
500
501 std::lock_guard<std::mutex> Lock(JDStatesMutex);
502 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
503 if (!JDS) {
504 std::ostringstream ErrStream;
505 ErrStream << "Could not register object platform sections for "
506 "unrecognized header "
507 << HeaderAddr.toPtr<void *>();
508 return make_error<StringError>(Args: ErrStream.str());
509 }
510
511 for (auto &[NameAddr, SymAddr, Flags] : Entries)
512 JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags};
513
514 return Error::success();
515}
516
517Error MachOPlatformRuntimeState::deregisterObjectSymbolTable(
518 ExecutorAddr HeaderAddr,
519 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
520 MachOExecutorSymbolFlags>> &Entries) {
521
522 std::lock_guard<std::mutex> Lock(JDStatesMutex);
523 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
524 if (!JDS) {
525 std::ostringstream ErrStream;
526 ErrStream << "Could not register object platform sections for "
527 "unrecognized header "
528 << HeaderAddr.toPtr<void *>();
529 return make_error<StringError>(Args: ErrStream.str());
530 }
531
532 for (auto &[NameAddr, SymAddr, Flags] : Entries)
533 JDS->SymbolTable.erase(x: NameAddr.toPtr<const char *>());
534
535 return Error::success();
536}
537
538Error MachOPlatformRuntimeState::registerObjectPlatformSections(
539 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
540 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
541
542 // FIXME: Reject platform section registration after the JITDylib is
543 // sealed?
544
545 ORC_RT_DEBUG({
546 printdbg("MachOPlatform: Registering object sections for %p.\n",
547 HeaderAddr.toPtr<void *>());
548 });
549
550 std::lock_guard<std::mutex> Lock(JDStatesMutex);
551 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
552 if (!JDS) {
553 std::ostringstream ErrStream;
554 ErrStream << "Could not register object platform sections for "
555 "unrecognized header "
556 << HeaderAddr.toPtr<void *>();
557 return make_error<StringError>(Args: ErrStream.str());
558 }
559
560 ORC_RT_DEBUG({
561 printdbg(" UnwindInfo: %s, UseCallbackStyleUnwindInfo: %s\n",
562 UnwindInfo ? "true" : "false",
563 UseCallbackStyleUnwindInfo ? "true" : "false");
564 });
565
566 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
567 ORC_RT_DEBUG({
568 printdbg(" Registering new-style unwind info for:\n"
569 " DWARF: %p -- %p\n"
570 " Compact-unwind: %p -- %p\n"
571 " for:\n",
572 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
573 UnwindInfo->DwarfSection.End.toPtr<void *>(),
574 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
575 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
576 });
577 for (auto &CodeRange : UnwindInfo->CodeRanges) {
578 JDS->UnwindSections.insert(KS: CodeRange.Start.toPtr<char *>(),
579 KE: CodeRange.End.toPtr<char *>(), V: *UnwindInfo);
580 ORC_RT_DEBUG({
581 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
582 CodeRange.End.toPtr<void *>());
583 });
584 }
585 }
586
587 for (auto &KV : Secs) {
588 // FIXME: Validate section ranges?
589 if (KV.first == "__TEXT,__eh_frame") {
590 if (!UseCallbackStyleUnwindInfo) {
591 // Use classic libunwind registration.
592 if (auto Err = registerEHFrames(EHFrameSection: KV.second.toSpan<const char>()))
593 return Err;
594 }
595 } else if (KV.first == "__DATA,__data") {
596 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
597 "Address already registered.");
598 auto S = KV.second.toSpan<char>();
599 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
600 std::vector<char>(S.begin(), S.end());
601 } else if (KV.first == "__DATA,__common") {
602 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
603 } else if (KV.first == "__DATA,__thread_data") {
604 if (auto Err = registerThreadDataSection(ThreadDataSection: KV.second.toSpan<const char>()))
605 return Err;
606 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
607 JDS->ObjCRuntimeRegistrationObjects.add(Sec: KV.second.toSpan<char>());
608 else if (KV.first == "__DATA,__mod_init_func")
609 JDS->ModInitsSections.add(Sec: KV.second.toSpan<void (*)()>());
610 else {
611 // Should this be a warning instead?
612 return make_error<StringError>(
613 Args: "Encountered unexpected section " +
614 std::string(KV.first.data(), KV.first.size()) +
615 " while registering object platform sections");
616 }
617 }
618
619 return Error::success();
620}
621
622Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
623 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
624 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
625 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
626 // TODO: Add a JITDylib prepare-for-teardown operation that clears all
627 // registered sections, causing this function to take the fast-path.
628 ORC_RT_DEBUG({
629 printdbg("MachOPlatform: Deregistering object sections for %p.\n",
630 HeaderAddr.toPtr<void *>());
631 });
632
633 std::lock_guard<std::mutex> Lock(JDStatesMutex);
634 auto *JDS = getJITDylibStateByHeader(DSOHandle: HeaderAddr.toPtr<void *>());
635 if (!JDS) {
636 std::ostringstream ErrStream;
637 ErrStream << "Could not register object platform sections for unrecognized "
638 "header "
639 << HeaderAddr.toPtr<void *>();
640 return make_error<StringError>(Args: ErrStream.str());
641 }
642
643 // FIXME: Implement faster-path by returning immediately if JDS is being
644 // torn down entirely?
645
646 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
647 // any Swift or ObjC. Once this happens we can clear (and no longer record)
648 // data section content, as the library could never be re-initialized.
649
650 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
651 ORC_RT_DEBUG({
652 printdbg(" Deregistering new-style unwind info for:\n"
653 " DWARF: %p -- %p\n"
654 " Compact-unwind: %p -- %p\n"
655 " for:\n",
656 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
657 UnwindInfo->DwarfSection.End.toPtr<void *>(),
658 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
659 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
660 });
661 for (auto &CodeRange : UnwindInfo->CodeRanges) {
662 JDS->UnwindSections.erase(KS: CodeRange.Start.toPtr<char *>(),
663 KE: CodeRange.End.toPtr<char *>());
664 ORC_RT_DEBUG({
665 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
666 CodeRange.End.toPtr<void *>());
667 });
668 }
669 }
670
671 for (auto &KV : Secs) {
672 // FIXME: Validate section ranges?
673 if (KV.first == "__TEXT,__eh_frame") {
674 if (!UseCallbackStyleUnwindInfo) {
675 // Use classic libunwind registration.
676 if (auto Err = deregisterEHFrames(EHFrameSection: KV.second.toSpan<const char>()))
677 return Err;
678 }
679 } else if (KV.first == "__DATA,__data") {
680 JDS->DataSectionContent.erase(x: KV.second.Start.toPtr<char *>());
681 } else if (KV.first == "__DATA,__common") {
682 JDS->ZeroInitRanges.erase(x: KV.second.Start.toPtr<char *>());
683 } else if (KV.first == "__DATA,__thread_data") {
684 if (auto Err =
685 deregisterThreadDataSection(ThreadDataSection: KV.second.toSpan<const char>()))
686 return Err;
687 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
688 JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(R: KV.second);
689 else if (KV.first == "__DATA,__mod_init_func")
690 JDS->ModInitsSections.removeIfPresent(R: KV.second);
691 else {
692 // Should this be a warning instead?
693 return make_error<StringError>(
694 Args: "Encountered unexpected section " +
695 std::string(KV.first.data(), KV.first.size()) +
696 " while deregistering object platform sections");
697 }
698 }
699 return Error::success();
700}
701
702const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
703
704void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
705 ORC_RT_DEBUG({
706 std::string S(Path.data(), Path.size());
707 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
708 });
709 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
710 if (auto H = dlopenImpl(Path, Mode))
711 return *H;
712 else {
713 // FIXME: Make dlerror thread safe.
714 DLFcnError = toString(Err: H.takeError());
715 return nullptr;
716 }
717}
718
719int MachOPlatformRuntimeState::dlupdate(void *DSOHandle) {
720 ORC_RT_DEBUG({
721 std::string S;
722 printdbg("MachOPlatform::dlupdate(%p) (%s)\n", DSOHandle, S.c_str());
723 });
724 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
725 if (auto Err = dlupdateImpl(DSOHandle)) {
726 // FIXME: Make dlerror thread safe.
727 DLFcnError = toString(Err: std::move(t&: Err));
728 return -1;
729 }
730 return 0;
731}
732
733int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
734 ORC_RT_DEBUG({
735 auto *JDS = getJITDylibStateByHeader(DSOHandle);
736 std::string DylibName;
737 if (JDS) {
738 std::string S;
739 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
740 } else
741 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
742 "invalid handle");
743 });
744 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
745 if (auto Err = dlcloseImpl(DSOHandle)) {
746 // FIXME: Make dlerror thread safe.
747 DLFcnError = toString(Err: std::move(t&: Err));
748 return -1;
749 }
750 return 0;
751}
752
753void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) {
754 std::unique_lock<std::mutex> Lock(JDStatesMutex);
755 auto *JDS = getJITDylibStateByHeader(DSOHandle);
756 if (!JDS) {
757 std::ostringstream ErrStream;
758 ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle;
759 DLFcnError = ErrStream.str();
760 return nullptr;
761 }
762
763 std::string MangledName = std::string("_") + Symbol;
764 std::pair<std::string_view, bool> Lookup(MangledName, false);
765 std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result;
766
767 if (auto Err = lookupSymbols(JDS&: *JDS, JDStatesLock&: Lock, Result: {&Result, 1}, Symbols: {&Lookup, 1})) {
768 DLFcnError = toString(Err: std::move(t&: Err));
769 return nullptr;
770 }
771
772 // Sign callable symbols as functions, to match dyld.
773 if ((Result.second & MachOExecutorSymbolFlags::Callable) ==
774 MachOExecutorSymbolFlags::Callable)
775 return reinterpret_cast<void *>(Result.first.toPtr<void(void)>());
776 return Result.first.toPtr<void *>();
777}
778
779int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
780 void *DSOHandle) {
781 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
782 std::lock_guard<std::mutex> Lock(JDStatesMutex);
783 auto *JDS = getJITDylibStateByHeader(DSOHandle);
784 if (!JDS) {
785 ORC_RT_DEBUG({
786 printdbg("MachOPlatformRuntimeState::registerAtExit called with "
787 "unrecognized dso handle %p\n",
788 DSOHandle);
789 });
790 return -1;
791 }
792 JDS->AtExits.push_back(x: {.Func: F, .Arg: Arg});
793 return 0;
794}
795
796void MachOPlatformRuntimeState::runAtExits(
797 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
798 auto AtExits = std::move(t&: JDS.AtExits);
799
800 // Unlock while running atexits, as they may trigger operations that modify
801 // JDStates.
802 JDStatesLock.unlock();
803 while (!AtExits.empty()) {
804 auto &AE = AtExits.back();
805 AE.Func(AE.Arg);
806 AtExits.pop_back();
807 }
808 JDStatesLock.lock();
809}
810
811void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
812 std::unique_lock<std::mutex> Lock(JDStatesMutex);
813 auto *JDS = getJITDylibStateByHeader(DSOHandle);
814 ORC_RT_DEBUG({
815 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
816 "dso_handle %p\n",
817 DSOHandle);
818 });
819 if (JDS)
820 runAtExits(JDStatesLock&: Lock, JDS&: *JDS);
821}
822
823Expected<std::pair<const char *, size_t>>
824MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
825 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
826 auto I = ThreadDataSections.upper_bound(x: ThreadData);
827 // Check that we have a valid entry covering this address.
828 if (I == ThreadDataSections.begin())
829 return make_error<StringError>(Args: "No thread local data section for key");
830 I = std::prev(x: I);
831 if (ThreadData >= I->first + I->second)
832 return make_error<StringError>(Args: "No thread local data section for key");
833 return *I;
834}
835
836MachOPlatformRuntimeState::JITDylibState *
837MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
838 auto I = JDStates.find(x: DSOHandle);
839 if (I == JDStates.end()) {
840 I = JDStates.insert(x: std::make_pair(x&: DSOHandle, y: JITDylibState())).first;
841 I->second.Header = DSOHandle;
842 }
843 return &I->second;
844}
845
846MachOPlatformRuntimeState::JITDylibState *
847MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
848 // FIXME: Avoid creating string once we have C++20.
849 auto I = JDNameToHeader.find(x: std::string(Name.data(), Name.size()));
850 if (I != JDNameToHeader.end())
851 return getJITDylibStateByHeader(DSOHandle: I->second);
852 return nullptr;
853}
854
855Error MachOPlatformRuntimeState::requestPushSymbols(
856 JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) {
857 Error OpErr = Error::success();
858 if (auto Err = WrapperFunction<SPSError(
859 SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>::
860 call(Dispatch: JITDispatch(&__orc_rt_macho_push_symbols_tag), Result&: OpErr,
861 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header), Args: Symbols)) {
862 cantFail(Err: std::move(t&: OpErr));
863 return std::move(t&: Err);
864 }
865 return OpErr;
866}
867
868Error MachOPlatformRuntimeState::lookupSymbols(
869 JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
870 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
871 span<std::pair<std::string_view, bool>> Symbols) {
872 assert(JDStatesLock.owns_lock() &&
873 "JDStatesLock should be locked at call-site");
874 assert(Result.size() == Symbols.size() &&
875 "Results and Symbols span sizes should match");
876
877 // Make an initial pass over the local symbol table.
878 std::vector<size_t> MissingSymbolIndexes;
879 for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) {
880 auto I = JDS.SymbolTable.find(x: Symbols[Idx].first);
881 if (I != JDS.SymbolTable.end())
882 Result[Idx] = I->second;
883 else
884 MissingSymbolIndexes.push_back(x: Idx);
885 }
886
887 // If everything has been resolved already then bail out early.
888 if (MissingSymbolIndexes.empty())
889 return Error::success();
890
891 // Otherwise call back to the controller to try to request that the symbol
892 // be materialized.
893 std::vector<std::pair<std::string_view, bool>> MissingSymbols;
894 MissingSymbols.reserve(n: MissingSymbolIndexes.size());
895 ORC_RT_DEBUG({
896 printdbg("requesting push of %i missing symbols...\n",
897 MissingSymbolIndexes.size());
898 });
899 for (auto MissingIdx : MissingSymbolIndexes)
900 MissingSymbols.push_back(x: Symbols[MissingIdx]);
901
902 JDStatesLock.unlock();
903 if (auto Err = requestPushSymbols(
904 JDS, Symbols: {MissingSymbols.data(), MissingSymbols.size()}))
905 return Err;
906 JDStatesLock.lock();
907
908 // Try to resolve the previously missing symbols locally.
909 std::vector<size_t> MissingRequiredSymbols;
910 for (auto MissingIdx : MissingSymbolIndexes) {
911 auto I = JDS.SymbolTable.find(x: Symbols[MissingIdx].first);
912 if (I != JDS.SymbolTable.end())
913 Result[MissingIdx] = I->second;
914 else {
915 if (Symbols[MissingIdx].second)
916 MissingRequiredSymbols.push_back(x: MissingIdx);
917 else
918 Result[MissingIdx] = {ExecutorAddr(), {}};
919 }
920 }
921
922 // Error out if any missing symbols could not be resolved.
923 if (!MissingRequiredSymbols.empty()) {
924 std::ostringstream ErrStream;
925 ErrStream << "Lookup could not find required symbols: [ ";
926 for (auto MissingIdx : MissingRequiredSymbols)
927 ErrStream << "\"" << Symbols[MissingIdx].first << "\" ";
928 ErrStream << "]";
929 return make_error<StringError>(Args: ErrStream.str());
930 }
931
932 return Error::success();
933}
934
935// eh-frame registration functions.
936// We expect these to be available for all processes.
937extern "C" void __register_frame(const void *);
938extern "C" void __deregister_frame(const void *);
939
940template <typename HandleFDEFn>
941void walkEHFrameSection(span<const char> EHFrameSection,
942 HandleFDEFn HandleFDE) {
943 const char *CurCFIRecord = EHFrameSection.data();
944 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
945
946 while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
947 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
948 if (Size == 0xffffffff)
949 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
950 else
951 Size += 4;
952 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
953
954 if (Offset != 0)
955 HandleFDE(CurCFIRecord);
956
957 CurCFIRecord += Size;
958 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
959 }
960}
961
962bool MachOPlatformRuntimeState::lookupUnwindSections(
963 void *Addr, unw_dynamic_unwind_sections &Info) {
964 ORC_RT_DEBUG(
965 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
966 std::lock_guard<std::mutex> Lock(JDStatesMutex);
967 for (auto &KV : JDStates) {
968 auto &JD = KV.second;
969 auto I = JD.UnwindSections.find(K: reinterpret_cast<char *>(Addr));
970 if (I != JD.UnwindSections.end()) {
971 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
972 Info.dwarf_section =
973 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
974 Info.dwarf_section_length = I->second.DwarfSection.size();
975 Info.compact_unwind_section =
976 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
977 Info.compact_unwind_section_length =
978 I->second.CompactUnwindSection.size();
979 return true;
980 }
981 }
982 return false;
983}
984
985int MachOPlatformRuntimeState::findDynamicUnwindSections(
986 uintptr_t addr, unw_dynamic_unwind_sections *info) {
987 if (!info)
988 return 0;
989 return MachOPlatformRuntimeState::get().lookupUnwindSections(Addr: (void *)addr,
990 Info&: *info);
991}
992
993Error MachOPlatformRuntimeState::registerEHFrames(
994 span<const char> EHFrameSection) {
995 walkEHFrameSection(EHFrameSection, HandleFDE: __register_frame);
996 return Error::success();
997}
998
999Error MachOPlatformRuntimeState::deregisterEHFrames(
1000 span<const char> EHFrameSection) {
1001 walkEHFrameSection(EHFrameSection, HandleFDE: __deregister_frame);
1002 return Error::success();
1003}
1004
1005Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(
1006 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1007 ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));
1008
1009 std::vector<char *> RegObjBases;
1010 JDS.ObjCRuntimeRegistrationObjects.processNewSections(
1011 ProcessSection: [&](span<char> RegObj) { RegObjBases.push_back(x: RegObj.data()); });
1012
1013 if (RegObjBases.empty())
1014 return Error::success();
1015
1016 if (!_objc_map_images || !_objc_load_image)
1017 return make_error<StringError>(
1018 Args: "Could not register Objective-C / Swift metadata: _objc_map_images / "
1019 "_objc_load_image not found");
1020
1021 // Release the lock while calling out to libobjc in case +load methods cause
1022 // reentering the orc runtime.
1023 JDStatesLock.unlock();
1024 std::vector<char *> Paths;
1025 Paths.resize(new_size: RegObjBases.size());
1026 _objc_map_images(count: RegObjBases.size(), paths: Paths.data(),
1027 mhdrs: reinterpret_cast<mach_header **>(RegObjBases.data()));
1028
1029 for (void *RegObjBase : RegObjBases)
1030 _objc_load_image(path: nullptr, mh: reinterpret_cast<mach_header *>(RegObjBase));
1031 JDStatesLock.lock();
1032
1033 return Error::success();
1034}
1035
1036Error MachOPlatformRuntimeState::runModInits(
1037 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1038 std::vector<span<void (*)()>> InitSections;
1039 InitSections.reserve(n: JDS.ModInitsSections.numNewSections());
1040
1041 // Copy initializer sections: If the JITDylib is unsealed then the
1042 // initializers could reach back into the JIT and cause more initializers to
1043 // be added.
1044 // FIXME: Skip unlock and run in-place on sealed JITDylibs?
1045 JDS.ModInitsSections.processNewSections(
1046 ProcessSection: [&](span<void (*)()> Inits) { InitSections.push_back(x: Inits); });
1047
1048 JDStatesLock.unlock();
1049 for (auto InitSec : InitSections)
1050 for (auto *Init : InitSec)
1051 Init();
1052 JDStatesLock.lock();
1053
1054 return Error::success();
1055}
1056
1057Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
1058 int Mode) {
1059 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1060
1061 // Try to find JITDylib state by name.
1062 auto *JDS = getJITDylibStateByName(Name: Path);
1063
1064 if (!JDS)
1065 return make_error<StringError>(Args: "No registered JTIDylib for path " +
1066 std::string(Path.data(), Path.size()));
1067
1068 // If this JITDylib is unsealed, or this is the first dlopen then run
1069 // full dlopen path (update deps, push and run initializers, update ref
1070 // counts on all JITDylibs in the dep tree).
1071 if (!JDS->referenced() || !JDS->Sealed) {
1072 if (auto Err = dlopenFull(JDStatesLock&: Lock, JDS&: *JDS))
1073 return std::move(t&: Err);
1074 }
1075
1076 // Bump the ref-count on this dylib.
1077 ++JDS->DlRefCount;
1078
1079 // Return the header address.
1080 return JDS->Header;
1081}
1082
1083Error MachOPlatformRuntimeState::dlopenFull(
1084 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1085 // Call back to the JIT to push the initializers.
1086 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1087 // Unlock so that we can accept the initializer update.
1088 JDStatesLock.unlock();
1089 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1090 SPSExecutorAddr)>::
1091 call(Dispatch: JITDispatch(&__orc_rt_macho_push_initializers_tag), Result&: DepInfo,
1092 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
1093 return Err;
1094 JDStatesLock.lock();
1095
1096 if (!DepInfo)
1097 return DepInfo.takeError();
1098
1099 if (auto Err = dlopenInitialize(JDStatesLock, JDS, DepInfo&: *DepInfo))
1100 return Err;
1101
1102 if (!DepInfo->empty()) {
1103 ORC_RT_DEBUG({
1104 printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
1105 JDS.Name.c_str());
1106 });
1107 std::ostringstream ErrStream;
1108 ErrStream << "Encountered unrecognized dep-info key headers "
1109 "while processing dlopen of "
1110 << JDS.Name;
1111 return make_error<StringError>(Args: ErrStream.str());
1112 }
1113
1114 return Error::success();
1115}
1116
1117Error MachOPlatformRuntimeState::dlopenInitialize(
1118 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
1119 MachOJITDylibDepInfoMap &DepInfo) {
1120 ORC_RT_DEBUG({
1121 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
1122 JDS.Name.c_str());
1123 });
1124
1125 // If the header is not present in the dep map then assume that we
1126 // already processed it earlier in the dlopenInitialize traversal and
1127 // return.
1128 // TODO: Keep a visited set instead so that we can error out on missing
1129 // entries?
1130 auto I = DepInfo.find(x: ExecutorAddr::fromPtr(Ptr: JDS.Header));
1131 if (I == DepInfo.end())
1132 return Error::success();
1133
1134 auto DI = std::move(t&: I->second);
1135 DepInfo.erase(position: I);
1136
1137 // We don't need to re-initialize sealed JITDylibs that have already been
1138 // initialized. Just check that their dep-map entry is empty as expected.
1139 if (JDS.Sealed) {
1140 if (!DI.DepHeaders.empty()) {
1141 std::ostringstream ErrStream;
1142 ErrStream << "Sealed JITDylib " << JDS.Header
1143 << " already has registered dependencies";
1144 return make_error<StringError>(Args: ErrStream.str());
1145 }
1146 if (JDS.referenced())
1147 return Error::success();
1148 } else
1149 JDS.Sealed = DI.Sealed;
1150
1151 // This is an unsealed or newly sealed JITDylib. Run initializers.
1152 std::vector<JITDylibState *> OldDeps;
1153 std::swap(x&: JDS.Deps, y&: OldDeps);
1154 JDS.Deps.reserve(n: DI.DepHeaders.size());
1155 for (auto DepHeaderAddr : DI.DepHeaders) {
1156 auto *DepJDS = getJITDylibStateByHeader(DSOHandle: DepHeaderAddr.toPtr<void *>());
1157 if (!DepJDS) {
1158 std::ostringstream ErrStream;
1159 ErrStream << "Encountered unrecognized dep header "
1160 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1161 << JDS.Name;
1162 return make_error<StringError>(Args: ErrStream.str());
1163 }
1164 ++DepJDS->LinkedAgainstRefCount;
1165 if (auto Err = dlopenInitialize(JDStatesLock, JDS&: *DepJDS, DepInfo))
1166 return Err;
1167 }
1168
1169 // Initialize this JITDylib.
1170 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))
1171 return Err;
1172 if (auto Err = runModInits(JDStatesLock, JDS))
1173 return Err;
1174
1175 // Decrement old deps.
1176 // FIXME: We should probably continue and just report deinitialize errors
1177 // here.
1178 for (auto *DepJDS : OldDeps) {
1179 --DepJDS->LinkedAgainstRefCount;
1180 if (!DepJDS->referenced())
1181 if (auto Err = dlcloseDeinitialize(JDStatesLock, JDS&: *DepJDS))
1182 return Err;
1183 }
1184
1185 return Error::success();
1186}
1187
1188Error MachOPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
1189 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1190
1191 // Try to find JITDylib state by DSOHandle.
1192 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1193
1194 if (!JDS) {
1195 std::ostringstream ErrStream;
1196 ErrStream << "No registered JITDylib for " << DSOHandle;
1197 return make_error<StringError>(Args: ErrStream.str());
1198 }
1199
1200 if (!JDS->referenced())
1201 return make_error<StringError>(Args: "dlupdate failed, JITDylib must be open.");
1202
1203 if (!JDS->Sealed) {
1204 if (auto Err = dlupdateFull(JDStatesLock&: Lock, JDS&: *JDS))
1205 return Err;
1206 }
1207
1208 return Error::success();
1209}
1210
1211Error MachOPlatformRuntimeState::dlupdateFull(
1212 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1213 // Call back to the JIT to push the initializers.
1214 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1215 // Unlock so that we can accept the initializer update.
1216 JDStatesLock.unlock();
1217 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1218 SPSExecutorAddr)>::
1219 call(Dispatch: JITDispatch(&__orc_rt_macho_push_initializers_tag), Result&: DepInfo,
1220 Args: ExecutorAddr::fromPtr(Ptr: JDS.Header)))
1221 return Err;
1222 JDStatesLock.lock();
1223
1224 if (!DepInfo)
1225 return DepInfo.takeError();
1226
1227 if (auto Err = dlupdateInitialize(JDStatesLock, JDS))
1228 return Err;
1229
1230 return Error::success();
1231}
1232
1233Error MachOPlatformRuntimeState::dlupdateInitialize(
1234 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1235 ORC_RT_DEBUG({
1236 printdbg("MachOPlatformRuntimeState::dlupdateInitialize(\"%s\")\n",
1237 JDS.Name.c_str());
1238 });
1239
1240 // Initialize this JITDylib.
1241 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))
1242 return Err;
1243 if (auto Err = runModInits(JDStatesLock, JDS))
1244 return Err;
1245
1246 return Error::success();
1247}
1248
1249Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1250 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1251
1252 // Try to find JITDylib state by header.
1253 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1254
1255 if (!JDS) {
1256 std::ostringstream ErrStream;
1257 ErrStream << "No registered JITDylib for " << DSOHandle;
1258 return make_error<StringError>(Args: ErrStream.str());
1259 }
1260
1261 // Bump the ref-count.
1262 --JDS->DlRefCount;
1263
1264 if (!JDS->referenced())
1265 return dlcloseDeinitialize(JDStatesLock&: Lock, JDS&: *JDS);
1266
1267 return Error::success();
1268}
1269
1270Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1271 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1272
1273 ORC_RT_DEBUG({
1274 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1275 JDS.Name.c_str());
1276 });
1277
1278 runAtExits(JDStatesLock, JDS);
1279
1280 // Reset mod-inits
1281 JDS.ModInitsSections.reset();
1282
1283 // Reset data section contents.
1284 for (auto &KV : JDS.DataSectionContent)
1285 memcpy(dest: KV.first, src: KV.second.data(), n: KV.second.size());
1286 for (auto &KV : JDS.ZeroInitRanges)
1287 memset(s: KV.first, c: 0, n: KV.second);
1288
1289 // Deinitialize any dependencies.
1290 for (auto *DepJDS : JDS.Deps) {
1291 --DepJDS->LinkedAgainstRefCount;
1292 if (!DepJDS->referenced())
1293 if (auto Err = dlcloseDeinitialize(JDStatesLock, JDS&: *DepJDS))
1294 return Err;
1295 }
1296
1297 return Error::success();
1298}
1299
1300class MachOPlatformRuntimeTLVManager {
1301public:
1302 void *getInstance(const char *ThreadData);
1303
1304private:
1305 std::unordered_map<const char *, char *> Instances;
1306 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1307};
1308
1309void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1310 auto I = Instances.find(x: ThreadData);
1311 if (I != Instances.end())
1312 return I->second;
1313
1314 auto TDS =
1315 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1316 if (!TDS) {
1317 __orc_rt_log_error(ErrMsg: toString(Err: TDS.takeError()).c_str());
1318 return nullptr;
1319 }
1320
1321 auto &Allocated = AllocatedSections[TDS->first];
1322 if (!Allocated) {
1323 Allocated = std::make_unique<char[]>(num: TDS->second);
1324 memcpy(dest: Allocated.get(), src: TDS->first, n: TDS->second);
1325 }
1326
1327 size_t ThreadDataDelta = ThreadData - TDS->first;
1328 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1329
1330 char *Instance = Allocated.get() + ThreadDataDelta;
1331 Instances[ThreadData] = Instance;
1332 return Instance;
1333}
1334
1335void destroyMachOTLVMgr(void *MachOTLVMgr) {
1336 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1337}
1338
1339Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1340 for (auto &WFC : WFCs)
1341 if (auto Err = WFC.runWithSPSRet<void>())
1342 return Err;
1343 return Error::success();
1344}
1345
1346} // end anonymous namespace
1347
1348//------------------------------------------------------------------------------
1349// JIT entry points
1350//------------------------------------------------------------------------------
1351
1352ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1353__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1354 return WrapperFunction<SPSError()>::handle(
1355 ArgData, ArgSize,
1356 Handler: []() { return MachOPlatformRuntimeState::create(); })
1357 .release();
1358}
1359
1360ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1361__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1362 return WrapperFunction<SPSError()>::handle(
1363 ArgData, ArgSize,
1364 Handler: []() { return MachOPlatformRuntimeState::destroy(); })
1365 .release();
1366}
1367
1368ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1369__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1370 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1371 ArgData, ArgSize,
1372 Handler: [](std::string &Name, ExecutorAddr HeaderAddr) {
1373 return MachOPlatformRuntimeState::get().registerJITDylib(
1374 Name: std::move(t&: Name), Header: HeaderAddr.toPtr<void *>());
1375 })
1376 .release();
1377}
1378
1379ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1380__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1381 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1382 ArgData, ArgSize,
1383 Handler: [](ExecutorAddr HeaderAddr) {
1384 return MachOPlatformRuntimeState::get().deregisterJITDylib(
1385 Header: HeaderAddr.toPtr<void *>());
1386 })
1387 .release();
1388}
1389
1390ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1391__orc_rt_macho_register_object_platform_sections(char *ArgData,
1392 size_t ArgSize) {
1393 return WrapperFunction<SPSError(SPSExecutorAddr,
1394 SPSOptional<SPSUnwindSectionInfo>,
1395 SPSMachOObjectPlatformSectionsMap)>::
1396 handle(ArgData, ArgSize,
1397 Handler: [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1398 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1399 &Secs) {
1400 return MachOPlatformRuntimeState::get()
1401 .registerObjectPlatformSections(HeaderAddr, UnwindInfo: std::move(t&: USI),
1402 Secs: std::move(t&: Secs));
1403 })
1404 .release();
1405}
1406
1407ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1408__orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
1409 using SymtabContainer = std::vector<
1410 std::tuple<ExecutorAddr, ExecutorAddr,
1411 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1412 return WrapperFunction<SPSError(
1413 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1414 SPSMachOExecutorSymbolFlags>>)>::
1415 handle(ArgData, ArgSize,
1416 Handler: [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1417 return MachOPlatformRuntimeState::get()
1418 .registerObjectSymbolTable(HeaderAddr, Entries: Symbols);
1419 })
1420 .release();
1421}
1422
1423ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1424__orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
1425 using SymtabContainer = std::vector<
1426 std::tuple<ExecutorAddr, ExecutorAddr,
1427 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1428 return WrapperFunction<SPSError(
1429 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1430 SPSMachOExecutorSymbolFlags>>)>::
1431 handle(ArgData, ArgSize,
1432 Handler: [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1433 return MachOPlatformRuntimeState::get()
1434 .deregisterObjectSymbolTable(HeaderAddr, Entries: Symbols);
1435 })
1436 .release();
1437}
1438
1439ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1440__orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1441 size_t ArgSize) {
1442 return WrapperFunction<SPSError(SPSExecutorAddr,
1443 SPSOptional<SPSUnwindSectionInfo>,
1444 SPSMachOObjectPlatformSectionsMap)>::
1445 handle(ArgData, ArgSize,
1446 Handler: [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1447 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1448 &Secs) {
1449 return MachOPlatformRuntimeState::get()
1450 .deregisterObjectPlatformSections(HeaderAddr, UnwindInfo: std::move(t&: USI),
1451 Secs: std::move(t&: Secs));
1452 })
1453 .release();
1454}
1455
1456ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1457__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1458 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1459 ArgData, ArgSize, Handler&: runWrapperFunctionCalls)
1460 .release();
1461}
1462
1463//------------------------------------------------------------------------------
1464// TLV support
1465//------------------------------------------------------------------------------
1466
1467ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1468 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1469 pthread_getspecific(key: D->Key));
1470 if (!TLVMgr) {
1471 TLVMgr = new MachOPlatformRuntimeTLVManager();
1472 if (pthread_setspecific(key: D->Key, pointer: TLVMgr)) {
1473 __orc_rt_log_error(ErrMsg: "Call to pthread_setspecific failed");
1474 return nullptr;
1475 }
1476 }
1477
1478 return TLVMgr->getInstance(
1479 ThreadData: reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1480}
1481
1482ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
1483__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1484 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1485 ArgData, ArgSize,
1486 Handler: []() -> Expected<uint64_t> {
1487 pthread_key_t Key;
1488 if (int Err = pthread_key_create(key: &Key, destr_function: destroyMachOTLVMgr)) {
1489 __orc_rt_log_error(ErrMsg: "Call to pthread_key_create failed");
1490 return make_error<StringError>(Args: strerror(errnum: Err));
1491 }
1492 return static_cast<uint64_t>(Key);
1493 })
1494 .release();
1495}
1496
1497//------------------------------------------------------------------------------
1498// cxa_atexit support
1499//------------------------------------------------------------------------------
1500
1501int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1502 void *dso_handle) {
1503 return MachOPlatformRuntimeState::get().registerAtExit(F: func, Arg: arg, DSOHandle: dso_handle);
1504}
1505
1506void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1507 MachOPlatformRuntimeState::get().runAtExits(DSOHandle: dso_handle);
1508}
1509
1510//------------------------------------------------------------------------------
1511// JIT'd dlfcn alternatives.
1512//------------------------------------------------------------------------------
1513
1514const char *__orc_rt_macho_jit_dlerror() {
1515 return MachOPlatformRuntimeState::get().dlerror();
1516}
1517
1518void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1519 return MachOPlatformRuntimeState::get().dlopen(Path: path, Mode: mode);
1520}
1521
1522int __orc_rt_macho_jit_dlupdate(void *dso_handle) {
1523 return MachOPlatformRuntimeState::get().dlupdate(DSOHandle: dso_handle);
1524}
1525
1526int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1527 return MachOPlatformRuntimeState::get().dlclose(DSOHandle: dso_handle);
1528}
1529
1530void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1531 return MachOPlatformRuntimeState::get().dlsym(DSOHandle: dso_handle, Symbol: symbol);
1532}
1533
1534//------------------------------------------------------------------------------
1535// MachO Run Program
1536//------------------------------------------------------------------------------
1537
1538ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1539 const char *EntrySymbolName,
1540 int argc, char *argv[]) {
1541 using MainTy = int (*)(int, char *[]);
1542
1543 void *H =
1544 __orc_rt_macho_jit_dlopen(path: JITDylibName, mode: orc_rt::macho::ORC_RT_RTLD_LAZY);
1545 if (!H) {
1546 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1547 return -1;
1548 }
1549
1550 auto *Main =
1551 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(dso_handle: H, symbol: EntrySymbolName));
1552
1553 if (!Main) {
1554 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1555 return -1;
1556 }
1557
1558 int Result = Main(argc, argv);
1559
1560 if (__orc_rt_macho_jit_dlclose(dso_handle: H) == -1)
1561 __orc_rt_log_error(ErrMsg: __orc_rt_macho_jit_dlerror());
1562
1563 return Result;
1564}
1565

source code of compiler-rt/lib/orc/macho_platform.cpp