1 //===- macho_platform.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the MachO runtime.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "macho_platform.h"
14 #include "common.h"
15 #include "debug.h"
16 #include "error.h"
17 #include "wrapper_function_utils.h"
18 
19 #include <algorithm>
20 #include <ios>
21 #include <map>
22 #include <mutex>
23 #include <sstream>
24 #include <unordered_map>
25 #include <unordered_set>
26 #include <vector>
27 
28 #define DEBUG_TYPE "macho_platform"
29 
30 using namespace __orc_rt;
31 using namespace __orc_rt::macho;
32 
33 // Declare function tags for functions in the JIT process.
34 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
35 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag)
36 
37 // Objective-C types.
38 struct objc_class;
39 struct objc_image_info;
40 struct objc_object;
41 struct objc_selector;
42 
43 using Class = objc_class *;
44 using id = objc_object *;
45 using SEL = objc_selector *;
46 
47 // Objective-C registration functions.
48 // These are weakly imported. If the Objective-C runtime has not been loaded
49 // then code containing Objective-C sections will generate an error.
50 extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT;
51 extern "C" Class objc_readClassPair(Class,
52                                     const objc_image_info *) ORC_RT_WEAK_IMPORT;
53 extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT;
54 
55 // Swift types.
56 class ProtocolRecord;
57 class ProtocolConformanceRecord;
58 class TypeMetadataRecord;
59 
60 extern "C" void
61 swift_registerProtocols(const ProtocolRecord *begin,
62                         const ProtocolRecord *end) ORC_RT_WEAK_IMPORT;
63 
64 extern "C" void swift_registerProtocolConformances(
65     const ProtocolConformanceRecord *begin,
66     const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT;
67 
68 extern "C" void swift_registerTypeMetadataRecords(
69     const TypeMetadataRecord *begin,
70     const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT;
71 
72 namespace {
73 
74 struct MachOJITDylibDepInfo {
75   bool Sealed = false;
76   std::vector<ExecutorAddr> DepHeaders;
77 };
78 
79 using MachOJITDylibDepInfoMap =
80     std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
81 
82 } // anonymous namespace
83 
84 namespace __orc_rt {
85 
86 using SPSMachOObjectPlatformSectionsMap =
87     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
88 
89 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
90 
91 using SPSMachOJITDylibDepInfoMap =
92     SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
93 
94 template <>
95 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
96 public:
97   static size_t size(const MachOJITDylibDepInfo &JDI) {
98     return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
99   }
100 
101   static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
102     return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
103                                                          JDI.DepHeaders);
104   }
105 
106   static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
107     return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
108                                                            JDI.DepHeaders);
109   }
110 };
111 
112 } // namespace __orc_rt
113 
114 namespace {
115 struct TLVDescriptor {
116   void *(*Thunk)(TLVDescriptor *) = nullptr;
117   unsigned long Key = 0;
118   unsigned long DataAddress = 0;
119 };
120 
121 class MachOPlatformRuntimeState {
122 private:
123   struct AtExitEntry {
124     void (*Func)(void *);
125     void *Arg;
126   };
127 
128   using AtExitsVector = std::vector<AtExitEntry>;
129 
130   struct JITDylibState {
131     std::string Name;
132     void *Header = nullptr;
133     bool Sealed = false;
134     size_t LinkedAgainstRefCount = 0;
135     size_t DlRefCount = 0;
136     std::vector<JITDylibState *> Deps;
137     AtExitsVector AtExits;
138     const objc_image_info *ObjCImageInfo = nullptr;
139     std::vector<span<void (*)()>> ModInitsSections;
140     std::vector<span<void (*)()>> ModInitsSectionsNew;
141     std::vector<span<uintptr_t>> ObjCClassListSections;
142     std::vector<span<uintptr_t>> ObjCClassListSectionsNew;
143     std::vector<span<uintptr_t>> ObjCSelRefsSections;
144     std::vector<span<uintptr_t>> ObjCSelRefsSectionsNew;
145     std::vector<span<char>> Swift5ProtoSections;
146     std::vector<span<char>> Swift5ProtoSectionsNew;
147     std::vector<span<char>> Swift5ProtosSections;
148     std::vector<span<char>> Swift5ProtosSectionsNew;
149     std::vector<span<char>> Swift5TypesSections;
150     std::vector<span<char>> Swift5TypesSectionsNew;
151 
152     bool referenced() const {
153       return LinkedAgainstRefCount != 0 || DlRefCount != 0;
154     }
155   };
156 
157 public:
158   static void initialize();
159   static MachOPlatformRuntimeState &get();
160   static void destroy();
161 
162   MachOPlatformRuntimeState() = default;
163 
164   // Delete copy and move constructors.
165   MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
166   MachOPlatformRuntimeState &
167   operator=(const MachOPlatformRuntimeState &) = delete;
168   MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
169   MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
170 
171   Error registerJITDylib(std::string Name, void *Header);
172   Error deregisterJITDylib(void *Header);
173   Error registerThreadDataSection(span<const char> ThreadDataSection);
174   Error deregisterThreadDataSection(span<const char> ThreadDataSection);
175   Error registerObjectPlatformSections(
176       ExecutorAddr HeaderAddr,
177       std::vector<std::pair<string_view, ExecutorAddrRange>> Secs);
178   Error deregisterObjectPlatformSections(
179       ExecutorAddr HeaderAddr,
180       std::vector<std::pair<string_view, ExecutorAddrRange>> Secs);
181 
182   const char *dlerror();
183   void *dlopen(string_view Name, int Mode);
184   int dlclose(void *DSOHandle);
185   void *dlsym(void *DSOHandle, string_view Symbol);
186 
187   int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
188   void runAtExits(JITDylibState &JDS);
189   void runAtExits(void *DSOHandle);
190 
191   /// Returns the base address of the section containing ThreadData.
192   Expected<std::pair<const char *, size_t>>
193   getThreadDataSectionFor(const char *ThreadData);
194 
195 private:
196   JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
197   JITDylibState *getJITDylibStateByName(string_view Path);
198 
199   Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
200                                                 string_view Symbol);
201 
202   static Error registerObjCSelectors(JITDylibState &JDS);
203   static Error registerObjCClasses(JITDylibState &JDS);
204   static Error registerSwift5Protocols(JITDylibState &JDS);
205   static Error registerSwift5ProtocolConformances(JITDylibState &JDS);
206   static Error registerSwift5Types(JITDylibState &JDS);
207   static Error runModInits(JITDylibState &JDS);
208 
209   Expected<void *> dlopenImpl(string_view Path, int Mode);
210   Error dlopenFull(JITDylibState &JDS);
211   Error dlopenInitialize(JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
212 
213   Error dlcloseImpl(void *DSOHandle);
214   Error dlcloseDeinitialize(JITDylibState &JDS);
215 
216   static MachOPlatformRuntimeState *MOPS;
217 
218   // FIXME: Move to thread-state.
219   std::string DLFcnError;
220 
221   std::recursive_mutex JDStatesMutex;
222   std::unordered_map<void *, JITDylibState> JDStates;
223   std::unordered_map<string_view, void *> JDNameToHeader;
224 
225   std::mutex ThreadDataSectionsMutex;
226   std::map<const char *, size_t> ThreadDataSections;
227 };
228 
229 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
230 
231 void MachOPlatformRuntimeState::initialize() {
232   assert(!MOPS && "MachOPlatformRuntimeState should be null");
233   MOPS = new MachOPlatformRuntimeState();
234 }
235 
236 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
237   assert(MOPS && "MachOPlatformRuntimeState not initialized");
238   return *MOPS;
239 }
240 
241 void MachOPlatformRuntimeState::destroy() {
242   assert(MOPS && "MachOPlatformRuntimeState not initialized");
243   delete MOPS;
244 }
245 
246 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
247                                                   void *Header) {
248   ORC_RT_DEBUG({
249     printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
250   });
251   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
252   if (JDStates.count(Header)) {
253     std::ostringstream ErrStream;
254     ErrStream << "Duplicate JITDylib registration for header " << Header
255               << " (name = " << Name << ")";
256     return make_error<StringError>(ErrStream.str());
257   }
258   if (JDNameToHeader.count(Name)) {
259     std::ostringstream ErrStream;
260     ErrStream << "Duplicate JITDylib registration for header " << Header
261               << " (header = " << Header << ")";
262     return make_error<StringError>(ErrStream.str());
263   }
264 
265   auto &JDS = JDStates[Header];
266   JDS.Name = std::move(Name);
267   JDS.Header = Header;
268   JDNameToHeader[JDS.Name] = Header;
269   return Error::success();
270 }
271 
272 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
273   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
274   auto I = JDStates.find(Header);
275   if (I == JDStates.end()) {
276     std::ostringstream ErrStream;
277     ErrStream << "Attempted to deregister unrecognized header " << Header;
278     return make_error<StringError>(ErrStream.str());
279   }
280 
281   // Remove std::string construction once we can use C++20.
282   auto J = JDNameToHeader.find(
283       std::string(I->second.Name.data(), I->second.Name.size()));
284   assert(J != JDNameToHeader.end() &&
285          "Missing JDNameToHeader entry for JITDylib");
286 
287   ORC_RT_DEBUG({
288     printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
289              Header);
290   });
291 
292   JDNameToHeader.erase(J);
293   JDStates.erase(I);
294   return Error::success();
295 }
296 
297 Error MachOPlatformRuntimeState::registerThreadDataSection(
298     span<const char> ThreadDataSection) {
299   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
300   auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
301   if (I != ThreadDataSections.begin()) {
302     auto J = std::prev(I);
303     if (J->first + J->second > ThreadDataSection.data())
304       return make_error<StringError>("Overlapping __thread_data sections");
305   }
306   ThreadDataSections.insert(
307       I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
308   return Error::success();
309 }
310 
311 Error MachOPlatformRuntimeState::deregisterThreadDataSection(
312     span<const char> ThreadDataSection) {
313   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
314   auto I = ThreadDataSections.find(ThreadDataSection.data());
315   if (I == ThreadDataSections.end())
316     return make_error<StringError>("Attempt to deregister unknown thread data "
317                                    "section");
318   ThreadDataSections.erase(I);
319   return Error::success();
320 }
321 
322 Error MachOPlatformRuntimeState::registerObjectPlatformSections(
323     ExecutorAddr HeaderAddr,
324     std::vector<std::pair<string_view, ExecutorAddrRange>> Secs) {
325   ORC_RT_DEBUG({
326     printdbg("MachOPlatform: Registering object sections for %p.\n",
327              HeaderAddr.toPtr<void *>());
328   });
329 
330   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
331   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
332   if (!JDS) {
333     std::ostringstream ErrStream;
334     ErrStream << "Could not register object platform sections for "
335                  "unrecognized header "
336               << HeaderAddr.toPtr<void *>();
337     return make_error<StringError>(ErrStream.str());
338   }
339 
340   for (auto &KV : Secs) {
341     // FIXME: Validate section ranges?
342     if (KV.first == "__DATA,__thread_data") {
343       if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
344         return Err;
345     } else if (KV.first == "__DATA,__objc_selrefs")
346       JDS->ObjCSelRefsSectionsNew.push_back(KV.second.toSpan<uintptr_t>());
347     else if (KV.first == "__DATA,__objc_classlist")
348       JDS->ObjCClassListSectionsNew.push_back(KV.second.toSpan<uintptr_t>());
349     else if (KV.first == "__TEXT,__swift5_protos")
350       JDS->Swift5ProtosSectionsNew.push_back(KV.second.toSpan<char>());
351     else if (KV.first == "__TEXT,__swift5_proto")
352       JDS->Swift5ProtoSectionsNew.push_back(KV.second.toSpan<char>());
353     else if (KV.first == "__TEXT,__swift5_types")
354       JDS->Swift5TypesSectionsNew.push_back(KV.second.toSpan<char>());
355     else if (KV.first == "__DATA,__mod_init_func")
356       JDS->ModInitsSectionsNew.push_back(KV.second.toSpan<void (*)()>());
357     else {
358       // Should this be a warning instead?
359       return make_error<StringError>(
360           "Encountered unexpected section " +
361           std::string(KV.first.data(), KV.first.size()) +
362           " while registering object platform sections");
363     }
364   }
365 
366   return Error::success();
367 }
368 
369 // Remove the given range from the given vector if present.
370 // Returns true if the range was removed, false otherwise.
371 template <typename T>
372 bool removeIfPresent(std::vector<span<T>> &V, ExecutorAddrRange R) {
373   auto RI = std::find_if(
374       V.rbegin(), V.rend(),
375       [RS = R.toSpan<T>()](const span<T> &E) { return E.data() == RS.data(); });
376   if (RI != V.rend()) {
377     V.erase(std::next(RI).base());
378     return true;
379   }
380   return false;
381 }
382 
383 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
384     ExecutorAddr HeaderAddr,
385     std::vector<std::pair<string_view, ExecutorAddrRange>> Secs) {
386   // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
387   // TODO: Add a JITDylib prepare-for-teardown operation that clears all
388   //       registered sections, causing this function to take the fast-path.
389   ORC_RT_DEBUG({
390     printdbg("MachOPlatform: Registering object sections for %p.\n",
391              HeaderAddr.toPtr<void *>());
392   });
393 
394   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
395   auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
396   if (!JDS) {
397     std::ostringstream ErrStream;
398     ErrStream << "Could not register object platform sections for unrecognized "
399                  "header "
400               << HeaderAddr.toPtr<void *>();
401     return make_error<StringError>(ErrStream.str());
402   }
403 
404   // FIXME: Implement faster-path by returning immediately if JDS is being
405   // torn down entirely?
406 
407   for (auto &KV : Secs) {
408     // FIXME: Validate section ranges?
409     if (KV.first == "__DATA,__thread_data") {
410       if (auto Err =
411               deregisterThreadDataSection(KV.second.toSpan<const char>()))
412         return Err;
413     } else if (KV.first == "__DATA,__objc_selrefs") {
414       if (!removeIfPresent(JDS->ObjCSelRefsSections, KV.second))
415         removeIfPresent(JDS->ObjCSelRefsSectionsNew, KV.second);
416     } else if (KV.first == "__DATA,__objc_classlist") {
417       if (!removeIfPresent(JDS->ObjCClassListSections, KV.second))
418         removeIfPresent(JDS->ObjCClassListSectionsNew, KV.second);
419     } else if (KV.first == "__TEXT,__swift5_protos") {
420       if (!removeIfPresent(JDS->Swift5ProtosSections, KV.second))
421         removeIfPresent(JDS->Swift5ProtosSectionsNew, KV.second);
422     } else if (KV.first == "__TEXT,__swift5_proto") {
423       if (!removeIfPresent(JDS->Swift5ProtoSections, KV.second))
424         removeIfPresent(JDS->Swift5ProtoSectionsNew, KV.second);
425     } else if (KV.first == "__TEXT,__swift5_types") {
426       if (!removeIfPresent(JDS->Swift5TypesSections, KV.second))
427         removeIfPresent(JDS->Swift5TypesSectionsNew, KV.second);
428     } else if (KV.first == "__DATA,__mod_init_func") {
429       if (!removeIfPresent(JDS->ModInitsSections, KV.second))
430         removeIfPresent(JDS->ModInitsSectionsNew, KV.second);
431     } else {
432       // Should this be a warning instead?
433       return make_error<StringError>(
434           "Encountered unexpected section " +
435           std::string(KV.first.data(), KV.first.size()) +
436           " while deregistering object platform sections");
437     }
438   }
439   return Error::success();
440 }
441 
442 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
443 
444 void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) {
445   ORC_RT_DEBUG({
446     std::string S(Path.data(), Path.size());
447     printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
448   });
449   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
450   if (auto H = dlopenImpl(Path, Mode))
451     return *H;
452   else {
453     // FIXME: Make dlerror thread safe.
454     DLFcnError = toString(H.takeError());
455     return nullptr;
456   }
457 }
458 
459 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
460   ORC_RT_DEBUG({
461     auto *JDS = getJITDylibStateByHeader(DSOHandle);
462     std::string DylibName;
463     if (JDS) {
464       std::string S;
465       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
466     } else
467       printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
468                "invalid handle");
469   });
470   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
471   if (auto Err = dlcloseImpl(DSOHandle)) {
472     // FIXME: Make dlerror thread safe.
473     DLFcnError = toString(std::move(Err));
474     return -1;
475   }
476   return 0;
477 }
478 
479 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) {
480   auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
481   if (!Addr) {
482     DLFcnError = toString(Addr.takeError());
483     return 0;
484   }
485 
486   return Addr->toPtr<void *>();
487 }
488 
489 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
490                                               void *DSOHandle) {
491   // FIXME: Handle out-of-memory errors, returning -1 if OOM.
492   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
493   auto *JDS = getJITDylibStateByHeader(DSOHandle);
494   if (!JDS) {
495     ORC_RT_DEBUG({
496       printdbg("MachOPlatformRuntimeState::registerAtExit called with "
497                "unrecognized dso handle %p\n",
498                DSOHandle);
499     });
500     return -1;
501   }
502   JDS->AtExits.push_back({F, Arg});
503   return 0;
504 }
505 
506 void MachOPlatformRuntimeState::runAtExits(JITDylibState &JDS) {
507   while (!JDS.AtExits.empty()) {
508     auto &AE = JDS.AtExits.back();
509     AE.Func(AE.Arg);
510     JDS.AtExits.pop_back();
511   }
512 }
513 
514 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
515   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
516   auto *JDS = getJITDylibStateByHeader(DSOHandle);
517   ORC_RT_DEBUG({
518     printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
519              "dso_handle %p\n",
520              DSOHandle);
521   });
522   if (JDS)
523     runAtExits(*JDS);
524 }
525 
526 Expected<std::pair<const char *, size_t>>
527 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
528   std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
529   auto I = ThreadDataSections.upper_bound(ThreadData);
530   // Check that we have a valid entry covering this address.
531   if (I == ThreadDataSections.begin())
532     return make_error<StringError>("No thread local data section for key");
533   I = std::prev(I);
534   if (ThreadData >= I->first + I->second)
535     return make_error<StringError>("No thread local data section for key");
536   return *I;
537 }
538 
539 MachOPlatformRuntimeState::JITDylibState *
540 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
541   auto I = JDStates.find(DSOHandle);
542   if (I == JDStates.end()) {
543     I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
544     I->second.Header = DSOHandle;
545   }
546   return &I->second;
547 }
548 
549 MachOPlatformRuntimeState::JITDylibState *
550 MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) {
551   // FIXME: Avoid creating string once we have C++20.
552   auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
553   if (I != JDNameToHeader.end())
554     return getJITDylibStateByHeader(I->second);
555   return nullptr;
556 }
557 
558 Expected<ExecutorAddr>
559 MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
560                                                   string_view Sym) {
561   Expected<ExecutorAddr> Result((ExecutorAddr()));
562   if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
563           SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag,
564                                              Result,
565                                              ExecutorAddr::fromPtr(DSOHandle),
566                                              Sym))
567     return std::move(Err);
568   return Result;
569 }
570 
571 template <typename T>
572 static void moveAppendSections(std::vector<span<T>> &Dst,
573                                std::vector<span<T>> &Src) {
574   if (Dst.empty()) {
575     Dst = std::move(Src);
576     return;
577   }
578 
579   Dst.reserve(Dst.size() + Src.size());
580   std::copy(Src.begin(), Src.end(), std::back_inserter(Dst));
581   Src.clear();
582 }
583 
584 Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) {
585 
586   if (JDS.ObjCSelRefsSectionsNew.empty())
587     return Error::success();
588 
589   if (ORC_RT_UNLIKELY(!sel_registerName))
590     return make_error<StringError>("sel_registerName is not available");
591 
592   for (const auto &ObjCSelRefs : JDS.ObjCSelRefsSectionsNew) {
593     for (uintptr_t &SelEntry : ObjCSelRefs) {
594       const char *SelName = reinterpret_cast<const char *>(SelEntry);
595       auto Sel = sel_registerName(SelName);
596       *reinterpret_cast<SEL *>(&SelEntry) = Sel;
597     }
598   }
599 
600   moveAppendSections(JDS.ObjCSelRefsSections, JDS.ObjCSelRefsSectionsNew);
601   return Error::success();
602 }
603 
604 Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) {
605 
606   if (JDS.ObjCClassListSectionsNew.empty())
607     return Error::success();
608 
609   if (ORC_RT_UNLIKELY(!objc_msgSend))
610     return make_error<StringError>("objc_msgSend is not available");
611   if (ORC_RT_UNLIKELY(!objc_readClassPair))
612     return make_error<StringError>("objc_readClassPair is not available");
613 
614   struct ObjCClassCompiled {
615     void *Metaclass;
616     void *Parent;
617     void *Cache1;
618     void *Cache2;
619     void *Data;
620   };
621 
622   auto ClassSelector = sel_registerName("class");
623 
624   for (const auto &ObjCClassList : JDS.ObjCClassListSectionsNew) {
625     for (uintptr_t ClassPtr : ObjCClassList) {
626       auto *Cls = reinterpret_cast<Class>(ClassPtr);
627       auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr);
628       objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector);
629       auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo);
630 
631       // FIXME: Improve diagnostic by reporting the failed class's name.
632       if (Registered != Cls)
633         return make_error<StringError>("Unable to register Objective-C class");
634     }
635   }
636 
637   moveAppendSections(JDS.ObjCClassListSections, JDS.ObjCClassListSectionsNew);
638   return Error::success();
639 }
640 
641 Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) {
642 
643   if (JDS.Swift5ProtosSectionsNew.empty())
644     return Error::success();
645 
646   if (ORC_RT_UNLIKELY(!swift_registerProtocols))
647     return make_error<StringError>("swift_registerProtocols is not available");
648 
649   for (const auto &Swift5Protocols : JDS.Swift5ProtoSectionsNew)
650     swift_registerProtocols(
651         reinterpret_cast<const ProtocolRecord *>(Swift5Protocols.data()),
652         reinterpret_cast<const ProtocolRecord *>(Swift5Protocols.data() +
653                                                  Swift5Protocols.size()));
654 
655   moveAppendSections(JDS.Swift5ProtoSections, JDS.Swift5ProtoSectionsNew);
656   return Error::success();
657 }
658 
659 Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances(
660     JITDylibState &JDS) {
661 
662   if (JDS.Swift5ProtosSectionsNew.empty())
663     return Error::success();
664 
665   if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances))
666     return make_error<StringError>(
667         "swift_registerProtocolConformances is not available");
668 
669   for (const auto &ProtoConfSec : JDS.Swift5ProtosSectionsNew)
670     swift_registerProtocolConformances(
671         reinterpret_cast<const ProtocolConformanceRecord *>(
672             ProtoConfSec.data()),
673         reinterpret_cast<const ProtocolConformanceRecord *>(
674             ProtoConfSec.data() + ProtoConfSec.size()));
675 
676   moveAppendSections(JDS.Swift5ProtosSections, JDS.Swift5ProtosSectionsNew);
677   return Error::success();
678 }
679 
680 Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) {
681 
682   if (JDS.Swift5TypesSectionsNew.empty())
683     return Error::success();
684 
685   if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords))
686     return make_error<StringError>(
687         "swift_registerTypeMetadataRecords is not available");
688 
689   for (const auto &TypeSec : JDS.Swift5TypesSectionsNew)
690     swift_registerTypeMetadataRecords(
691         reinterpret_cast<const TypeMetadataRecord *>(TypeSec.data()),
692         reinterpret_cast<const TypeMetadataRecord *>(TypeSec.data() +
693                                                      TypeSec.size()));
694 
695   moveAppendSections(JDS.Swift5TypesSections, JDS.Swift5TypesSectionsNew);
696   return Error::success();
697 }
698 
699 Error MachOPlatformRuntimeState::runModInits(JITDylibState &JDS) {
700 
701   for (const auto &ModInits : JDS.ModInitsSectionsNew) {
702     for (void (*Init)() : ModInits)
703       (*Init)();
704   }
705 
706   moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew);
707   return Error::success();
708 }
709 
710 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(string_view Path,
711                                                        int Mode) {
712   // Try to find JITDylib state by name.
713   auto *JDS = getJITDylibStateByName(Path);
714 
715   if (!JDS)
716     return make_error<StringError>("No registered JTIDylib for path " +
717                                    std::string(Path.data(), Path.size()));
718 
719   // If this JITDylib is unsealed, or this is the first dlopen then run
720   // full dlopen path (update deps, push and run initializers, update ref
721   // counts on all JITDylibs in the dep tree).
722   if (!JDS->referenced() || !JDS->Sealed) {
723     if (auto Err = dlopenFull(*JDS))
724       return std::move(Err);
725   }
726 
727   // Bump the ref-count on this dylib.
728   ++JDS->DlRefCount;
729 
730   // Return the header address.
731   return JDS->Header;
732 }
733 
734 Error MachOPlatformRuntimeState::dlopenFull(JITDylibState &JDS) {
735   // Call back to the JIT to push the initializers.
736   Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
737   if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
738           SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag,
739                                   DepInfo, ExecutorAddr::fromPtr(JDS.Header)))
740     return Err;
741   if (!DepInfo)
742     return DepInfo.takeError();
743 
744   if (auto Err = dlopenInitialize(JDS, *DepInfo))
745     return Err;
746 
747   if (!DepInfo->empty()) {
748     ORC_RT_DEBUG({
749       printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
750                JDS.Name.c_str());
751     });
752     std::ostringstream ErrStream;
753     ErrStream << "Encountered unrecognized dep-info key headers "
754                  "while processing dlopen of "
755               << JDS.Name;
756     return make_error<StringError>(ErrStream.str());
757   }
758 
759   return Error::success();
760 }
761 
762 Error MachOPlatformRuntimeState::dlopenInitialize(
763     JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo) {
764   ORC_RT_DEBUG({
765     printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
766              JDS.Name.c_str());
767   });
768 
769   // If the header is not present in the dep map then assume that we
770   // already processed it earlier in the dlopenInitialize traversal and
771   // return.
772   // TODO: Keep a visited set instead so that we can error out on missing
773   //       entries?
774   auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
775   if (I == DepInfo.end())
776     return Error::success();
777 
778   auto DI = std::move(I->second);
779   DepInfo.erase(I);
780 
781   // We don't need to re-initialize sealed JITDylibs that have already been
782   // initialized. Just check that their dep-map entry is empty as expected.
783   if (JDS.Sealed) {
784     if (!DI.DepHeaders.empty()) {
785       std::ostringstream ErrStream;
786       ErrStream << "Sealed JITDylib " << JDS.Header
787                 << " already has registered dependencies";
788       return make_error<StringError>(ErrStream.str());
789     }
790     if (JDS.referenced())
791       return Error::success();
792   } else
793     JDS.Sealed = DI.Sealed;
794 
795   // This is an unsealed or newly sealed JITDylib. Run initializers.
796   std::vector<JITDylibState *> OldDeps;
797   std::swap(JDS.Deps, OldDeps);
798   JDS.Deps.reserve(DI.DepHeaders.size());
799   for (auto DepHeaderAddr : DI.DepHeaders) {
800     auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
801     if (!DepJDS) {
802       std::ostringstream ErrStream;
803       ErrStream << "Encountered unrecognized dep header "
804                 << DepHeaderAddr.toPtr<void *>() << " while initializing "
805                 << JDS.Name;
806       return make_error<StringError>(ErrStream.str());
807     }
808     ++DepJDS->LinkedAgainstRefCount;
809     if (auto Err = dlopenInitialize(*DepJDS, DepInfo))
810       return Err;
811   }
812 
813   // Initialize this JITDylib.
814   if (auto Err = registerObjCSelectors(JDS))
815     return Err;
816   if (auto Err = registerObjCClasses(JDS))
817     return Err;
818   if (auto Err = registerSwift5Protocols(JDS))
819     return Err;
820   if (auto Err = registerSwift5ProtocolConformances(JDS))
821     return Err;
822   if (auto Err = registerSwift5Types(JDS))
823     return Err;
824   if (auto Err = runModInits(JDS))
825     return Err;
826 
827   // Decrement old deps.
828   // FIXME: We should probably continue and just report deinitialize errors
829   // here.
830   for (auto *DepJDS : OldDeps) {
831     --DepJDS->LinkedAgainstRefCount;
832     if (!DepJDS->referenced())
833       if (auto Err = dlcloseDeinitialize(*DepJDS))
834         return Err;
835   }
836 
837   return Error::success();
838 }
839 
840 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
841   // Try to find JITDylib state by header.
842   auto *JDS = getJITDylibStateByHeader(DSOHandle);
843 
844   if (!JDS) {
845     std::ostringstream ErrStream;
846     ErrStream << "No registered JITDylib for " << DSOHandle;
847     return make_error<StringError>(ErrStream.str());
848   }
849 
850   // Bump the ref-count.
851   --JDS->DlRefCount;
852 
853   if (!JDS->referenced())
854     return dlcloseDeinitialize(*JDS);
855 
856   return Error::success();
857 }
858 
859 Error MachOPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) {
860 
861   ORC_RT_DEBUG({
862     printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
863              JDS.Name.c_str());
864   });
865 
866   runAtExits(JDS);
867 
868   // Reset mod-inits
869   moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew);
870   JDS.ModInitsSectionsNew = std::move(JDS.ModInitsSections);
871 
872   // Deinitialize any dependencies.
873   for (auto *DepJDS : JDS.Deps) {
874     --DepJDS->LinkedAgainstRefCount;
875     if (!DepJDS->referenced())
876       if (auto Err = dlcloseDeinitialize(*DepJDS))
877         return Err;
878   }
879 
880   return Error::success();
881 }
882 
883 class MachOPlatformRuntimeTLVManager {
884 public:
885   void *getInstance(const char *ThreadData);
886 
887 private:
888   std::unordered_map<const char *, char *> Instances;
889   std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
890 };
891 
892 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
893   auto I = Instances.find(ThreadData);
894   if (I != Instances.end())
895     return I->second;
896 
897   auto TDS =
898       MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
899   if (!TDS) {
900     __orc_rt_log_error(toString(TDS.takeError()).c_str());
901     return nullptr;
902   }
903 
904   auto &Allocated = AllocatedSections[TDS->first];
905   if (!Allocated) {
906     Allocated = std::make_unique<char[]>(TDS->second);
907     memcpy(Allocated.get(), TDS->first, TDS->second);
908   }
909 
910   size_t ThreadDataDelta = ThreadData - TDS->first;
911   assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
912 
913   char *Instance = Allocated.get() + ThreadDataDelta;
914   Instances[ThreadData] = Instance;
915   return Instance;
916 }
917 
918 void destroyMachOTLVMgr(void *MachOTLVMgr) {
919   delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
920 }
921 
922 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
923   for (auto &WFC : WFCs)
924     if (auto Err = WFC.runWithSPSRet<void>())
925       return Err;
926   return Error::success();
927 }
928 
929 } // end anonymous namespace
930 
931 //------------------------------------------------------------------------------
932 //                             JIT entry points
933 //------------------------------------------------------------------------------
934 
935 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
936 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
937   MachOPlatformRuntimeState::initialize();
938   return WrapperFunctionResult().release();
939 }
940 
941 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
942 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
943   MachOPlatformRuntimeState::destroy();
944   return WrapperFunctionResult().release();
945 }
946 
947 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
948 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
949   return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
950              ArgData, ArgSize,
951              [](std::string &Name, ExecutorAddr HeaderAddr) {
952                return MachOPlatformRuntimeState::get().registerJITDylib(
953                    std::move(Name), HeaderAddr.toPtr<void *>());
954              })
955       .release();
956 }
957 
958 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
959 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
960   return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
961              ArgData, ArgSize,
962              [](ExecutorAddr HeaderAddr) {
963                return MachOPlatformRuntimeState::get().deregisterJITDylib(
964                    HeaderAddr.toPtr<void *>());
965              })
966       .release();
967 }
968 
969 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
970 __orc_rt_macho_register_object_platform_sections(char *ArgData,
971                                                  size_t ArgSize) {
972   return WrapperFunction<SPSError(SPSExecutorAddr,
973                                   SPSMachOObjectPlatformSectionsMap)>::
974       handle(ArgData, ArgSize,
975              [](ExecutorAddr HeaderAddr,
976                 std::vector<std::pair<string_view, ExecutorAddrRange>> &Secs) {
977                return MachOPlatformRuntimeState::get()
978                    .registerObjectPlatformSections(HeaderAddr, std::move(Secs));
979              })
980           .release();
981 }
982 
983 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
984 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
985                                                    size_t ArgSize) {
986   return WrapperFunction<SPSError(SPSExecutorAddr,
987                                   SPSMachOObjectPlatformSectionsMap)>::
988       handle(ArgData, ArgSize,
989              [](ExecutorAddr HeaderAddr,
990                 std::vector<std::pair<string_view, ExecutorAddrRange>> &Secs) {
991                return MachOPlatformRuntimeState::get()
992                    .deregisterObjectPlatformSections(HeaderAddr,
993                                                      std::move(Secs));
994              })
995           .release();
996 }
997 
998 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
999 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1000   return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1001              ArgData, ArgSize, runWrapperFunctionCalls)
1002       .release();
1003 }
1004 
1005 //------------------------------------------------------------------------------
1006 //                            TLV support
1007 //------------------------------------------------------------------------------
1008 
1009 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1010   auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1011       pthread_getspecific(D->Key));
1012   if (!TLVMgr) {
1013     TLVMgr = new MachOPlatformRuntimeTLVManager();
1014     if (pthread_setspecific(D->Key, TLVMgr)) {
1015       __orc_rt_log_error("Call to pthread_setspecific failed");
1016       return nullptr;
1017     }
1018   }
1019 
1020   return TLVMgr->getInstance(
1021       reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1022 }
1023 
1024 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult
1025 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1026   return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1027              ArgData, ArgSize,
1028              []() -> Expected<uint64_t> {
1029                pthread_key_t Key;
1030                if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
1031                  __orc_rt_log_error("Call to pthread_key_create failed");
1032                  return make_error<StringError>(strerror(Err));
1033                }
1034                return static_cast<uint64_t>(Key);
1035              })
1036       .release();
1037 }
1038 
1039 //------------------------------------------------------------------------------
1040 //                           cxa_atexit support
1041 //------------------------------------------------------------------------------
1042 
1043 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1044                               void *dso_handle) {
1045   return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
1046 }
1047 
1048 void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1049   MachOPlatformRuntimeState::get().runAtExits(dso_handle);
1050 }
1051 
1052 //------------------------------------------------------------------------------
1053 //                        JIT'd dlfcn alternatives.
1054 //------------------------------------------------------------------------------
1055 
1056 const char *__orc_rt_macho_jit_dlerror() {
1057   return MachOPlatformRuntimeState::get().dlerror();
1058 }
1059 
1060 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1061   return MachOPlatformRuntimeState::get().dlopen(path, mode);
1062 }
1063 
1064 int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1065   return MachOPlatformRuntimeState::get().dlclose(dso_handle);
1066 }
1067 
1068 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1069   return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
1070 }
1071 
1072 //------------------------------------------------------------------------------
1073 //                             MachO Run Program
1074 //------------------------------------------------------------------------------
1075 
1076 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1077                                                     const char *EntrySymbolName,
1078                                                     int argc, char *argv[]) {
1079   using MainTy = int (*)(int, char *[]);
1080 
1081   void *H = __orc_rt_macho_jit_dlopen(JITDylibName,
1082                                       __orc_rt::macho::ORC_RT_RTLD_LAZY);
1083   if (!H) {
1084     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1085     return -1;
1086   }
1087 
1088   auto *Main =
1089       reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
1090 
1091   if (!Main) {
1092     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1093     return -1;
1094   }
1095 
1096   int Result = Main(argc, argv);
1097 
1098   if (__orc_rt_macho_jit_dlclose(H) == -1)
1099     __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1100 
1101   return Result;
1102 }
1103