1 //===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
10
11 #include "llvm/BinaryFormat/MachO.h"
12 #include "llvm/ExecutionEngine/JITLink/MachO.h"
13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
14 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
15 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
18 #include "llvm/ExecutionEngine/Orc/MachOBuilder.h"
19 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
20 #include "llvm/Support/BinaryByteStream.h"
21 #include "llvm/Support/Debug.h"
22 #include <optional>
23
24 #define DEBUG_TYPE "orc"
25
26 using namespace llvm;
27 using namespace llvm::orc;
28 using namespace llvm::orc::shared;
29
30 namespace llvm {
31 namespace orc {
32 namespace shared {
33
34 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
35 using SPSMachOJITDylibDepInfoMap =
36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
37
38 class SPSMachOExecutorSymbolFlags;
39
40 template <>
41 class SPSSerializationTraits<SPSMachOJITDylibDepInfo,
42 MachOPlatform::MachOJITDylibDepInfo> {
43 public:
size(const MachOPlatform::MachOJITDylibDepInfo & DDI)44 static size_t size(const MachOPlatform::MachOJITDylibDepInfo &DDI) {
45 return SPSMachOJITDylibDepInfo::AsArgList::size(DDI.Sealed, DDI.DepHeaders);
46 }
47
serialize(SPSOutputBuffer & OB,const MachOPlatform::MachOJITDylibDepInfo & DDI)48 static bool serialize(SPSOutputBuffer &OB,
49 const MachOPlatform::MachOJITDylibDepInfo &DDI) {
50 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, DDI.Sealed,
51 DDI.DepHeaders);
52 }
53
deserialize(SPSInputBuffer & IB,MachOPlatform::MachOJITDylibDepInfo & DDI)54 static bool deserialize(SPSInputBuffer &IB,
55 MachOPlatform::MachOJITDylibDepInfo &DDI) {
56 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.Sealed,
57 DDI.DepHeaders);
58 }
59 };
60
61 template <>
62 class SPSSerializationTraits<SPSMachOExecutorSymbolFlags,
63 MachOPlatform::MachOExecutorSymbolFlags> {
64 private:
65 using UT = std::underlying_type_t<MachOPlatform::MachOExecutorSymbolFlags>;
66
67 public:
size(const MachOPlatform::MachOExecutorSymbolFlags & SF)68 static size_t size(const MachOPlatform::MachOExecutorSymbolFlags &SF) {
69 return sizeof(UT);
70 }
71
serialize(SPSOutputBuffer & OB,const MachOPlatform::MachOExecutorSymbolFlags & SF)72 static bool serialize(SPSOutputBuffer &OB,
73 const MachOPlatform::MachOExecutorSymbolFlags &SF) {
74 return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF));
75 }
76
deserialize(SPSInputBuffer & IB,MachOPlatform::MachOExecutorSymbolFlags & SF)77 static bool deserialize(SPSInputBuffer &IB,
78 MachOPlatform::MachOExecutorSymbolFlags &SF) {
79 UT Tmp;
80 if (!SPSArgList<UT>::deserialize(IB, Tmp))
81 return false;
82 SF = static_cast<MachOPlatform::MachOExecutorSymbolFlags>(Tmp);
83 return true;
84 }
85 };
86
87 } // namespace shared
88 } // namespace orc
89 } // namespace llvm
90
91 namespace {
92
93 using SPSRegisterSymbolsArgs =
94 SPSArgList<SPSExecutorAddr,
95 SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
96 SPSMachOExecutorSymbolFlags>>>;
97
createPlatformGraph(MachOPlatform & MOP,std::string Name)98 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP,
99 std::string Name) {
100 unsigned PointerSize;
101 llvm::endianness Endianness;
102 const auto &TT = MOP.getExecutionSession().getTargetTriple();
103
104 switch (TT.getArch()) {
105 case Triple::aarch64:
106 case Triple::x86_64:
107 PointerSize = 8;
108 Endianness = llvm::endianness::little;
109 break;
110 default:
111 llvm_unreachable("Unrecognized architecture");
112 }
113
114 return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize,
115 Endianness,
116 jitlink::getGenericEdgeKindName);
117 }
118
119 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
120 class MachOPlatformCompleteBootstrapMaterializationUnit
121 : public MaterializationUnit {
122 public:
123 using SymbolTableVector =
124 SmallVector<std::tuple<ExecutorAddr, ExecutorAddr,
125 MachOPlatform::MachOExecutorSymbolFlags>>;
126
MachOPlatformCompleteBootstrapMaterializationUnit(MachOPlatform & MOP,StringRef PlatformJDName,SymbolStringPtr CompleteBootstrapSymbol,SymbolTableVector SymTab,shared::AllocActions DeferredAAs,ExecutorAddr MachOHeaderAddr,ExecutorAddr PlatformBootstrap,ExecutorAddr PlatformShutdown,ExecutorAddr RegisterJITDylib,ExecutorAddr DeregisterJITDylib,ExecutorAddr RegisterObjectSymbolTable,ExecutorAddr DeregisterObjectSymbolTable)127 MachOPlatformCompleteBootstrapMaterializationUnit(
128 MachOPlatform &MOP, StringRef PlatformJDName,
129 SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab,
130 shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr,
131 ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown,
132 ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib,
133 ExecutorAddr RegisterObjectSymbolTable,
134 ExecutorAddr DeregisterObjectSymbolTable)
135 : MaterializationUnit(
136 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
137 MOP(MOP), PlatformJDName(PlatformJDName),
138 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
139 SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)),
140 MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap),
141 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
142 DeregisterJITDylib(DeregisterJITDylib),
143 RegisterObjectSymbolTable(RegisterObjectSymbolTable),
144 DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {}
145
getName() const146 StringRef getName() const override {
147 return "MachOPlatformCompleteBootstrap";
148 }
149
materialize(std::unique_ptr<MaterializationResponsibility> R)150 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
151 using namespace jitlink;
152 auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
153 auto &PlaceholderSection =
154 G->createSection("__orc_rt_cplt_bs", MemProt::Read);
155 auto &PlaceholderBlock =
156 G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
157 G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
158 Linkage::Strong, Scope::Hidden, false, true);
159
160 // Reserve space for the stolen actions, plus two extras.
161 G->allocActions().reserve(DeferredAAs.size() + 3);
162
163 // 1. Bootstrap the platform support code.
164 G->allocActions().push_back(
165 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(PlatformBootstrap)),
166 cantFail(
167 WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
168
169 // 2. Register the platform JITDylib.
170 G->allocActions().push_back(
171 {cantFail(WrapperFunctionCall::Create<
172 SPSArgList<SPSString, SPSExecutorAddr>>(
173 RegisterJITDylib, PlatformJDName, MachOHeaderAddr)),
174 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
175 DeregisterJITDylib, MachOHeaderAddr))});
176
177 // 3. Register deferred symbols.
178 G->allocActions().push_back(
179 {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
180 RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)),
181 cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
182 DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))});
183
184 // 4. Add the deferred actions to the graph.
185 std::move(DeferredAAs.begin(), DeferredAAs.end(),
186 std::back_inserter(G->allocActions()));
187
188 MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
189 }
190
discard(const JITDylib & JD,const SymbolStringPtr & Sym)191 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
192
193 private:
194 MachOPlatform &MOP;
195 StringRef PlatformJDName;
196 SymbolStringPtr CompleteBootstrapSymbol;
197 SymbolTableVector SymTab;
198 shared::AllocActions DeferredAAs;
199 ExecutorAddr MachOHeaderAddr;
200 ExecutorAddr PlatformBootstrap;
201 ExecutorAddr PlatformShutdown;
202 ExecutorAddr RegisterJITDylib;
203 ExecutorAddr DeregisterJITDylib;
204 ExecutorAddr RegisterObjectSymbolTable;
205 ExecutorAddr DeregisterObjectSymbolTable;
206 };
207
208 static StringRef ObjCRuntimeObjectSectionsData[] = {
209 MachOObjCCatListSectionName, MachOObjCClassListSectionName,
210 MachOObjCClassRefsSectionName, MachOObjCConstSectionName,
211 MachOObjCDataSectionName, MachOObjCSelRefsSectionName};
212
213 static StringRef ObjCRuntimeObjectSectionsText[] = {
214 MachOObjCClassNameSectionName, MachOObjCMethNameSectionName,
215 MachOObjCMethTypeSectionName, MachOSwift5TypesSectionName,
216 MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName,
217 MachOSwift5EntrySectionName, MachOSwift5ProtoSectionName,
218 MachOSwift5ProtosSectionName};
219
220 static StringRef ObjCRuntimeObjectSectionName =
221 "__llvm_jitlink_ObjCRuntimeRegistrationObject";
222
223 static StringRef ObjCImageInfoSymbolName =
224 "__llvm_jitlink_macho_objc_imageinfo";
225
226 struct ObjCImageInfoFlags {
227 uint16_t SwiftABIVersion;
228 uint16_t SwiftVersion;
229 bool HasCategoryClassProperties;
230 bool HasSignedObjCClassROs;
231
232 static constexpr uint32_t SIGNED_CLASS_RO = (1 << 4);
233 static constexpr uint32_t HAS_CATEGORY_CLASS_PROPERTIES = (1 << 6);
234
ObjCImageInfoFlags__anon55adfa710111::ObjCImageInfoFlags235 explicit ObjCImageInfoFlags(uint32_t RawFlags) {
236 HasSignedObjCClassROs = RawFlags & SIGNED_CLASS_RO;
237 HasCategoryClassProperties = RawFlags & HAS_CATEGORY_CLASS_PROPERTIES;
238 SwiftABIVersion = (RawFlags >> 8) & 0xFF;
239 SwiftVersion = (RawFlags >> 16) & 0xFFFF;
240 }
241
rawFlags__anon55adfa710111::ObjCImageInfoFlags242 uint32_t rawFlags() const {
243 uint32_t Result = 0;
244 if (HasCategoryClassProperties)
245 Result |= HAS_CATEGORY_CLASS_PROPERTIES;
246 if (HasSignedObjCClassROs)
247 Result |= SIGNED_CLASS_RO;
248 Result |= (SwiftABIVersion << 8);
249 Result |= (SwiftVersion << 16);
250 return Result;
251 }
252 };
253 } // end anonymous namespace
254
255 namespace llvm {
256 namespace orc {
257
Create(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<DefinitionGenerator> OrcRuntime,HeaderOptions PlatformJDOpts,MachOHeaderMUBuilder BuildMachOHeaderMU,std::optional<SymbolAliasMap> RuntimeAliases)258 Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(
259 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
260 JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime,
261 HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder BuildMachOHeaderMU,
262 std::optional<SymbolAliasMap> RuntimeAliases) {
263
264 // If the target is not supported then bail out immediately.
265 if (!supportedTarget(ES.getTargetTriple()))
266 return make_error<StringError>("Unsupported MachOPlatform triple: " +
267 ES.getTargetTriple().str(),
268 inconvertibleErrorCode());
269
270 auto &EPC = ES.getExecutorProcessControl();
271
272 // Create default aliases if the caller didn't supply any.
273 if (!RuntimeAliases)
274 RuntimeAliases = standardPlatformAliases(ES);
275
276 // Define the aliases.
277 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
278 return std::move(Err);
279
280 // Add JIT-dispatch function support symbols.
281 if (auto Err = PlatformJD.define(
282 absoluteSymbols({{ES.intern("___orc_rt_jit_dispatch"),
283 {EPC.getJITDispatchInfo().JITDispatchFunction,
284 JITSymbolFlags::Exported}},
285 {ES.intern("___orc_rt_jit_dispatch_ctx"),
286 {EPC.getJITDispatchInfo().JITDispatchContext,
287 JITSymbolFlags::Exported}}})))
288 return std::move(Err);
289
290 // Create the instance.
291 Error Err = Error::success();
292 auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform(
293 ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime),
294 std::move(PlatformJDOpts), std::move(BuildMachOHeaderMU), Err));
295 if (Err)
296 return std::move(Err);
297 return std::move(P);
298 }
299
300 Expected<std::unique_ptr<MachOPlatform>>
Create(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,HeaderOptions PlatformJDOpts,MachOHeaderMUBuilder BuildMachOHeaderMU,std::optional<SymbolAliasMap> RuntimeAliases)301 MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
302 JITDylib &PlatformJD, const char *OrcRuntimePath,
303 HeaderOptions PlatformJDOpts,
304 MachOHeaderMUBuilder BuildMachOHeaderMU,
305 std::optional<SymbolAliasMap> RuntimeAliases) {
306
307 // Create a generator for the ORC runtime archive.
308 auto OrcRuntimeArchiveGenerator =
309 StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
310 if (!OrcRuntimeArchiveGenerator)
311 return OrcRuntimeArchiveGenerator.takeError();
312
313 return Create(ES, ObjLinkingLayer, PlatformJD,
314 std::move(*OrcRuntimeArchiveGenerator),
315 std::move(PlatformJDOpts), std::move(BuildMachOHeaderMU),
316 std::move(RuntimeAliases));
317 }
318
setupJITDylib(JITDylib & JD)319 Error MachOPlatform::setupJITDylib(JITDylib &JD) {
320 return setupJITDylib(JD, /*Opts=*/{});
321 }
322
setupJITDylib(JITDylib & JD,HeaderOptions Opts)323 Error MachOPlatform::setupJITDylib(JITDylib &JD, HeaderOptions Opts) {
324 if (auto Err = JD.define(BuildMachOHeaderMU(*this, std::move(Opts))))
325 return Err;
326
327 return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError();
328 }
329
teardownJITDylib(JITDylib & JD)330 Error MachOPlatform::teardownJITDylib(JITDylib &JD) {
331 std::lock_guard<std::mutex> Lock(PlatformMutex);
332 auto I = JITDylibToHeaderAddr.find(&JD);
333 if (I != JITDylibToHeaderAddr.end()) {
334 assert(HeaderAddrToJITDylib.count(I->second) &&
335 "HeaderAddrToJITDylib missing entry");
336 HeaderAddrToJITDylib.erase(I->second);
337 JITDylibToHeaderAddr.erase(I);
338 }
339 JITDylibToPThreadKey.erase(&JD);
340 return Error::success();
341 }
342
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)343 Error MachOPlatform::notifyAdding(ResourceTracker &RT,
344 const MaterializationUnit &MU) {
345 auto &JD = RT.getJITDylib();
346 const auto &InitSym = MU.getInitializerSymbol();
347 if (!InitSym)
348 return Error::success();
349
350 RegisteredInitSymbols[&JD].add(InitSym,
351 SymbolLookupFlags::WeaklyReferencedSymbol);
352 LLVM_DEBUG({
353 dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU "
354 << MU.getName() << "\n";
355 });
356 return Error::success();
357 }
358
notifyRemoving(ResourceTracker & RT)359 Error MachOPlatform::notifyRemoving(ResourceTracker &RT) {
360 llvm_unreachable("Not supported yet");
361 }
362
addAliases(ExecutionSession & ES,SymbolAliasMap & Aliases,ArrayRef<std::pair<const char *,const char * >> AL)363 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
364 ArrayRef<std::pair<const char *, const char *>> AL) {
365 for (auto &KV : AL) {
366 auto AliasName = ES.intern(KV.first);
367 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
368 Aliases[std::move(AliasName)] = {ES.intern(KV.second),
369 JITSymbolFlags::Exported};
370 }
371 }
372
standardPlatformAliases(ExecutionSession & ES)373 SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) {
374 SymbolAliasMap Aliases;
375 addAliases(ES, Aliases, requiredCXXAliases());
376 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
377 return Aliases;
378 }
379
380 ArrayRef<std::pair<const char *, const char *>>
requiredCXXAliases()381 MachOPlatform::requiredCXXAliases() {
382 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
383 {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}};
384
385 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
386 }
387
388 ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases()389 MachOPlatform::standardRuntimeUtilityAliases() {
390 static const std::pair<const char *, const char *>
391 StandardRuntimeUtilityAliases[] = {
392 {"___orc_rt_run_program", "___orc_rt_macho_run_program"},
393 {"___orc_rt_jit_dlerror", "___orc_rt_macho_jit_dlerror"},
394 {"___orc_rt_jit_dlopen", "___orc_rt_macho_jit_dlopen"},
395 {"___orc_rt_jit_dlclose", "___orc_rt_macho_jit_dlclose"},
396 {"___orc_rt_jit_dlsym", "___orc_rt_macho_jit_dlsym"},
397 {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}};
398
399 return ArrayRef<std::pair<const char *, const char *>>(
400 StandardRuntimeUtilityAliases);
401 }
402
supportedTarget(const Triple & TT)403 bool MachOPlatform::supportedTarget(const Triple &TT) {
404 switch (TT.getArch()) {
405 case Triple::aarch64:
406 case Triple::x86_64:
407 return true;
408 default:
409 return false;
410 }
411 }
412
getPointerEdgeKind(jitlink::LinkGraph & G)413 jitlink::Edge::Kind MachOPlatform::getPointerEdgeKind(jitlink::LinkGraph &G) {
414 switch (G.getTargetTriple().getArch()) {
415 case Triple::aarch64:
416 return jitlink::aarch64::Pointer64;
417 case Triple::x86_64:
418 return jitlink::x86_64::Pointer64;
419 default:
420 llvm_unreachable("Unsupported architecture");
421 }
422 }
423
424 MachOPlatform::MachOExecutorSymbolFlags
flagsForSymbol(jitlink::Symbol & Sym)425 MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) {
426 MachOPlatform::MachOExecutorSymbolFlags Flags{};
427 if (Sym.getLinkage() == jitlink::Linkage::Weak)
428 Flags |= MachOExecutorSymbolFlags::Weak;
429
430 if (Sym.isCallable())
431 Flags |= MachOExecutorSymbolFlags::Callable;
432
433 return Flags;
434 }
435
MachOPlatform(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,HeaderOptions PlatformJDOpts,MachOHeaderMUBuilder BuildMachOHeaderMU,Error & Err)436 MachOPlatform::MachOPlatform(
437 ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
438 JITDylib &PlatformJD,
439 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
440 HeaderOptions PlatformJDOpts, MachOHeaderMUBuilder BuildMachOHeaderMU,
441 Error &Err)
442 : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer),
443 BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) {
444 ErrorAsOutParameter _(&Err);
445 ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this));
446 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
447
448 BootstrapInfo BI;
449 Bootstrap = &BI;
450
451 // Bootstrap process -- here be phase-ordering dragons.
452 //
453 // The MachOPlatform class uses allocation actions to register metadata
454 // sections with the ORC runtime, however the runtime contains metadata
455 // registration functions that have their own metadata that they need to
456 // register (e.g. the frame-info registration functions have frame-info).
457 // We can't use an ordinary lookup to find these registration functions
458 // because their address is needed during the link of the containing graph
459 // itself (to build the allocation actions that will call the registration
460 // functions). Further complicating the situation (a) the graph containing
461 // the registration functions is allowed to depend on other graphs (e.g. the
462 // graph containing the ORC runtime RTTI support) so we need to handle an
463 // unknown set of dependencies during bootstrap, and (b) these graphs may
464 // be linked concurrently if the user has installed a concurrent dispatcher.
465 //
466 // We satisfy these constraints by implementing a bootstrap phase during which
467 // allocation actions generated by MachOPlatform are appended to a list of
468 // deferred allocation actions, rather than to the graphs themselves. At the
469 // end of the bootstrap process the deferred actions are attached to a final
470 // "complete-bootstrap" graph that causes them to be run.
471 //
472 // The bootstrap steps are as follows:
473 //
474 // 1. Request the graph containing the mach header. This graph is guaranteed
475 // not to have any metadata so the fact that the registration functions
476 // are not available yet is not a problem.
477 //
478 // 2. Look up the registration functions and discard the results. This will
479 // trigger linking of the graph containing these functions, and
480 // consequently any graphs that it depends on. We do not use the lookup
481 // result to find the addresses of the functions requested (as described
482 // above the lookup will return too late for that), instead we capture the
483 // addresses in a post-allocation pass injected by the platform runtime
484 // during bootstrap only.
485 //
486 // 3. During bootstrap the MachOPlatformPlugin keeps a count of the number of
487 // graphs being linked (potentially concurrently), and we block until all
488 // of these graphs have completed linking. This is to avoid a race on the
489 // deferred-actions vector: the lookup for the runtime registration
490 // functions may return while some functions (those that are being
491 // incidentally linked in, but aren't reachable via the runtime functions)
492 // are still being linked, and we need to capture any allocation actions
493 // for this incidental code before we proceed.
494 //
495 // 4. Once all active links are complete we transfer the deferred actions to
496 // a newly added CompleteBootstrap graph and then request a symbol from
497 // the CompleteBootstrap graph to trigger materialization. This will cause
498 // all deferred actions to be run, and once this lookup returns we can
499 // proceed.
500 //
501 // 5. Finally, we associate runtime support methods in MachOPlatform with
502 // the corresponding jit-dispatch tag variables in the ORC runtime to make
503 // the support methods callable. The bootstrap is now complete.
504
505 // Step (1) Add header materialization unit and request.
506 if ((Err = PlatformJD.define(
507 this->BuildMachOHeaderMU(*this, std::move(PlatformJDOpts)))))
508 return;
509 if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError()))
510 return;
511
512 // Step (2) Request runtime registration functions to trigger
513 // materialization..
514 if ((Err = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
515 SymbolLookupSet(
516 {PlatformBootstrap.Name, PlatformShutdown.Name,
517 RegisterJITDylib.Name, DeregisterJITDylib.Name,
518 RegisterObjectSymbolTable.Name,
519 DeregisterObjectSymbolTable.Name,
520 RegisterObjectPlatformSections.Name,
521 DeregisterObjectPlatformSections.Name,
522 CreatePThreadKey.Name}))
523 .takeError()))
524 return;
525
526 // Step (3) Wait for any incidental linker work to complete.
527 {
528 std::unique_lock<std::mutex> Lock(BI.Mutex);
529 BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
530 Bootstrap = nullptr;
531 }
532
533 // Step (4) Add complete-bootstrap materialization unit and request.
534 auto BootstrapCompleteSymbol = ES.intern("__orc_rt_macho_complete_bootstrap");
535 if ((Err = PlatformJD.define(
536 std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>(
537 *this, PlatformJD.getName(), BootstrapCompleteSymbol,
538 std::move(BI.SymTab), std::move(BI.DeferredAAs),
539 BI.MachOHeaderAddr, PlatformBootstrap.Addr,
540 PlatformShutdown.Addr, RegisterJITDylib.Addr,
541 DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr,
542 DeregisterObjectSymbolTable.Addr))))
543 return;
544 if ((Err = ES.lookup(makeJITDylibSearchOrder(
545 &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
546 std::move(BootstrapCompleteSymbol))
547 .takeError()))
548 return;
549
550 // (5) Associate runtime support functions.
551 if ((Err = associateRuntimeSupportFunctions()))
552 return;
553 }
554
associateRuntimeSupportFunctions()555 Error MachOPlatform::associateRuntimeSupportFunctions() {
556 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
557
558 using PushInitializersSPSSig =
559 SPSExpected<SPSMachOJITDylibDepInfoMap>(SPSExecutorAddr);
560 WFs[ES.intern("___orc_rt_macho_push_initializers_tag")] =
561 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
562 this, &MachOPlatform::rt_pushInitializers);
563
564 using PushSymbolsSPSSig =
565 SPSError(SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>);
566 WFs[ES.intern("___orc_rt_macho_push_symbols_tag")] =
567 ES.wrapAsyncWithSPS<PushSymbolsSPSSig>(this,
568 &MachOPlatform::rt_pushSymbols);
569
570 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
571 }
572
pushInitializersLoop(PushInitializersSendResultFn SendResult,JITDylibSP JD)573 void MachOPlatform::pushInitializersLoop(
574 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
575 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
576 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
577 SmallVector<JITDylib *, 16> Worklist({JD.get()});
578
579 ES.runSessionLocked([&]() {
580 while (!Worklist.empty()) {
581 // FIXME: Check for defunct dylibs.
582
583 auto DepJD = Worklist.back();
584 Worklist.pop_back();
585
586 // If we've already visited this JITDylib on this iteration then continue.
587 if (JDDepMap.count(DepJD))
588 continue;
589
590 // Add dep info.
591 auto &DM = JDDepMap[DepJD];
592 DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
593 for (auto &KV : O) {
594 if (KV.first == DepJD)
595 continue;
596 DM.push_back(KV.first);
597 Worklist.push_back(KV.first);
598 }
599 });
600
601 // Add any registered init symbols.
602 auto RISItr = RegisteredInitSymbols.find(DepJD);
603 if (RISItr != RegisteredInitSymbols.end()) {
604 NewInitSymbols[DepJD] = std::move(RISItr->second);
605 RegisteredInitSymbols.erase(RISItr);
606 }
607 }
608 });
609
610 // If there are no further init symbols to look up then send the link order
611 // (as a list of header addresses) to the caller.
612 if (NewInitSymbols.empty()) {
613
614 // To make the list intelligible to the runtime we need to convert all
615 // JITDylib pointers to their header addresses. Only include JITDylibs
616 // that appear in the JITDylibToHeaderAddr map (i.e. those that have been
617 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
618 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
619 HeaderAddrs.reserve(JDDepMap.size());
620 {
621 std::lock_guard<std::mutex> Lock(PlatformMutex);
622 for (auto &KV : JDDepMap) {
623 auto I = JITDylibToHeaderAddr.find(KV.first);
624 if (I != JITDylibToHeaderAddr.end())
625 HeaderAddrs[KV.first] = I->second;
626 }
627 }
628
629 // Build the dep info map to return.
630 MachOJITDylibDepInfoMap DIM;
631 DIM.reserve(JDDepMap.size());
632 for (auto &KV : JDDepMap) {
633 auto HI = HeaderAddrs.find(KV.first);
634 // Skip unmanaged JITDylibs.
635 if (HI == HeaderAddrs.end())
636 continue;
637 auto H = HI->second;
638 MachOJITDylibDepInfo DepInfo;
639 for (auto &Dep : KV.second) {
640 auto HJ = HeaderAddrs.find(Dep);
641 if (HJ != HeaderAddrs.end())
642 DepInfo.DepHeaders.push_back(HJ->second);
643 }
644 DIM.push_back(std::make_pair(H, std::move(DepInfo)));
645 }
646 SendResult(DIM);
647 return;
648 }
649
650 // Otherwise issue a lookup and re-run this phase when it completes.
651 lookupInitSymbolsAsync(
652 [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
653 if (Err)
654 SendResult(std::move(Err));
655 else
656 pushInitializersLoop(std::move(SendResult), JD);
657 },
658 ES, std::move(NewInitSymbols));
659 }
660
rt_pushInitializers(PushInitializersSendResultFn SendResult,ExecutorAddr JDHeaderAddr)661 void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
662 ExecutorAddr JDHeaderAddr) {
663 JITDylibSP JD;
664 {
665 std::lock_guard<std::mutex> Lock(PlatformMutex);
666 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
667 if (I != HeaderAddrToJITDylib.end())
668 JD = I->second;
669 }
670
671 LLVM_DEBUG({
672 dbgs() << "MachOPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
673 if (JD)
674 dbgs() << "pushing initializers for " << JD->getName() << "\n";
675 else
676 dbgs() << "No JITDylib for header address.\n";
677 });
678
679 if (!JD) {
680 SendResult(make_error<StringError>("No JITDylib with header addr " +
681 formatv("{0:x}", JDHeaderAddr),
682 inconvertibleErrorCode()));
683 return;
684 }
685
686 pushInitializersLoop(std::move(SendResult), JD);
687 }
688
rt_pushSymbols(PushSymbolsInSendResultFn SendResult,ExecutorAddr Handle,const std::vector<std::pair<StringRef,bool>> & SymbolNames)689 void MachOPlatform::rt_pushSymbols(
690 PushSymbolsInSendResultFn SendResult, ExecutorAddr Handle,
691 const std::vector<std::pair<StringRef, bool>> &SymbolNames) {
692
693 JITDylib *JD = nullptr;
694
695 {
696 std::lock_guard<std::mutex> Lock(PlatformMutex);
697 auto I = HeaderAddrToJITDylib.find(Handle);
698 if (I != HeaderAddrToJITDylib.end())
699 JD = I->second;
700 }
701 LLVM_DEBUG({
702 dbgs() << "MachOPlatform::rt_pushSymbols(";
703 if (JD)
704 dbgs() << "\"" << JD->getName() << "\", [ ";
705 else
706 dbgs() << "<invalid handle " << Handle << ">, [ ";
707 for (auto &Name : SymbolNames)
708 dbgs() << "\"" << Name.first << "\" ";
709 dbgs() << "])\n";
710 });
711
712 if (!JD) {
713 SendResult(make_error<StringError>("No JITDylib associated with handle " +
714 formatv("{0:x}", Handle),
715 inconvertibleErrorCode()));
716 return;
717 }
718
719 SymbolLookupSet LS;
720 for (auto &[Name, Required] : SymbolNames)
721 LS.add(ES.intern(Name), Required
722 ? SymbolLookupFlags::RequiredSymbol
723 : SymbolLookupFlags::WeaklyReferencedSymbol);
724
725 ES.lookup(
726 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
727 std::move(LS), SymbolState::Ready,
728 [SendResult = std::move(SendResult)](Expected<SymbolMap> Result) mutable {
729 SendResult(Result.takeError());
730 },
731 NoDependenciesToRegister);
732 }
733
createPThreadKey()734 Expected<uint64_t> MachOPlatform::createPThreadKey() {
735 if (!CreatePThreadKey.Addr)
736 return make_error<StringError>(
737 "Attempting to create pthread key in target, but runtime support has "
738 "not been loaded yet",
739 inconvertibleErrorCode());
740
741 Expected<uint64_t> Result(0);
742 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
743 CreatePThreadKey.Addr, Result))
744 return std::move(Err);
745 return Result;
746 }
747
modifyPassConfig(MaterializationResponsibility & MR,jitlink::LinkGraph & LG,jitlink::PassConfiguration & Config)748 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
749 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
750 jitlink::PassConfiguration &Config) {
751
752 using namespace jitlink;
753
754 bool InBootstrapPhase =
755 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
756
757 // If we're in the bootstrap phase then increment the active graphs.
758 if (InBootstrapPhase) {
759 Config.PrePrunePasses.push_back(
760 [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
761 Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
762 return bootstrapPipelineRecordRuntimeFunctions(G);
763 });
764 }
765
766 // --- Handle Initializers ---
767 if (auto InitSymbol = MR.getInitializerSymbol()) {
768
769 // If the initializer symbol is the MachOHeader start symbol then just
770 // register it and then bail out -- the header materialization unit
771 // definitely doesn't need any other passes.
772 if (InitSymbol == MP.MachOHeaderStartSymbol && !InBootstrapPhase) {
773 Config.PostAllocationPasses.push_back([this, &MR](LinkGraph &G) {
774 return associateJITDylibHeaderSymbol(G, MR);
775 });
776 return;
777 }
778
779 // If the object contains an init symbol other than the header start symbol
780 // then add passes to preserve, process and register the init
781 // sections/symbols.
782 Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) {
783 if (auto Err = preserveImportantSections(G, MR))
784 return Err;
785 return processObjCImageInfo(G, MR);
786 });
787 Config.PostPrunePasses.push_back(
788 [this](LinkGraph &G) { return createObjCRuntimeObject(G); });
789 Config.PostAllocationPasses.push_back(
790 [this, &MR](LinkGraph &G) { return populateObjCRuntimeObject(G, MR); });
791 }
792
793 // Insert TLV lowering at the start of the PostPrunePasses, since we want
794 // it to run before GOT/PLT lowering.
795 Config.PostPrunePasses.insert(
796 Config.PostPrunePasses.begin(),
797 [this, &JD = MR.getTargetJITDylib()](LinkGraph &G) {
798 return fixTLVSectionsAndEdges(G, JD);
799 });
800
801 // Add symbol table prepare and register passes: These will add strings for
802 // all symbols to the c-strings section, and build a symbol table registration
803 // call.
804 auto JITSymTabInfo = std::make_shared<JITSymTabVector>();
805 Config.PostPrunePasses.push_back([this, JITSymTabInfo](LinkGraph &G) {
806 return prepareSymbolTableRegistration(G, *JITSymTabInfo);
807 });
808 Config.PostFixupPasses.push_back([this, &MR, JITSymTabInfo,
809 InBootstrapPhase](LinkGraph &G) {
810 return addSymbolTableRegistration(G, MR, *JITSymTabInfo, InBootstrapPhase);
811 });
812
813 // Add a pass to register the final addresses of any special sections in the
814 // object with the runtime.
815 Config.PostAllocationPasses.push_back(
816 [this, &JD = MR.getTargetJITDylib(), InBootstrapPhase](LinkGraph &G) {
817 return registerObjectPlatformSections(G, JD, InBootstrapPhase);
818 });
819
820 // If we're in the bootstrap phase then steal allocation actions and then
821 // decrement the active graphs.
822 if (InBootstrapPhase)
823 Config.PostFixupPasses.push_back(
824 [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
825 }
826
827 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
getSyntheticSymbolDependencies(MaterializationResponsibility & MR)828 MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
829 MaterializationResponsibility &MR) {
830 std::lock_guard<std::mutex> Lock(PluginMutex);
831 auto I = InitSymbolDeps.find(&MR);
832 if (I != InitSymbolDeps.end()) {
833 SyntheticSymbolDependenciesMap Result;
834 Result[MR.getInitializerSymbol()] = std::move(I->second);
835 InitSymbolDeps.erase(&MR);
836 return Result;
837 }
838 return SyntheticSymbolDependenciesMap();
839 }
840
bootstrapPipelineStart(jitlink::LinkGraph & G)841 Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineStart(
842 jitlink::LinkGraph &G) {
843 // Increment the active graphs count in BootstrapInfo.
844 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
845 ++MP.Bootstrap.load()->ActiveGraphs;
846 return Error::success();
847 }
848
849 Error MachOPlatform::MachOPlatformPlugin::
bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph & G)850 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
851 // Record bootstrap function names.
852 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
853 {*MP.MachOHeaderStartSymbol, &MP.Bootstrap.load()->MachOHeaderAddr},
854 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
855 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
856 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
857 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
858 {*MP.RegisterObjectSymbolTable.Name, &MP.RegisterObjectSymbolTable.Addr},
859 {*MP.DeregisterObjectSymbolTable.Name,
860 &MP.DeregisterObjectSymbolTable.Addr},
861 {*MP.RegisterObjectPlatformSections.Name,
862 &MP.RegisterObjectPlatformSections.Addr},
863 {*MP.DeregisterObjectPlatformSections.Name,
864 &MP.DeregisterObjectPlatformSections.Addr},
865 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr},
866 {*MP.RegisterObjCRuntimeObject.Name, &MP.RegisterObjCRuntimeObject.Addr},
867 {*MP.DeregisterObjCRuntimeObject.Name,
868 &MP.DeregisterObjCRuntimeObject.Addr}};
869
870 bool RegisterMachOHeader = false;
871
872 for (auto *Sym : G.defined_symbols()) {
873 for (auto &RTSym : RuntimeSymbols) {
874 if (Sym->hasName() && Sym->getName() == RTSym.first) {
875 if (*RTSym.second)
876 return make_error<StringError>(
877 "Duplicate " + RTSym.first +
878 " detected during MachOPlatform bootstrap",
879 inconvertibleErrorCode());
880
881 if (Sym->getName() == *MP.MachOHeaderStartSymbol)
882 RegisterMachOHeader = true;
883
884 *RTSym.second = Sym->getAddress();
885 }
886 }
887 }
888
889 if (RegisterMachOHeader) {
890 // If this graph defines the macho header symbol then create the internal
891 // mapping between it and PlatformJD.
892 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
893 MP.JITDylibToHeaderAddr[&MP.PlatformJD] =
894 MP.Bootstrap.load()->MachOHeaderAddr;
895 MP.HeaderAddrToJITDylib[MP.Bootstrap.load()->MachOHeaderAddr] =
896 &MP.PlatformJD;
897 }
898
899 return Error::success();
900 }
901
bootstrapPipelineEnd(jitlink::LinkGraph & G)902 Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd(
903 jitlink::LinkGraph &G) {
904 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
905 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
906 --MP.Bootstrap.load()->ActiveGraphs;
907 // Notify Bootstrap->CV while holding the mutex because the mutex is
908 // also keeping Bootstrap->CV alive.
909 if (MP.Bootstrap.load()->ActiveGraphs == 0)
910 MP.Bootstrap.load()->CV.notify_all();
911 return Error::success();
912 }
913
associateJITDylibHeaderSymbol(jitlink::LinkGraph & G,MaterializationResponsibility & MR)914 Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
915 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
916 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
917 return Sym->getName() == *MP.MachOHeaderStartSymbol;
918 });
919 assert(I != G.defined_symbols().end() && "Missing MachO header start symbol");
920
921 auto &JD = MR.getTargetJITDylib();
922 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
923 auto HeaderAddr = (*I)->getAddress();
924 MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
925 MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
926 // We can unconditionally add these actions to the Graph because this pass
927 // isn't used during bootstrap.
928 G.allocActions().push_back(
929 {cantFail(
930 WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>(
931 MP.RegisterJITDylib.Addr, JD.getName(), HeaderAddr)),
932 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
933 MP.DeregisterJITDylib.Addr, HeaderAddr))});
934 return Error::success();
935 }
936
preserveImportantSections(jitlink::LinkGraph & G,MaterializationResponsibility & MR)937 Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections(
938 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
939 // __objc_imageinfo is "important": we want to preserve it and record its
940 // address in the first graph that it appears in, then verify and discard it
941 // in all subsequent graphs. In this pass we preserve unconditionally -- we'll
942 // manually throw it away in the processObjCImageInfo pass.
943 if (auto *ObjCImageInfoSec =
944 G.findSectionByName(MachOObjCImageInfoSectionName)) {
945 if (ObjCImageInfoSec->blocks_size() != 1)
946 return make_error<StringError>(
947 "In " + G.getName() +
948 "__DATA,__objc_imageinfo contains multiple blocks",
949 inconvertibleErrorCode());
950 G.addAnonymousSymbol(**ObjCImageInfoSec->blocks().begin(), 0, 0, false,
951 true);
952
953 for (auto *B : ObjCImageInfoSec->blocks())
954 if (!B->edges_empty())
955 return make_error<StringError>("In " + G.getName() + ", " +
956 MachOObjCImageInfoSectionName +
957 " contains references to symbols",
958 inconvertibleErrorCode());
959 }
960
961 // Init sections are important: We need to preserve them and so that their
962 // addresses can be captured and reported to the ORC runtime in
963 // registerObjectPlatformSections.
964 JITLinkSymbolSet InitSectionSymbols;
965 for (auto &InitSectionName : MachOInitSectionNames) {
966 // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may
967 // remove it later.
968 if (InitSectionName == MachOObjCImageInfoSectionName)
969 continue;
970
971 // Skip non-init sections.
972 auto *InitSection = G.findSectionByName(InitSectionName);
973 if (!InitSection)
974 continue;
975
976 // Make a pass over live symbols in the section: those blocks are already
977 // preserved.
978 DenseSet<jitlink::Block *> AlreadyLiveBlocks;
979 for (auto &Sym : InitSection->symbols()) {
980 auto &B = Sym->getBlock();
981 if (Sym->isLive() && Sym->getOffset() == 0 &&
982 Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
983 InitSectionSymbols.insert(Sym);
984 AlreadyLiveBlocks.insert(&B);
985 }
986 }
987
988 // Add anonymous symbols to preserve any not-already-preserved blocks.
989 for (auto *B : InitSection->blocks())
990 if (!AlreadyLiveBlocks.count(B))
991 InitSectionSymbols.insert(
992 &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
993 }
994
995 if (!InitSectionSymbols.empty()) {
996 std::lock_guard<std::mutex> Lock(PluginMutex);
997 InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
998 }
999
1000 return Error::success();
1001 }
1002
processObjCImageInfo(jitlink::LinkGraph & G,MaterializationResponsibility & MR)1003 Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
1004 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
1005
1006 // If there's an ObjC imagine info then either
1007 // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In
1008 // this case we name and record it.
1009 // OR
1010 // (2) We already have a recorded __objc_imageinfo for this JITDylib,
1011 // in which case we just verify it.
1012 auto *ObjCImageInfo = G.findSectionByName(MachOObjCImageInfoSectionName);
1013 if (!ObjCImageInfo)
1014 return Error::success();
1015
1016 auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
1017
1018 // Check that the section is not empty if present.
1019 if (ObjCImageInfoBlocks.empty())
1020 return make_error<StringError>("Empty " + MachOObjCImageInfoSectionName +
1021 " section in " + G.getName(),
1022 inconvertibleErrorCode());
1023
1024 // Check that there's only one block in the section.
1025 if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
1026 return make_error<StringError>("Multiple blocks in " +
1027 MachOObjCImageInfoSectionName +
1028 " section in " + G.getName(),
1029 inconvertibleErrorCode());
1030
1031 // Check that the __objc_imageinfo section is unreferenced.
1032 // FIXME: We could optimize this check if Symbols had a ref-count.
1033 for (auto &Sec : G.sections()) {
1034 if (&Sec != ObjCImageInfo)
1035 for (auto *B : Sec.blocks())
1036 for (auto &E : B->edges())
1037 if (E.getTarget().isDefined() &&
1038 &E.getTarget().getBlock().getSection() == ObjCImageInfo)
1039 return make_error<StringError>(MachOObjCImageInfoSectionName +
1040 " is referenced within file " +
1041 G.getName(),
1042 inconvertibleErrorCode());
1043 }
1044
1045 auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
1046 auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
1047 auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness());
1048 auto Flags =
1049 support::endian::read32(ObjCImageInfoData + 4, G.getEndianness());
1050
1051 // Lock the mutex while we verify / update the ObjCImageInfos map.
1052 std::lock_guard<std::mutex> Lock(PluginMutex);
1053
1054 auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib());
1055 if (ObjCImageInfoItr != ObjCImageInfos.end()) {
1056 // We've already registered an __objc_imageinfo section. Verify the
1057 // content of this new section matches, then delete it.
1058 if (ObjCImageInfoItr->second.Version != Version)
1059 return make_error<StringError>(
1060 "ObjC version in " + G.getName() +
1061 " does not match first registered version",
1062 inconvertibleErrorCode());
1063 if (ObjCImageInfoItr->second.Flags != Flags)
1064 if (Error E = mergeImageInfoFlags(G, MR, ObjCImageInfoItr->second, Flags))
1065 return E;
1066
1067 // __objc_imageinfo is valid. Delete the block.
1068 for (auto *S : ObjCImageInfo->symbols())
1069 G.removeDefinedSymbol(*S);
1070 G.removeBlock(ObjCImageInfoBlock);
1071 } else {
1072 LLVM_DEBUG({
1073 dbgs() << "MachOPlatform: Registered __objc_imageinfo for "
1074 << MR.getTargetJITDylib().getName() << " in " << G.getName()
1075 << "; flags = " << formatv("{0:x4}", Flags) << "\n";
1076 });
1077 // We haven't registered an __objc_imageinfo section yet. Register and
1078 // move on. The section should already be marked no-dead-strip.
1079 G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName,
1080 ObjCImageInfoBlock.getSize(), jitlink::Linkage::Strong,
1081 jitlink::Scope::Hidden, false, true);
1082 if (auto Err = MR.defineMaterializing(
1083 {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName),
1084 JITSymbolFlags()}}))
1085 return Err;
1086 ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags, false};
1087 }
1088
1089 return Error::success();
1090 }
1091
mergeImageInfoFlags(jitlink::LinkGraph & G,MaterializationResponsibility & MR,ObjCImageInfo & Info,uint32_t NewFlags)1092 Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags(
1093 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
1094 ObjCImageInfo &Info, uint32_t NewFlags) {
1095 if (Info.Flags == NewFlags)
1096 return Error::success();
1097
1098 ObjCImageInfoFlags Old(Info.Flags);
1099 ObjCImageInfoFlags New(NewFlags);
1100
1101 // Check for incompatible flags.
1102 if (Old.SwiftABIVersion && New.SwiftABIVersion &&
1103 Old.SwiftABIVersion != New.SwiftABIVersion)
1104 return make_error<StringError>("Swift ABI version in " + G.getName() +
1105 " does not match first registered flags",
1106 inconvertibleErrorCode());
1107
1108 if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties)
1109 return make_error<StringError>("ObjC category class property support in " +
1110 G.getName() +
1111 " does not match first registered flags",
1112 inconvertibleErrorCode());
1113 if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs)
1114 return make_error<StringError>("ObjC class_ro_t pointer signing in " +
1115 G.getName() +
1116 " does not match first registered flags",
1117 inconvertibleErrorCode());
1118
1119 // If we cannot change the flags, ignore any remaining differences. Adding
1120 // Swift or changing its version are unlikely to cause problems in practice.
1121 if (Info.Finalized)
1122 return Error::success();
1123
1124 // Use the minimum Swift version.
1125 if (Old.SwiftVersion && New.SwiftVersion)
1126 New.SwiftVersion = std::min(Old.SwiftVersion, New.SwiftVersion);
1127 else if (Old.SwiftVersion)
1128 New.SwiftVersion = Old.SwiftVersion;
1129 // Add a Swift ABI version if it was pure objc before.
1130 if (!New.SwiftABIVersion)
1131 New.SwiftABIVersion = Old.SwiftABIVersion;
1132
1133 LLVM_DEBUG({
1134 dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for "
1135 << MR.getTargetJITDylib().getName() << " (was "
1136 << formatv("{0:x4}", Old.rawFlags()) << ")"
1137 << " with " << G.getName() << " (" << formatv("{0:x4}", NewFlags)
1138 << ")"
1139 << " -> " << formatv("{0:x4}", New.rawFlags()) << "\n";
1140 });
1141
1142 Info.Flags = New.rawFlags();
1143 return Error::success();
1144 }
1145
fixTLVSectionsAndEdges(jitlink::LinkGraph & G,JITDylib & JD)1146 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
1147 jitlink::LinkGraph &G, JITDylib &JD) {
1148
1149 // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr.
1150 for (auto *Sym : G.external_symbols())
1151 if (Sym->getName() == "__tlv_bootstrap") {
1152 Sym->setName("___orc_rt_macho_tlv_get_addr");
1153 break;
1154 }
1155
1156 // Store key in __thread_vars struct fields.
1157 if (auto *ThreadDataSec = G.findSectionByName(MachOThreadVarsSectionName)) {
1158 std::optional<uint64_t> Key;
1159 {
1160 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1161 auto I = MP.JITDylibToPThreadKey.find(&JD);
1162 if (I != MP.JITDylibToPThreadKey.end())
1163 Key = I->second;
1164 }
1165
1166 if (!Key) {
1167 if (auto KeyOrErr = MP.createPThreadKey())
1168 Key = *KeyOrErr;
1169 else
1170 return KeyOrErr.takeError();
1171 }
1172
1173 uint64_t PlatformKeyBits =
1174 support::endian::byte_swap(*Key, G.getEndianness());
1175
1176 for (auto *B : ThreadDataSec->blocks()) {
1177 if (B->getSize() != 3 * G.getPointerSize())
1178 return make_error<StringError>("__thread_vars block at " +
1179 formatv("{0:x}", B->getAddress()) +
1180 " has unexpected size",
1181 inconvertibleErrorCode());
1182
1183 auto NewBlockContent = G.allocateBuffer(B->getSize());
1184 llvm::copy(B->getContent(), NewBlockContent.data());
1185 memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits,
1186 G.getPointerSize());
1187 B->setContent(NewBlockContent);
1188 }
1189 }
1190
1191 // Transform any TLV edges into GOT edges.
1192 for (auto *B : G.blocks())
1193 for (auto &E : B->edges())
1194 if (E.getKind() ==
1195 jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable)
1196 E.setKind(jitlink::x86_64::
1197 RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable);
1198
1199 return Error::success();
1200 }
1201
1202 std::optional<MachOPlatform::MachOPlatformPlugin::UnwindSections>
findUnwindSectionInfo(jitlink::LinkGraph & G)1203 MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo(
1204 jitlink::LinkGraph &G) {
1205 using namespace jitlink;
1206
1207 UnwindSections US;
1208
1209 // ScanSection records a section range and adds any executable blocks that
1210 // that section points to to the CodeBlocks vector.
1211 SmallVector<Block *> CodeBlocks;
1212 auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) {
1213 if (Sec.blocks().empty())
1214 return;
1215 SecRange = (*Sec.blocks().begin())->getRange();
1216 for (auto *B : Sec.blocks()) {
1217 auto R = B->getRange();
1218 SecRange.Start = std::min(SecRange.Start, R.Start);
1219 SecRange.End = std::max(SecRange.End, R.End);
1220 for (auto &E : B->edges()) {
1221 if (!E.getTarget().isDefined())
1222 continue;
1223 auto &TargetBlock = E.getTarget().getBlock();
1224 auto &TargetSection = TargetBlock.getSection();
1225 if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec)
1226 CodeBlocks.push_back(&TargetBlock);
1227 }
1228 }
1229 };
1230
1231 if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName))
1232 ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection);
1233
1234 if (Section *CUInfoSec =
1235 G.findSectionByName(MachOCompactUnwindInfoSectionName))
1236 ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection);
1237
1238 // If we didn't find any pointed-to code-blocks then there's no need to
1239 // register any info.
1240 if (CodeBlocks.empty())
1241 return std::nullopt;
1242
1243 // We have info to register. Sort the code blocks into address order and
1244 // build a list of contiguous address ranges covering them all.
1245 llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) {
1246 return LHS->getAddress() < RHS->getAddress();
1247 });
1248 for (auto *B : CodeBlocks) {
1249 if (US.CodeRanges.empty() || US.CodeRanges.back().End != B->getAddress())
1250 US.CodeRanges.push_back(B->getRange());
1251 else
1252 US.CodeRanges.back().End = B->getRange().End;
1253 }
1254
1255 LLVM_DEBUG({
1256 dbgs() << "MachOPlatform identified unwind info in " << G.getName() << ":\n"
1257 << " DWARF: ";
1258 if (US.DwarfSection.Start)
1259 dbgs() << US.DwarfSection << "\n";
1260 else
1261 dbgs() << "none\n";
1262 dbgs() << " Compact-unwind: ";
1263 if (US.CompactUnwindSection.Start)
1264 dbgs() << US.CompactUnwindSection << "\n";
1265 else
1266 dbgs() << "none\n"
1267 << "for code ranges:\n";
1268 for (auto &CR : US.CodeRanges)
1269 dbgs() << " " << CR << "\n";
1270 if (US.CodeRanges.size() >= G.sections_size())
1271 dbgs() << "WARNING: High number of discontiguous code ranges! "
1272 "Padding may be interfering with coalescing.\n";
1273 });
1274
1275 return US;
1276 }
1277
registerObjectPlatformSections(jitlink::LinkGraph & G,JITDylib & JD,bool InBootstrapPhase)1278 Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
1279 jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) {
1280
1281 // Get a pointer to the thread data section if there is one. It will be used
1282 // below.
1283 jitlink::Section *ThreadDataSection =
1284 G.findSectionByName(MachOThreadDataSectionName);
1285
1286 // Handle thread BSS section if there is one.
1287 if (auto *ThreadBSSSection = G.findSectionByName(MachOThreadBSSSectionName)) {
1288 // If there's already a thread data section in this graph then merge the
1289 // thread BSS section content into it, otherwise just treat the thread
1290 // BSS section as the thread data section.
1291 if (ThreadDataSection)
1292 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
1293 else
1294 ThreadDataSection = ThreadBSSSection;
1295 }
1296
1297 SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs;
1298
1299 // Collect data sections to register.
1300 StringRef DataSections[] = {MachODataDataSectionName,
1301 MachODataCommonSectionName,
1302 MachOEHFrameSectionName};
1303 for (auto &SecName : DataSections) {
1304 if (auto *Sec = G.findSectionByName(SecName)) {
1305 jitlink::SectionRange R(*Sec);
1306 if (!R.empty())
1307 MachOPlatformSecs.push_back({SecName, R.getRange()});
1308 }
1309 }
1310
1311 // Having merged thread BSS (if present) and thread data (if present),
1312 // record the resulting section range.
1313 if (ThreadDataSection) {
1314 jitlink::SectionRange R(*ThreadDataSection);
1315 if (!R.empty())
1316 MachOPlatformSecs.push_back({MachOThreadDataSectionName, R.getRange()});
1317 }
1318
1319 // If any platform sections were found then add an allocation action to call
1320 // the registration function.
1321 StringRef PlatformSections[] = {MachOModInitFuncSectionName,
1322 ObjCRuntimeObjectSectionName};
1323
1324 for (auto &SecName : PlatformSections) {
1325 auto *Sec = G.findSectionByName(SecName);
1326 if (!Sec)
1327 continue;
1328 jitlink::SectionRange R(*Sec);
1329 if (R.empty())
1330 continue;
1331
1332 MachOPlatformSecs.push_back({SecName, R.getRange()});
1333 }
1334
1335 std::optional<std::tuple<SmallVector<ExecutorAddrRange>, ExecutorAddrRange,
1336 ExecutorAddrRange>>
1337 UnwindInfo;
1338 if (auto UI = findUnwindSectionInfo(G))
1339 UnwindInfo = std::make_tuple(std::move(UI->CodeRanges), UI->DwarfSection,
1340 UI->CompactUnwindSection);
1341
1342 if (!MachOPlatformSecs.empty() || UnwindInfo) {
1343 // Dump the scraped inits.
1344 LLVM_DEBUG({
1345 dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n";
1346 for (auto &KV : MachOPlatformSecs)
1347 dbgs() << " " << KV.first << ": " << KV.second << "\n";
1348 });
1349
1350 using SPSRegisterObjectPlatformSectionsArgs = SPSArgList<
1351 SPSExecutorAddr,
1352 SPSOptional<SPSTuple<SPSSequence<SPSExecutorAddrRange>,
1353 SPSExecutorAddrRange, SPSExecutorAddrRange>>,
1354 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>;
1355
1356 shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase)
1357 ? G.allocActions()
1358 : MP.Bootstrap.load()->DeferredAAs;
1359
1360 ExecutorAddr HeaderAddr;
1361 {
1362 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1363 auto I = MP.JITDylibToHeaderAddr.find(&JD);
1364 assert(I != MP.JITDylibToHeaderAddr.end() &&
1365 "No header registered for JD");
1366 assert(I->second && "Null header registered for JD");
1367 HeaderAddr = I->second;
1368 }
1369 allocActions.push_back(
1370 {cantFail(
1371 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
1372 MP.RegisterObjectPlatformSections.Addr, HeaderAddr, UnwindInfo,
1373 MachOPlatformSecs)),
1374 cantFail(
1375 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
1376 MP.DeregisterObjectPlatformSections.Addr, HeaderAddr,
1377 UnwindInfo, MachOPlatformSecs))});
1378 }
1379
1380 return Error::success();
1381 }
1382
createObjCRuntimeObject(jitlink::LinkGraph & G)1383 Error MachOPlatform::MachOPlatformPlugin::createObjCRuntimeObject(
1384 jitlink::LinkGraph &G) {
1385
1386 bool NeedTextSegment = false;
1387 size_t NumRuntimeSections = 0;
1388
1389 for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData)
1390 if (G.findSectionByName(ObjCRuntimeSectionName))
1391 ++NumRuntimeSections;
1392
1393 for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) {
1394 if (G.findSectionByName(ObjCRuntimeSectionName)) {
1395 ++NumRuntimeSections;
1396 NeedTextSegment = true;
1397 }
1398 }
1399
1400 // Early out for no runtime sections.
1401 if (NumRuntimeSections == 0)
1402 return Error::success();
1403
1404 // If there were any runtime sections then we need to add an __objc_imageinfo
1405 // section.
1406 ++NumRuntimeSections;
1407
1408 size_t MachOSize = sizeof(MachO::mach_header_64) +
1409 (NeedTextSegment + 1) * sizeof(MachO::segment_command_64) +
1410 NumRuntimeSections * sizeof(MachO::section_64);
1411
1412 auto &Sec = G.createSection(ObjCRuntimeObjectSectionName,
1413 MemProt::Read | MemProt::Write);
1414 G.createMutableContentBlock(Sec, MachOSize, ExecutorAddr(), 16, 0, true);
1415
1416 return Error::success();
1417 }
1418
populateObjCRuntimeObject(jitlink::LinkGraph & G,MaterializationResponsibility & MR)1419 Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject(
1420 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
1421
1422 auto *ObjCRuntimeObjectSec =
1423 G.findSectionByName(ObjCRuntimeObjectSectionName);
1424
1425 if (!ObjCRuntimeObjectSec)
1426 return Error::success();
1427
1428 switch (G.getTargetTriple().getArch()) {
1429 case Triple::aarch64:
1430 case Triple::x86_64:
1431 // Supported.
1432 break;
1433 default:
1434 return make_error<StringError>("Unrecognized MachO arch in triple " +
1435 G.getTargetTriple().str(),
1436 inconvertibleErrorCode());
1437 }
1438
1439 auto &SecBlock = **ObjCRuntimeObjectSec->blocks().begin();
1440
1441 struct SecDesc {
1442 MachO::section_64 Sec;
1443 unique_function<void(size_t RecordOffset)> AddFixups;
1444 };
1445
1446 std::vector<SecDesc> TextSections, DataSections;
1447 auto AddSection = [&](SecDesc &SD, jitlink::Section &GraphSec) {
1448 jitlink::SectionRange SR(GraphSec);
1449 StringRef FQName = GraphSec.getName();
1450 memset(&SD.Sec, 0, sizeof(MachO::section_64));
1451 memcpy(SD.Sec.sectname, FQName.drop_front(7).data(), FQName.size() - 7);
1452 memcpy(SD.Sec.segname, FQName.data(), 6);
1453 SD.Sec.addr = SR.getStart() - SecBlock.getAddress();
1454 SD.Sec.size = SR.getSize();
1455 SD.Sec.flags = MachO::S_REGULAR;
1456 };
1457
1458 // Add the __objc_imageinfo section.
1459 {
1460 DataSections.push_back({});
1461 auto &SD = DataSections.back();
1462 memset(&SD.Sec, 0, sizeof(SD.Sec));
1463 memcpy(SD.Sec.sectname, "__objc_imageinfo", 16);
1464 strcpy(SD.Sec.segname, "__DATA");
1465 SD.Sec.size = 8;
1466 SD.AddFixups = [&](size_t RecordOffset) {
1467 auto PointerEdge = getPointerEdgeKind(G);
1468
1469 // Look for an existing __objc_imageinfo symbol.
1470 jitlink::Symbol *ObjCImageInfoSym = nullptr;
1471 for (auto *Sym : G.external_symbols())
1472 if (Sym->getName() == ObjCImageInfoSymbolName) {
1473 ObjCImageInfoSym = Sym;
1474 break;
1475 }
1476 if (!ObjCImageInfoSym)
1477 for (auto *Sym : G.absolute_symbols())
1478 if (Sym->getName() == ObjCImageInfoSymbolName) {
1479 ObjCImageInfoSym = Sym;
1480 break;
1481 }
1482 if (!ObjCImageInfoSym)
1483 for (auto *Sym : G.defined_symbols())
1484 if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) {
1485 ObjCImageInfoSym = Sym;
1486 std::optional<uint32_t> Flags;
1487 {
1488 std::lock_guard<std::mutex> Lock(PluginMutex);
1489 auto It = ObjCImageInfos.find(&MR.getTargetJITDylib());
1490 if (It != ObjCImageInfos.end()) {
1491 It->second.Finalized = true;
1492 Flags = It->second.Flags;
1493 }
1494 }
1495
1496 if (Flags) {
1497 // We own the definition of __objc_image_info; write the final
1498 // merged flags value.
1499 auto Content = Sym->getBlock().getMutableContent(G);
1500 assert(Content.size() == 8 &&
1501 "__objc_image_info size should have been verified already");
1502 support::endian::write32(&Content[4], *Flags, G.getEndianness());
1503 }
1504 break;
1505 }
1506 if (!ObjCImageInfoSym)
1507 ObjCImageInfoSym =
1508 &G.addExternalSymbol(ObjCImageInfoSymbolName, 8, false);
1509
1510 SecBlock.addEdge(PointerEdge,
1511 RecordOffset + ((char *)&SD.Sec.addr - (char *)&SD.Sec),
1512 *ObjCImageInfoSym, -SecBlock.getAddress().getValue());
1513 };
1514 }
1515
1516 for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) {
1517 if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) {
1518 DataSections.push_back({});
1519 AddSection(DataSections.back(), *GraphSec);
1520 }
1521 }
1522
1523 for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) {
1524 if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) {
1525 TextSections.push_back({});
1526 AddSection(TextSections.back(), *GraphSec);
1527 }
1528 }
1529
1530 assert(ObjCRuntimeObjectSec->blocks_size() == 1 &&
1531 "Unexpected number of blocks in runtime sections object");
1532
1533 // Build the header struct up-front. This also gives us a chance to check
1534 // that the triple is supported, which we'll assume below.
1535 MachO::mach_header_64 Hdr;
1536 Hdr.magic = MachO::MH_MAGIC_64;
1537 switch (G.getTargetTriple().getArch()) {
1538 case Triple::aarch64:
1539 Hdr.cputype = MachO::CPU_TYPE_ARM64;
1540 Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL;
1541 break;
1542 case Triple::x86_64:
1543 Hdr.cputype = MachO::CPU_TYPE_X86_64;
1544 Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL;
1545 break;
1546 default:
1547 llvm_unreachable("Unsupported architecture");
1548 }
1549
1550 Hdr.filetype = MachO::MH_DYLIB;
1551 Hdr.ncmds = 1 + !TextSections.empty();
1552 Hdr.sizeofcmds =
1553 Hdr.ncmds * sizeof(MachO::segment_command_64) +
1554 (TextSections.size() + DataSections.size()) * sizeof(MachO::section_64);
1555 Hdr.flags = 0;
1556 Hdr.reserved = 0;
1557
1558 auto SecContent = SecBlock.getAlreadyMutableContent();
1559 char *P = SecContent.data();
1560 auto WriteMachOStruct = [&](auto S) {
1561 if (G.getEndianness() != llvm::endianness::native)
1562 MachO::swapStruct(S);
1563 memcpy(P, &S, sizeof(S));
1564 P += sizeof(S);
1565 };
1566
1567 auto WriteSegment = [&](StringRef Name, std::vector<SecDesc> &Secs) {
1568 MachO::segment_command_64 SegLC;
1569 memset(&SegLC, 0, sizeof(SegLC));
1570 memcpy(SegLC.segname, Name.data(), Name.size());
1571 SegLC.cmd = MachO::LC_SEGMENT_64;
1572 SegLC.cmdsize = sizeof(MachO::segment_command_64) +
1573 Secs.size() * sizeof(MachO::section_64);
1574 SegLC.nsects = Secs.size();
1575 WriteMachOStruct(SegLC);
1576 for (auto &SD : Secs) {
1577 if (SD.AddFixups)
1578 SD.AddFixups(P - SecContent.data());
1579 WriteMachOStruct(SD.Sec);
1580 }
1581 };
1582
1583 WriteMachOStruct(Hdr);
1584 if (!TextSections.empty())
1585 WriteSegment("__TEXT", TextSections);
1586 if (!DataSections.empty())
1587 WriteSegment("__DATA", DataSections);
1588
1589 assert(P == SecContent.end() && "Underflow writing ObjC runtime object");
1590 return Error::success();
1591 }
1592
prepareSymbolTableRegistration(jitlink::LinkGraph & G,JITSymTabVector & JITSymTabInfo)1593 Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration(
1594 jitlink::LinkGraph &G, JITSymTabVector &JITSymTabInfo) {
1595
1596 auto *CStringSec = G.findSectionByName(MachOCStringSectionName);
1597 if (!CStringSec)
1598 CStringSec = &G.createSection(MachOCStringSectionName,
1599 MemProt::Read | MemProt::Exec);
1600
1601 // Make a map of existing strings so that we can re-use them:
1602 DenseMap<StringRef, jitlink::Symbol *> ExistingStrings;
1603 for (auto *Sym : CStringSec->symbols()) {
1604
1605 // The LinkGraph builder should have created single strings blocks, and all
1606 // plugins should have maintained this invariant.
1607 auto Content = Sym->getBlock().getContent();
1608 ExistingStrings.insert(
1609 std::make_pair(StringRef(Content.data(), Content.size()), Sym));
1610 }
1611
1612 // Add all symbol names to the string section, and record the symbols for
1613 // those names.
1614 {
1615 SmallVector<jitlink::Symbol *> SymsToProcess;
1616 for (auto *Sym : G.defined_symbols())
1617 SymsToProcess.push_back(Sym);
1618 for (auto *Sym : G.absolute_symbols())
1619 SymsToProcess.push_back(Sym);
1620
1621 for (auto *Sym : SymsToProcess) {
1622 if (!Sym->hasName())
1623 continue;
1624
1625 auto I = ExistingStrings.find(Sym->getName());
1626 if (I == ExistingStrings.end()) {
1627 auto &NameBlock = G.createMutableContentBlock(
1628 *CStringSec, G.allocateCString(Sym->getName()), orc::ExecutorAddr(),
1629 1, 0);
1630 auto &SymbolNameSym = G.addAnonymousSymbol(
1631 NameBlock, 0, NameBlock.getSize(), false, true);
1632 JITSymTabInfo.push_back({Sym, &SymbolNameSym});
1633 } else
1634 JITSymTabInfo.push_back({Sym, I->second});
1635 }
1636 }
1637
1638 return Error::success();
1639 }
1640
addSymbolTableRegistration(jitlink::LinkGraph & G,MaterializationResponsibility & MR,JITSymTabVector & JITSymTabInfo,bool InBootstrapPhase)1641 Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration(
1642 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
1643 JITSymTabVector &JITSymTabInfo, bool InBootstrapPhase) {
1644
1645 ExecutorAddr HeaderAddr;
1646 {
1647 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1648 auto I = MP.JITDylibToHeaderAddr.find(&MR.getTargetJITDylib());
1649 assert(I != MP.JITDylibToHeaderAddr.end() && "No header registered for JD");
1650 assert(I->second && "Null header registered for JD");
1651 HeaderAddr = I->second;
1652 }
1653
1654 SymbolTableVector LocalSymTab;
1655 auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab
1656 : MP.Bootstrap.load()->SymTab;
1657 for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo)
1658 SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(),
1659 flagsForSymbol(*OriginalSymbol)});
1660
1661 // Bail out if we're in the bootstrap phase -- registration of thees symbols
1662 // will be attached to the bootstrap graph.
1663 if (LLVM_UNLIKELY(InBootstrapPhase))
1664 return Error::success();
1665
1666 shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase)
1667 ? G.allocActions()
1668 : MP.Bootstrap.load()->DeferredAAs;
1669 allocActions.push_back(
1670 {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
1671 MP.RegisterObjectSymbolTable.Addr, HeaderAddr, SymTab)),
1672 cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>(
1673 MP.DeregisterObjectSymbolTable.Addr, HeaderAddr, SymTab))});
1674
1675 return Error::success();
1676 }
1677
1678 template <typename MachOTraits>
createHeaderBlock(MachOPlatform & MOP,const MachOPlatform::HeaderOptions & Opts,JITDylib & JD,jitlink::LinkGraph & G,jitlink::Section & HeaderSection)1679 jitlink::Block &createHeaderBlock(MachOPlatform &MOP,
1680 const MachOPlatform::HeaderOptions &Opts,
1681 JITDylib &JD, jitlink::LinkGraph &G,
1682 jitlink::Section &HeaderSection) {
1683 auto HdrInfo =
1684 getMachOHeaderInfoFromTriple(MOP.getExecutionSession().getTargetTriple());
1685 MachOBuilder<MachOTraits> B(HdrInfo.PageSize);
1686
1687 B.Header.filetype = MachO::MH_DYLIB;
1688 B.Header.cputype = HdrInfo.CPUType;
1689 B.Header.cpusubtype = HdrInfo.CPUSubType;
1690
1691 if (Opts.IDDylib)
1692 B.template addLoadCommand<MachO::LC_ID_DYLIB>(
1693 Opts.IDDylib->Name, Opts.IDDylib->Timestamp,
1694 Opts.IDDylib->CurrentVersion, Opts.IDDylib->CompatibilityVersion);
1695 else
1696 B.template addLoadCommand<MachO::LC_ID_DYLIB>(JD.getName(), 0, 0, 0);
1697
1698 for (auto &D : Opts.LoadDylibs)
1699 B.template addLoadCommand<MachO::LC_LOAD_DYLIB>(
1700 D.Name, D.Timestamp, D.CurrentVersion, D.CompatibilityVersion);
1701 for (auto &P : Opts.RPaths)
1702 B.template addLoadCommand<MachO::LC_RPATH>(P);
1703
1704 auto HeaderContent = G.allocateBuffer(B.layout());
1705 B.write(HeaderContent);
1706
1707 return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
1708 0);
1709 }
1710
SimpleMachOHeaderMU(MachOPlatform & MOP,SymbolStringPtr HeaderStartSymbol,MachOPlatform::HeaderOptions Opts)1711 SimpleMachOHeaderMU::SimpleMachOHeaderMU(MachOPlatform &MOP,
1712 SymbolStringPtr HeaderStartSymbol,
1713 MachOPlatform::HeaderOptions Opts)
1714 : MaterializationUnit(
1715 createHeaderInterface(MOP, std::move(HeaderStartSymbol))),
1716 MOP(MOP), Opts(std::move(Opts)) {}
1717
materialize(std::unique_ptr<MaterializationResponsibility> R)1718 void SimpleMachOHeaderMU::materialize(
1719 std::unique_ptr<MaterializationResponsibility> R) {
1720 auto G = createPlatformGraph(MOP, "<MachOHeaderMU>");
1721 addMachOHeader(R->getTargetJITDylib(), *G, R->getInitializerSymbol());
1722 MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
1723 }
1724
discard(const JITDylib & JD,const SymbolStringPtr & Sym)1725 void SimpleMachOHeaderMU::discard(const JITDylib &JD,
1726 const SymbolStringPtr &Sym) {}
1727
addMachOHeader(JITDylib & JD,jitlink::LinkGraph & G,const SymbolStringPtr & InitializerSymbol)1728 void SimpleMachOHeaderMU::addMachOHeader(
1729 JITDylib &JD, jitlink::LinkGraph &G,
1730 const SymbolStringPtr &InitializerSymbol) {
1731 auto &HeaderSection = G.createSection("__header", MemProt::Read);
1732 auto &HeaderBlock = createHeaderBlock(JD, G, HeaderSection);
1733
1734 // Init symbol is header-start symbol.
1735 G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, HeaderBlock.getSize(),
1736 jitlink::Linkage::Strong, jitlink::Scope::Default, false,
1737 true);
1738 for (auto &HS : AdditionalHeaderSymbols)
1739 G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(),
1740 jitlink::Linkage::Strong, jitlink::Scope::Default, false,
1741 true);
1742 }
1743
1744 jitlink::Block &
createHeaderBlock(JITDylib & JD,jitlink::LinkGraph & G,jitlink::Section & HeaderSection)1745 SimpleMachOHeaderMU::createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G,
1746 jitlink::Section &HeaderSection) {
1747 switch (MOP.getExecutionSession().getTargetTriple().getArch()) {
1748 case Triple::aarch64:
1749 case Triple::x86_64:
1750 return ::createHeaderBlock<MachO64LE>(MOP, Opts, JD, G, HeaderSection);
1751 default:
1752 llvm_unreachable("Unsupported architecture");
1753 }
1754 }
1755
createHeaderInterface(MachOPlatform & MOP,const SymbolStringPtr & HeaderStartSymbol)1756 MaterializationUnit::Interface SimpleMachOHeaderMU::createHeaderInterface(
1757 MachOPlatform &MOP, const SymbolStringPtr &HeaderStartSymbol) {
1758 SymbolFlagsMap HeaderSymbolFlags;
1759
1760 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
1761 for (auto &HS : AdditionalHeaderSymbols)
1762 HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] =
1763 JITSymbolFlags::Exported;
1764
1765 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
1766 HeaderStartSymbol);
1767 }
1768
getMachOHeaderInfoFromTriple(const Triple & TT)1769 MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT) {
1770 switch (TT.getArch()) {
1771 case Triple::aarch64:
1772 return {/* PageSize = */ 16 * 1024,
1773 /* CPUType = */ MachO::CPU_TYPE_ARM64,
1774 /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL};
1775 case Triple::x86_64:
1776 return {/* PageSize = */ 4 * 1024,
1777 /* CPUType = */ MachO::CPU_TYPE_X86_64,
1778 /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL};
1779 default:
1780 llvm_unreachable("Unrecognized architecture");
1781 }
1782 }
1783
1784 } // End namespace orc.
1785 } // End namespace llvm.
1786