1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10 
11 #include "llvm/BinaryFormat/ELF.h"
12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
14 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
15 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
16 #include "llvm/Support/BinaryByteStream.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "orc"
20 
21 using namespace llvm;
22 using namespace llvm::orc;
23 using namespace llvm::orc::shared;
24 
25 namespace {
26 
27 class DSOHandleMaterializationUnit : public MaterializationUnit {
28 public:
DSOHandleMaterializationUnit(ELFNixPlatform & ENP,const SymbolStringPtr & DSOHandleSymbol)29   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
30                                const SymbolStringPtr &DSOHandleSymbol)
31       : MaterializationUnit(createDSOHandleSectionSymbols(ENP, DSOHandleSymbol),
32                             DSOHandleSymbol),
33         ENP(ENP) {}
34 
getName() const35   StringRef getName() const override { return "DSOHandleMU"; }
36 
materialize(std::unique_ptr<MaterializationResponsibility> R)37   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
38     unsigned PointerSize;
39     support::endianness Endianness;
40     jitlink::Edge::Kind EdgeKind;
41     const auto &TT =
42         ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
43 
44     switch (TT.getArch()) {
45     case Triple::x86_64:
46       PointerSize = 8;
47       Endianness = support::endianness::little;
48       EdgeKind = jitlink::x86_64::Pointer64;
49       break;
50     default:
51       llvm_unreachable("Unrecognized architecture");
52     }
53 
54     // void *__dso_handle = &__dso_handle;
55     auto G = std::make_unique<jitlink::LinkGraph>(
56         "<DSOHandleMU>", TT, PointerSize, Endianness,
57         jitlink::getGenericEdgeKindName);
58     auto &DSOHandleSection =
59         G->createSection(".data.__dso_handle", jitlink::MemProt::Read);
60     auto &DSOHandleBlock = G->createContentBlock(
61         DSOHandleSection, getDSOHandleContent(PointerSize), 0, 8, 0);
62     auto &DSOHandleSymbol = G->addDefinedSymbol(
63         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
64         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
65     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
66 
67     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
68   }
69 
discard(const JITDylib & JD,const SymbolStringPtr & Sym)70   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
71 
72 private:
73   static SymbolFlagsMap
createDSOHandleSectionSymbols(ELFNixPlatform & ENP,const SymbolStringPtr & DSOHandleSymbol)74   createDSOHandleSectionSymbols(ELFNixPlatform &ENP,
75                                 const SymbolStringPtr &DSOHandleSymbol) {
76     SymbolFlagsMap SymbolFlags;
77     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
78     return SymbolFlags;
79   }
80 
getDSOHandleContent(size_t PointerSize)81   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
82     static const char Content[8] = {0};
83     assert(PointerSize <= sizeof Content);
84     return {Content, PointerSize};
85   }
86 
87   ELFNixPlatform &ENP;
88 };
89 
90 StringRef EHFrameSectionName = ".eh_frame";
91 StringRef InitArrayFuncSectionName = ".init_array";
92 
93 StringRef ThreadBSSSectionName = ".tbss";
94 StringRef ThreadDataSectionName = ".tdata";
95 
96 StringRef InitSectionNames[] = {InitArrayFuncSectionName};
97 
98 } // end anonymous namespace
99 
100 namespace llvm {
101 namespace orc {
102 
103 Expected<std::unique_ptr<ELFNixPlatform>>
Create(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,Optional<SymbolAliasMap> RuntimeAliases)104 ELFNixPlatform::Create(ExecutionSession &ES,
105                        ObjectLinkingLayer &ObjLinkingLayer,
106                        JITDylib &PlatformJD, const char *OrcRuntimePath,
107                        Optional<SymbolAliasMap> RuntimeAliases) {
108 
109   auto &EPC = ES.getExecutorProcessControl();
110 
111   // If the target is not supported then bail out immediately.
112   if (!supportedTarget(EPC.getTargetTriple()))
113     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
114                                        EPC.getTargetTriple().str(),
115                                    inconvertibleErrorCode());
116 
117   // Create default aliases if the caller didn't supply any.
118   if (!RuntimeAliases)
119     RuntimeAliases = standardPlatformAliases(ES);
120 
121   // Define the aliases.
122   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
123     return std::move(Err);
124 
125   // Add JIT-dispatch function support symbols.
126   if (auto Err = PlatformJD.define(absoluteSymbols(
127           {{ES.intern("__orc_rt_jit_dispatch"),
128             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
129              JITSymbolFlags::Exported}},
130            {ES.intern("__orc_rt_jit_dispatch_ctx"),
131             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
132              JITSymbolFlags::Exported}}})))
133     return std::move(Err);
134 
135   // Create a generator for the ORC runtime archive.
136   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
137       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
138   if (!OrcRuntimeArchiveGenerator)
139     return OrcRuntimeArchiveGenerator.takeError();
140 
141   // Create the instance.
142   Error Err = Error::success();
143   auto P = std::unique_ptr<ELFNixPlatform>(
144       new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD,
145                          std::move(*OrcRuntimeArchiveGenerator), Err));
146   if (Err)
147     return std::move(Err);
148   return std::move(P);
149 }
150 
setupJITDylib(JITDylib & JD)151 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
152   return JD.define(
153       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
154   return Error::success();
155 }
156 
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)157 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
158                                    const MaterializationUnit &MU) {
159   auto &JD = RT.getJITDylib();
160   const auto &InitSym = MU.getInitializerSymbol();
161   if (!InitSym)
162     return Error::success();
163 
164   RegisteredInitSymbols[&JD].add(InitSym,
165                                  SymbolLookupFlags::WeaklyReferencedSymbol);
166   LLVM_DEBUG({
167     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
168            << " for MU " << MU.getName() << "\n";
169   });
170   return Error::success();
171 }
172 
notifyRemoving(ResourceTracker & RT)173 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
174   llvm_unreachable("Not supported yet");
175 }
176 
addAliases(ExecutionSession & ES,SymbolAliasMap & Aliases,ArrayRef<std::pair<const char *,const char * >> AL)177 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
178                        ArrayRef<std::pair<const char *, const char *>> AL) {
179   for (auto &KV : AL) {
180     auto AliasName = ES.intern(KV.first);
181     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
182     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
183                                      JITSymbolFlags::Exported};
184   }
185 }
186 
standardPlatformAliases(ExecutionSession & ES)187 SymbolAliasMap ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES) {
188   SymbolAliasMap Aliases;
189   addAliases(ES, Aliases, requiredCXXAliases());
190   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
191   return Aliases;
192 }
193 
194 ArrayRef<std::pair<const char *, const char *>>
requiredCXXAliases()195 ELFNixPlatform::requiredCXXAliases() {
196   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
197       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
198       {"atexit", "__orc_rt_elfnix_atexit"}};
199 
200   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
201 }
202 
203 ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases()204 ELFNixPlatform::standardRuntimeUtilityAliases() {
205   static const std::pair<const char *, const char *>
206       StandardRuntimeUtilityAliases[] = {
207           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
208           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
209 
210   return ArrayRef<std::pair<const char *, const char *>>(
211       StandardRuntimeUtilityAliases);
212 }
213 
isInitializerSection(StringRef SecName)214 bool ELFNixPlatform::isInitializerSection(StringRef SecName) {
215   for (auto &Name : InitSectionNames) {
216     if (Name.equals(SecName))
217       return true;
218   }
219   return false;
220 }
221 
supportedTarget(const Triple & TT)222 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
223   switch (TT.getArch()) {
224   case Triple::x86_64:
225     return true;
226   default:
227     return false;
228   }
229 }
230 
ELFNixPlatform(ExecutionSession & ES,ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,Error & Err)231 ELFNixPlatform::ELFNixPlatform(
232     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
233     JITDylib &PlatformJD,
234     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
235     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
236       DSOHandleSymbol(ES.intern("__dso_handle")) {
237   ErrorAsOutParameter _(&Err);
238 
239   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
240 
241   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
242 
243   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
244   // the platform now), so set it up.
245   if (auto E2 = setupJITDylib(PlatformJD)) {
246     Err = std::move(E2);
247     return;
248   }
249 
250   RegisteredInitSymbols[&PlatformJD].add(
251       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
252 
253   // Associate wrapper function tags with JIT-side function implementations.
254   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
255     Err = std::move(E2);
256     return;
257   }
258 
259   // Lookup addresses of runtime functions callable by the platform,
260   // call the platform bootstrap function to initialize the platform-state
261   // object in the executor.
262   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
263     Err = std::move(E2);
264     return;
265   }
266 }
267 
associateRuntimeSupportFunctions(JITDylib & PlatformJD)268 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
269   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
270 
271   using GetInitializersSPSSig =
272       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
273   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
274       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
275           this, &ELFNixPlatform::rt_getInitializers);
276 
277   using GetDeinitializersSPSSig =
278       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
279   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
280       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
281           this, &ELFNixPlatform::rt_getDeinitializers);
282 
283   using LookupSymbolSPSSig =
284       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
285   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
286       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
287                                               &ELFNixPlatform::rt_lookupSymbol);
288 
289   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
290 }
291 
getInitializersBuildSequencePhase(SendInitializerSequenceFn SendResult,JITDylib & JD,std::vector<JITDylibSP> DFSLinkOrder)292 void ELFNixPlatform::getInitializersBuildSequencePhase(
293     SendInitializerSequenceFn SendResult, JITDylib &JD,
294     std::vector<JITDylibSP> DFSLinkOrder) {
295   ELFNixJITDylibInitializerSequence FullInitSeq;
296   {
297     std::lock_guard<std::mutex> Lock(PlatformMutex);
298     for (auto &InitJD : reverse(DFSLinkOrder)) {
299       LLVM_DEBUG({
300         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
301                << "\" to sequence\n";
302       });
303       auto ISItr = InitSeqs.find(InitJD.get());
304       if (ISItr != InitSeqs.end()) {
305         FullInitSeq.emplace_back(std::move(ISItr->second));
306         InitSeqs.erase(ISItr);
307       }
308     }
309   }
310 
311   SendResult(std::move(FullInitSeq));
312 }
313 
getInitializersLookupPhase(SendInitializerSequenceFn SendResult,JITDylib & JD)314 void ELFNixPlatform::getInitializersLookupPhase(
315     SendInitializerSequenceFn SendResult, JITDylib &JD) {
316 
317   auto DFSLinkOrder = JD.getDFSLinkOrder();
318   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
319   ES.runSessionLocked([&]() {
320     for (auto &InitJD : DFSLinkOrder) {
321       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
322       if (RISItr != RegisteredInitSymbols.end()) {
323         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
324         RegisteredInitSymbols.erase(RISItr);
325       }
326     }
327   });
328 
329   // If there are no further init symbols to look up then move on to the next
330   // phase.
331   if (NewInitSymbols.empty()) {
332     getInitializersBuildSequencePhase(std::move(SendResult), JD,
333                                       std::move(DFSLinkOrder));
334     return;
335   }
336 
337   // Otherwise issue a lookup and re-run this phase when it completes.
338   lookupInitSymbolsAsync(
339       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
340         if (Err)
341           SendResult(std::move(Err));
342         else
343           getInitializersLookupPhase(std::move(SendResult), JD);
344       },
345       ES, std::move(NewInitSymbols));
346 }
347 
rt_getInitializers(SendInitializerSequenceFn SendResult,StringRef JDName)348 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
349                                         StringRef JDName) {
350   LLVM_DEBUG({
351     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
352   });
353 
354   JITDylib *JD = ES.getJITDylibByName(JDName);
355   if (!JD) {
356     LLVM_DEBUG({
357       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
358     });
359     SendResult(make_error<StringError>("No JITDylib named " + JDName,
360                                        inconvertibleErrorCode()));
361     return;
362   }
363 
364   getInitializersLookupPhase(std::move(SendResult), *JD);
365 }
366 
rt_getDeinitializers(SendDeinitializerSequenceFn SendResult,ExecutorAddr Handle)367 void ELFNixPlatform::rt_getDeinitializers(
368     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
369   LLVM_DEBUG({
370     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\""
371            << formatv("{0:x}", Handle.getValue()) << "\")\n";
372   });
373 
374   JITDylib *JD = nullptr;
375 
376   {
377     std::lock_guard<std::mutex> Lock(PlatformMutex);
378     auto I = HandleAddrToJITDylib.find(Handle.getValue());
379     if (I != HandleAddrToJITDylib.end())
380       JD = I->second;
381   }
382 
383   if (!JD) {
384     LLVM_DEBUG({
385       dbgs() << "  No JITDylib for handle "
386              << formatv("{0:x}", Handle.getValue()) << "\n";
387     });
388     SendResult(make_error<StringError>("No JITDylib associated with handle " +
389                                            formatv("{0:x}", Handle.getValue()),
390                                        inconvertibleErrorCode()));
391     return;
392   }
393 
394   SendResult(ELFNixJITDylibDeinitializerSequence());
395 }
396 
rt_lookupSymbol(SendSymbolAddressFn SendResult,ExecutorAddr Handle,StringRef SymbolName)397 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
398                                      ExecutorAddr Handle,
399                                      StringRef SymbolName) {
400   LLVM_DEBUG({
401     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\""
402            << formatv("{0:x}", Handle.getValue()) << "\")\n";
403   });
404 
405   JITDylib *JD = nullptr;
406 
407   {
408     std::lock_guard<std::mutex> Lock(PlatformMutex);
409     auto I = HandleAddrToJITDylib.find(Handle.getValue());
410     if (I != HandleAddrToJITDylib.end())
411       JD = I->second;
412   }
413 
414   if (!JD) {
415     LLVM_DEBUG({
416       dbgs() << "  No JITDylib for handle "
417              << formatv("{0:x}", Handle.getValue()) << "\n";
418     });
419     SendResult(make_error<StringError>("No JITDylib associated with handle " +
420                                            formatv("{0:x}", Handle.getValue()),
421                                        inconvertibleErrorCode()));
422     return;
423   }
424 
425   // Use functor class to work around XL build compiler issue on AIX.
426   class RtLookupNotifyComplete {
427   public:
428     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
429         : SendResult(std::move(SendResult)) {}
430     void operator()(Expected<SymbolMap> Result) {
431       if (Result) {
432         assert(Result->size() == 1 && "Unexpected result map count");
433         SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
434       } else {
435         SendResult(Result.takeError());
436       }
437     }
438 
439   private:
440     SendSymbolAddressFn SendResult;
441   };
442 
443   ES.lookup(
444       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
445       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
446       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
447 }
448 
bootstrapELFNixRuntime(JITDylib & PlatformJD)449 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
450 
451   std::pair<const char *, ExecutorAddr *> Symbols[] = {
452       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
453       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
454       {"__orc_rt_elfnix_register_object_sections",
455        &orc_rt_elfnix_register_object_sections},
456       {"__orc_rt_elfnix_create_pthread_key",
457        &orc_rt_elfnix_create_pthread_key}};
458 
459   SymbolLookupSet RuntimeSymbols;
460   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
461   for (const auto &KV : Symbols) {
462     auto Name = ES.intern(KV.first);
463     RuntimeSymbols.add(Name);
464     AddrsToRecord.push_back({std::move(Name), KV.second});
465   }
466 
467   auto RuntimeSymbolAddrs = ES.lookup(
468       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
469   if (!RuntimeSymbolAddrs)
470     return RuntimeSymbolAddrs.takeError();
471 
472   for (const auto &KV : AddrsToRecord) {
473     auto &Name = KV.first;
474     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
475     KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress());
476   }
477 
478   auto PJDDSOHandle = ES.lookup(
479       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
480   if (!PJDDSOHandle)
481     return PJDDSOHandle.takeError();
482 
483   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
484           orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress()))
485     return Err;
486 
487   // FIXME: Ordering is fuzzy here. We're probably best off saying
488   // "behavior is undefined if code that uses the runtime is added before
489   // the platform constructor returns", then move all this to the constructor.
490   RuntimeBootstrapped = true;
491   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
492   {
493     std::lock_guard<std::mutex> Lock(PlatformMutex);
494     DeferredPOSRs = std::move(BootstrapPOSRs);
495   }
496 
497   for (auto &D : DeferredPOSRs)
498     if (auto Err = registerPerObjectSections(D))
499       return Err;
500 
501   return Error::success();
502 }
503 
registerInitInfo(JITDylib & JD,ArrayRef<jitlink::Section * > InitSections)504 Error ELFNixPlatform::registerInitInfo(
505     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
506 
507   std::unique_lock<std::mutex> Lock(PlatformMutex);
508 
509   ELFNixJITDylibInitializers *InitSeq = nullptr;
510   {
511     auto I = InitSeqs.find(&JD);
512     if (I == InitSeqs.end()) {
513       // If there's no init sequence entry yet then we need to look up the
514       // header symbol to force creation of one.
515       Lock.unlock();
516 
517       auto SearchOrder =
518           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
519       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
520         return Err;
521 
522       Lock.lock();
523       I = InitSeqs.find(&JD);
524       assert(I != InitSeqs.end() &&
525              "Entry missing after header symbol lookup?");
526     }
527     InitSeq = &I->second;
528   }
529 
530   for (auto *Sec : InitSections) {
531     // FIXME: Avoid copy here.
532     jitlink::SectionRange R(*Sec);
533     InitSeq->InitSections[Sec->getName()].push_back(
534         {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())});
535   }
536 
537   return Error::success();
538 }
539 
registerPerObjectSections(const ELFPerObjectSectionsToRegister & POSR)540 Error ELFNixPlatform::registerPerObjectSections(
541     const ELFPerObjectSectionsToRegister &POSR) {
542 
543   if (!orc_rt_elfnix_register_object_sections)
544     return make_error<StringError>("Attempting to register per-object "
545                                    "sections, but runtime support has not "
546                                    "been loaded yet",
547                                    inconvertibleErrorCode());
548 
549   Error ErrResult = Error::success();
550   if (auto Err = ES.callSPSWrapper<shared::SPSError(
551                      SPSELFPerObjectSectionsToRegister)>(
552           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
553     return Err;
554   return ErrResult;
555 }
556 
createPThreadKey()557 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
558   if (!orc_rt_elfnix_create_pthread_key)
559     return make_error<StringError>(
560         "Attempting to create pthread key in target, but runtime support has "
561         "not been loaded yet",
562         inconvertibleErrorCode());
563 
564   Expected<uint64_t> Result(0);
565   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
566           orc_rt_elfnix_create_pthread_key, Result))
567     return std::move(Err);
568   return Result;
569 }
570 
modifyPassConfig(MaterializationResponsibility & MR,jitlink::LinkGraph & LG,jitlink::PassConfiguration & Config)571 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
572     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
573     jitlink::PassConfiguration &Config) {
574 
575   // If the initializer symbol is the __dso_handle symbol then just add
576   // the DSO handle support passes.
577   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
578     addDSOHandleSupportPasses(MR, Config);
579     // The DSOHandle materialization unit doesn't require any other
580     // support, so we can bail out early.
581     return;
582   }
583 
584   // If the object contains initializers then add passes to record them.
585   if (MR.getInitializerSymbol())
586     addInitializerSupportPasses(MR, Config);
587 
588   // Add passes for eh-frame and TLV support.
589   addEHAndTLVSupportPasses(MR, Config);
590 }
591 
592 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
getSyntheticSymbolDependencies(MaterializationResponsibility & MR)593 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
594     MaterializationResponsibility &MR) {
595   std::lock_guard<std::mutex> Lock(PluginMutex);
596   auto I = InitSymbolDeps.find(&MR);
597   if (I != InitSymbolDeps.end()) {
598     SyntheticSymbolDependenciesMap Result;
599     Result[MR.getInitializerSymbol()] = std::move(I->second);
600     InitSymbolDeps.erase(&MR);
601     return Result;
602   }
603   return SyntheticSymbolDependenciesMap();
604 }
605 
addInitializerSupportPasses(MaterializationResponsibility & MR,jitlink::PassConfiguration & Config)606 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
607     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
608 
609   /// Preserve init sections.
610   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
611     if (auto Err = preserveInitSections(G, MR))
612       return Err;
613     return Error::success();
614   });
615 
616   Config.PostFixupPasses.push_back(
617       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
618         return registerInitSections(G, JD);
619       });
620 }
621 
addDSOHandleSupportPasses(MaterializationResponsibility & MR,jitlink::PassConfiguration & Config)622 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
623     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
624 
625   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
626                                             jitlink::LinkGraph &G) -> Error {
627     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
628       return Sym->getName() == *MP.DSOHandleSymbol;
629     });
630     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
631     {
632       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
633       JITTargetAddress HandleAddr = (*I)->getAddress();
634       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
635       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
636       MP.InitSeqs.insert(std::make_pair(
637           &JD,
638           ELFNixJITDylibInitializers(JD.getName(), ExecutorAddr(HandleAddr))));
639     }
640     return Error::success();
641   });
642 }
643 
addEHAndTLVSupportPasses(MaterializationResponsibility & MR,jitlink::PassConfiguration & Config)644 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
645     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
646 
647   // Insert TLV lowering at the start of the PostPrunePasses, since we want
648   // it to run before GOT/PLT lowering.
649 
650   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
651   // pass has done. Because the TLS descriptor need to be allocate in GOT.
652   Config.PostPrunePasses.push_back(
653       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
654         return fixTLVSectionsAndEdges(G, JD);
655       });
656 
657   // Add a pass to register the final addresses of the eh-frame and TLV sections
658   // with the runtime.
659   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
660     ELFPerObjectSectionsToRegister POSR;
661 
662     if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
663       jitlink::SectionRange R(*EHFrameSection);
664       if (!R.empty())
665         POSR.EHFrameSection = {ExecutorAddr(R.getStart()),
666                                ExecutorAddr(R.getEnd())};
667     }
668 
669     // Get a pointer to the thread data section if there is one. It will be used
670     // below.
671     jitlink::Section *ThreadDataSection =
672         G.findSectionByName(ThreadDataSectionName);
673 
674     // Handle thread BSS section if there is one.
675     if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
676       // If there's already a thread data section in this graph then merge the
677       // thread BSS section content into it, otherwise just treat the thread
678       // BSS section as the thread data section.
679       if (ThreadDataSection)
680         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
681       else
682         ThreadDataSection = ThreadBSSSection;
683     }
684 
685     // Having merged thread BSS (if present) and thread data (if present),
686     // record the resulting section range.
687     if (ThreadDataSection) {
688       jitlink::SectionRange R(*ThreadDataSection);
689       if (!R.empty())
690         POSR.ThreadDataSection = {ExecutorAddr(R.getStart()),
691                                   ExecutorAddr(R.getEnd())};
692     }
693 
694     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
695 
696       // If we're still bootstrapping the runtime then just record this
697       // frame for now.
698       if (!MP.RuntimeBootstrapped) {
699         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
700         MP.BootstrapPOSRs.push_back(POSR);
701         return Error::success();
702       }
703 
704       // Otherwise register it immediately.
705       if (auto Err = MP.registerPerObjectSections(POSR))
706         return Err;
707     }
708 
709     return Error::success();
710   });
711 }
712 
preserveInitSections(jitlink::LinkGraph & G,MaterializationResponsibility & MR)713 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
714     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
715 
716   JITLinkSymbolSet InitSectionSymbols;
717   for (auto &InitSectionName : InitSectionNames) {
718     // Skip non-init sections.
719     auto *InitSection = G.findSectionByName(InitSectionName);
720     if (!InitSection)
721       continue;
722 
723     // Make a pass over live symbols in the section: those blocks are already
724     // preserved.
725     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
726     for (auto &Sym : InitSection->symbols()) {
727       auto &B = Sym->getBlock();
728       if (Sym->isLive() && Sym->getOffset() == 0 &&
729           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
730         InitSectionSymbols.insert(Sym);
731         AlreadyLiveBlocks.insert(&B);
732       }
733     }
734 
735     // Add anonymous symbols to preserve any not-already-preserved blocks.
736     for (auto *B : InitSection->blocks())
737       if (!AlreadyLiveBlocks.count(B))
738         InitSectionSymbols.insert(
739             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
740   }
741 
742   if (!InitSectionSymbols.empty()) {
743     std::lock_guard<std::mutex> Lock(PluginMutex);
744     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
745   }
746 
747   return Error::success();
748 }
749 
registerInitSections(jitlink::LinkGraph & G,JITDylib & JD)750 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
751     jitlink::LinkGraph &G, JITDylib &JD) {
752 
753   SmallVector<jitlink::Section *> InitSections;
754 
755   LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; });
756 
757   for (auto InitSectionName : InitSectionNames) {
758     if (auto *Sec = G.findSectionByName(InitSectionName)) {
759       InitSections.push_back(Sec);
760     }
761   }
762 
763   // Dump the scraped inits.
764   LLVM_DEBUG({
765     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
766     for (auto *Sec : InitSections) {
767       jitlink::SectionRange R(*Sec);
768       dbgs() << "  " << Sec->getName() << ": "
769              << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
770     }
771   });
772 
773   return MP.registerInitInfo(JD, InitSections);
774 }
775 
fixTLVSectionsAndEdges(jitlink::LinkGraph & G,JITDylib & JD)776 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
777     jitlink::LinkGraph &G, JITDylib &JD) {
778 
779   // TODO implement TLV support
780   for (auto *Sym : G.external_symbols())
781     if (Sym->getName() == "__tls_get_addr") {
782       Sym->setName("___orc_rt_elfnix_tls_get_addr");
783     }
784 
785   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
786 
787   if (TLSInfoEntrySection) {
788     Optional<uint64_t> Key;
789     {
790       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
791       auto I = MP.JITDylibToPThreadKey.find(&JD);
792       if (I != MP.JITDylibToPThreadKey.end())
793         Key = I->second;
794     }
795     if (!Key) {
796       if (auto KeyOrErr = MP.createPThreadKey())
797         Key = *KeyOrErr;
798       else
799         return KeyOrErr.takeError();
800     }
801 
802     uint64_t PlatformKeyBits =
803         support::endian::byte_swap(*Key, G.getEndianness());
804 
805     for (auto *B : TLSInfoEntrySection->blocks()) {
806       // FIXME: The TLS descriptor byte length may different with different
807       // ISA
808       assert(B->getSize() == (G.getPointerSize() * 2) &&
809              "TLS descriptor must be 2 words length");
810       auto TLSInfoEntryContent = B->getMutableContent(G);
811       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
812     }
813   }
814 
815   return Error::success();
816 }
817 
818 } // End namespace orc.
819 } // End namespace llvm.
820