1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
13 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
14 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
15 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
16 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
17 #include "llvm/ExecutionEngine/Orc/TargetProcessControl.h"
18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Mangler.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Support/DynamicLibrary.h"
24 
25 #include <map>
26 
27 #define DEBUG_TYPE "orc"
28 
29 using namespace llvm;
30 using namespace llvm::orc;
31 
32 namespace {
33 
34 /// Adds helper function decls and wrapper functions that call the helper with
35 /// some additional prefix arguments.
36 ///
37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
38 /// args i32 4 and i16 12345, this function will add:
39 ///
40 /// declare i8 @bar(i32, i16, i8, i64)
41 ///
42 /// define i8 @foo(i8, i64) {
43 /// entry:
44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
45 ///   ret i8 %2
46 /// }
47 ///
addHelperAndWrapper(Module & M,StringRef WrapperName,FunctionType * WrapperFnType,GlobalValue::VisibilityTypes WrapperVisibility,StringRef HelperName,ArrayRef<Value * > HelperPrefixArgs)48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
49                               FunctionType *WrapperFnType,
50                               GlobalValue::VisibilityTypes WrapperVisibility,
51                               StringRef HelperName,
52                               ArrayRef<Value *> HelperPrefixArgs) {
53   std::vector<Type *> HelperArgTypes;
54   for (auto *Arg : HelperPrefixArgs)
55     HelperArgTypes.push_back(Arg->getType());
56   for (auto *T : WrapperFnType->params())
57     HelperArgTypes.push_back(T);
58   auto *HelperFnType =
59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
61                                     HelperName, M);
62 
63   auto *WrapperFn = Function::Create(
64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
65   WrapperFn->setVisibility(WrapperVisibility);
66 
67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
68   IRBuilder<> IB(EntryBlock);
69 
70   std::vector<Value *> HelperArgs;
71   for (auto *Arg : HelperPrefixArgs)
72     HelperArgs.push_back(Arg);
73   for (auto &Arg : WrapperFn->args())
74     HelperArgs.push_back(&Arg);
75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
76   if (HelperFn->getReturnType()->isVoidTy())
77     IB.CreateRetVoid();
78   else
79     IB.CreateRet(HelperResult);
80 
81   return WrapperFn;
82 }
83 
84 class GenericLLVMIRPlatformSupport;
85 
86 /// orc::Platform component of Generic LLVM IR Platform support.
87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
88 class GenericLLVMIRPlatform : public Platform {
89 public:
GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport & S)90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
91   Error setupJITDylib(JITDylib &JD) override;
92   Error notifyAdding(ResourceTracker &RT,
93                      const MaterializationUnit &MU) override;
notifyRemoving(ResourceTracker & RT)94   Error notifyRemoving(ResourceTracker &RT) override {
95     // Noop -- Nothing to do (yet).
96     return Error::success();
97   }
98 
99 private:
100   GenericLLVMIRPlatformSupport &S;
101 };
102 
103 /// This transform parses llvm.global_ctors to produce a single initialization
104 /// function for the module, records the function, then deletes
105 /// llvm.global_ctors.
106 class GlobalCtorDtorScraper {
107 public:
108 
GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport & PS,StringRef InitFunctionPrefix)109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
110                         StringRef InitFunctionPrefix)
111     : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {}
112   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
113                                         MaterializationResponsibility &R);
114 
115 private:
116   GenericLLVMIRPlatformSupport &PS;
117   StringRef InitFunctionPrefix;
118 };
119 
120 /// Generic IR Platform Support
121 ///
122 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
123 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
124 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
125 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
126 public:
GenericLLVMIRPlatformSupport(LLJIT & J)127   GenericLLVMIRPlatformSupport(LLJIT &J)
128       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) {
129 
130     getExecutionSession().setPlatform(
131         std::make_unique<GenericLLVMIRPlatform>(*this));
132 
133     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix));
134 
135     SymbolMap StdInterposes;
136 
137     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
138         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
139                            JITSymbolFlags::Exported);
140     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
141         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
142                            JITSymbolFlags());
143 
144     cantFail(
145         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
146     cantFail(setupJITDylib(J.getMainJITDylib()));
147     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
148   }
149 
getExecutionSession()150   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
151 
152   /// Adds a module that defines the __dso_handle global.
setupJITDylib(JITDylib & JD)153   Error setupJITDylib(JITDylib &JD) {
154 
155     // Add per-jitdylib standard interposes.
156     SymbolMap PerJDInterposes;
157     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
158         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
159                            JITSymbolFlags());
160     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
161 
162     auto Ctx = std::make_unique<LLVMContext>();
163     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
164     M->setDataLayout(J.getDataLayout());
165 
166     auto *Int64Ty = Type::getInt64Ty(*Ctx);
167     auto *DSOHandle = new GlobalVariable(
168         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
169         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
170         "__dso_handle");
171     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
172     DSOHandle->setInitializer(
173         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
174 
175     auto *GenericIRPlatformSupportTy =
176         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
177 
178     auto *PlatformInstanceDecl = new GlobalVariable(
179         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
180         nullptr, "__lljit.platform_support_instance");
181 
182     auto *VoidTy = Type::getVoidTy(*Ctx);
183     addHelperAndWrapper(
184         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
185         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
186         {PlatformInstanceDecl, DSOHandle});
187 
188     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
189   }
190 
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)191   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
192     auto &JD = RT.getJITDylib();
193     if (auto &InitSym = MU.getInitializerSymbol())
194       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
195     else {
196       // If there's no identified init symbol attached, but there is a symbol
197       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
198       // an init function. Add the symbol to both the InitSymbols map (which
199       // will trigger a lookup to materialize the module) and the InitFunctions
200       // map (which holds the names of the symbols to execute).
201       for (auto &KV : MU.getSymbols())
202         if ((*KV.first).startswith(InitFunctionPrefix)) {
203           InitSymbols[&JD].add(KV.first,
204                                SymbolLookupFlags::WeaklyReferencedSymbol);
205           InitFunctions[&JD].add(KV.first);
206         }
207     }
208     return Error::success();
209   }
210 
initialize(JITDylib & JD)211   Error initialize(JITDylib &JD) override {
212     LLVM_DEBUG({
213       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
214     });
215     if (auto Initializers = getInitializers(JD)) {
216       LLVM_DEBUG(
217           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
218       for (auto InitFnAddr : *Initializers) {
219         LLVM_DEBUG({
220           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
221                  << "...\n";
222         });
223         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
224         InitFn();
225       }
226     } else
227       return Initializers.takeError();
228     return Error::success();
229   }
230 
deinitialize(JITDylib & JD)231   Error deinitialize(JITDylib &JD) override {
232     LLVM_DEBUG({
233       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
234     });
235     if (auto Deinitializers = getDeinitializers(JD)) {
236       LLVM_DEBUG({
237         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
238       });
239       for (auto DeinitFnAddr : *Deinitializers) {
240         LLVM_DEBUG({
241           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
242                  << "...\n";
243         });
244         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
245         DeinitFn();
246       }
247     } else
248       return Deinitializers.takeError();
249 
250     return Error::success();
251   }
252 
registerInitFunc(JITDylib & JD,SymbolStringPtr InitName)253   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
254     getExecutionSession().runSessionLocked([&]() {
255         InitFunctions[&JD].add(InitName);
256       });
257   }
258 
259 private:
260 
getInitializers(JITDylib & JD)261   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
262     if (auto Err = issueInitLookups(JD))
263       return std::move(Err);
264 
265     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
266     std::vector<JITDylibSP> DFSLinkOrder;
267 
268     getExecutionSession().runSessionLocked([&]() {
269       DFSLinkOrder = JD.getDFSLinkOrder();
270 
271       for (auto &NextJD : DFSLinkOrder) {
272         auto IFItr = InitFunctions.find(NextJD.get());
273         if (IFItr != InitFunctions.end()) {
274           LookupSymbols[NextJD.get()] = std::move(IFItr->second);
275           InitFunctions.erase(IFItr);
276         }
277       }
278     });
279 
280     LLVM_DEBUG({
281       dbgs() << "JITDylib init order is [ ";
282       for (auto &JD : llvm::reverse(DFSLinkOrder))
283         dbgs() << "\"" << JD->getName() << "\" ";
284       dbgs() << "]\n";
285       dbgs() << "Looking up init functions:\n";
286       for (auto &KV : LookupSymbols)
287         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
288     });
289 
290     auto &ES = getExecutionSession();
291     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
292 
293     if (!LookupResult)
294       return LookupResult.takeError();
295 
296     std::vector<JITTargetAddress> Initializers;
297     while (!DFSLinkOrder.empty()) {
298       auto &NextJD = *DFSLinkOrder.back();
299       DFSLinkOrder.pop_back();
300       auto InitsItr = LookupResult->find(&NextJD);
301       if (InitsItr == LookupResult->end())
302         continue;
303       for (auto &KV : InitsItr->second)
304         Initializers.push_back(KV.second.getAddress());
305     }
306 
307     return Initializers;
308   }
309 
getDeinitializers(JITDylib & JD)310   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
311     auto &ES = getExecutionSession();
312 
313     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
314 
315     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
316     std::vector<JITDylibSP> DFSLinkOrder;
317 
318     ES.runSessionLocked([&]() {
319       DFSLinkOrder = JD.getDFSLinkOrder();
320 
321       for (auto &NextJD : DFSLinkOrder) {
322         auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
323         auto DIFItr = DeInitFunctions.find(NextJD.get());
324         if (DIFItr != DeInitFunctions.end()) {
325           LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
326           DeInitFunctions.erase(DIFItr);
327         }
328         JDLookupSymbols.add(LLJITRunAtExits,
329                             SymbolLookupFlags::WeaklyReferencedSymbol);
330       }
331     });
332 
333     LLVM_DEBUG({
334       dbgs() << "JITDylib deinit order is [ ";
335       for (auto &JD : DFSLinkOrder)
336         dbgs() << "\"" << JD->getName() << "\" ";
337       dbgs() << "]\n";
338       dbgs() << "Looking up deinit functions:\n";
339       for (auto &KV : LookupSymbols)
340         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
341     });
342 
343     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
344 
345     if (!LookupResult)
346       return LookupResult.takeError();
347 
348     std::vector<JITTargetAddress> DeInitializers;
349     for (auto &NextJD : DFSLinkOrder) {
350       auto DeInitsItr = LookupResult->find(NextJD.get());
351       assert(DeInitsItr != LookupResult->end() &&
352              "Every JD should have at least __lljit_run_atexits");
353 
354       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
355       if (RunAtExitsItr != DeInitsItr->second.end())
356         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
357 
358       for (auto &KV : DeInitsItr->second)
359         if (KV.first != LLJITRunAtExits)
360           DeInitializers.push_back(KV.second.getAddress());
361     }
362 
363     return DeInitializers;
364   }
365 
366   /// Issue lookups for all init symbols required to initialize JD (and any
367   /// JITDylibs that it depends on).
issueInitLookups(JITDylib & JD)368   Error issueInitLookups(JITDylib &JD) {
369     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
370     std::vector<JITDylibSP> DFSLinkOrder;
371 
372     getExecutionSession().runSessionLocked([&]() {
373       DFSLinkOrder = JD.getDFSLinkOrder();
374 
375       for (auto &NextJD : DFSLinkOrder) {
376         auto ISItr = InitSymbols.find(NextJD.get());
377         if (ISItr != InitSymbols.end()) {
378           RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
379           InitSymbols.erase(ISItr);
380         }
381       }
382     });
383 
384     return Platform::lookupInitSymbols(getExecutionSession(),
385                                        RequiredInitSymbols)
386         .takeError();
387   }
388 
registerAtExitHelper(void * Self,void (* F)(void *),void * Ctx,void * DSOHandle)389   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
390                                    void *DSOHandle) {
391     LLVM_DEBUG({
392       dbgs() << "Registering atexit function " << (void *)F << " for JD "
393              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
394     });
395     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
396         F, Ctx, DSOHandle);
397   }
398 
runAtExitsHelper(void * Self,void * DSOHandle)399   static void runAtExitsHelper(void *Self, void *DSOHandle) {
400     LLVM_DEBUG({
401       dbgs() << "Running atexit functions for JD "
402              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
403     });
404     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
405         DSOHandle);
406   }
407 
408   // Constructs an LLVM IR module containing platform runtime globals,
409   // functions, and interposes.
createPlatformRuntimeModule()410   ThreadSafeModule createPlatformRuntimeModule() {
411     auto Ctx = std::make_unique<LLVMContext>();
412     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
413     M->setDataLayout(J.getDataLayout());
414 
415     auto *GenericIRPlatformSupportTy =
416         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
417 
418     auto *PlatformInstanceDecl = new GlobalVariable(
419         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
420         nullptr, "__lljit.platform_support_instance");
421 
422     auto *Int8Ty = Type::getInt8Ty(*Ctx);
423     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
424     auto *VoidTy = Type::getVoidTy(*Ctx);
425     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
426     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
427     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
428 
429     addHelperAndWrapper(
430         *M, "__cxa_atexit",
431         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
432                           false),
433         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
434         {PlatformInstanceDecl});
435 
436     return ThreadSafeModule(std::move(M), std::move(Ctx));
437   }
438 
439   LLJIT &J;
440   std::string InitFunctionPrefix;
441   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
442   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
443   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
444   ItaniumCXAAtExitSupport AtExitMgr;
445 };
446 
setupJITDylib(JITDylib & JD)447 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
448   return S.setupJITDylib(JD);
449 }
450 
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)451 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
452                                           const MaterializationUnit &MU) {
453   return S.notifyAdding(RT, MU);
454 }
455 
456 Expected<ThreadSafeModule>
operator ()(ThreadSafeModule TSM,MaterializationResponsibility & R)457 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
458                                   MaterializationResponsibility &R) {
459   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
460     auto &Ctx = M.getContext();
461     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
462 
463     // If there's no llvm.global_ctors or it's just a decl then skip.
464     if (!GlobalCtors || GlobalCtors->isDeclaration())
465       return Error::success();
466 
467     std::string InitFunctionName;
468     raw_string_ostream(InitFunctionName)
469         << InitFunctionPrefix << M.getModuleIdentifier();
470 
471     MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
472     auto InternedName = Mangle(InitFunctionName);
473     if (auto Err =
474             R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}}))
475       return Err;
476 
477     auto *InitFunc =
478         Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false),
479                          GlobalValue::ExternalLinkage, InitFunctionName, &M);
480     InitFunc->setVisibility(GlobalValue::HiddenVisibility);
481     std::vector<std::pair<Function *, unsigned>> Inits;
482     for (auto E : getConstructors(M))
483       Inits.push_back(std::make_pair(E.Func, E.Priority));
484     llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS,
485                          const std::pair<Function *, unsigned> &RHS) {
486       return LHS.first < RHS.first;
487     });
488     auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc);
489     IRBuilder<> IB(EntryBlock);
490     for (auto &KV : Inits)
491       IB.CreateCall(KV.first);
492     IB.CreateRetVoid();
493 
494     PS.registerInitFunc(R.getTargetJITDylib(), InternedName);
495     GlobalCtors->eraseFromParent();
496     return Error::success();
497   });
498 
499   if (Err)
500     return std::move(Err);
501 
502   return std::move(TSM);
503 }
504 
505 class MachOPlatformSupport : public LLJIT::PlatformSupport {
506 public:
507   using DLOpenType = void *(*)(const char *Name, int Mode);
508   using DLCloseType = int (*)(void *Handle);
509   using DLSymType = void *(*)(void *Handle, const char *Name);
510   using DLErrorType = const char *(*)();
511 
512   struct DlFcnValues {
513     Optional<void *> RTLDDefault;
514     DLOpenType dlopen = nullptr;
515     DLCloseType dlclose = nullptr;
516     DLSymType dlsym = nullptr;
517     DLErrorType dlerror = nullptr;
518   };
519 
520   static Expected<std::unique_ptr<MachOPlatformSupport>>
Create(LLJIT & J,JITDylib & PlatformJITDylib)521   Create(LLJIT &J, JITDylib &PlatformJITDylib) {
522 
523     // Make process symbols visible.
524     {
525       std::string ErrMsg;
526       auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg);
527       if (!Lib.isValid())
528         return make_error<StringError>(std::move(ErrMsg),
529                                        inconvertibleErrorCode());
530     }
531 
532     DlFcnValues DlFcn;
533 
534     // Add support for RTLDDefault on known platforms.
535 #ifdef __APPLE__
536     DlFcn.RTLDDefault = reinterpret_cast<void *>(-2);
537 #endif // __APPLE__
538 
539     if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen"))
540       return std::move(Err);
541     if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose"))
542       return std::move(Err);
543     if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym"))
544       return std::move(Err);
545     if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror"))
546       return std::move(Err);
547 
548     std::unique_ptr<MachOPlatformSupport> MP(
549         new MachOPlatformSupport(J, PlatformJITDylib, DlFcn));
550     return std::move(MP);
551   }
552 
initialize(JITDylib & JD)553   Error initialize(JITDylib &JD) override {
554     LLVM_DEBUG({
555       dbgs() << "MachOPlatformSupport initializing \"" << JD.getName()
556              << "\"\n";
557     });
558 
559     auto InitSeq = MP.getInitializerSequence(JD);
560     if (!InitSeq)
561       return InitSeq.takeError();
562 
563     // If ObjC is not enabled but there are JIT'd ObjC inits then return
564     // an error.
565     if (!objCRegistrationEnabled())
566       for (auto &KV : *InitSeq) {
567         if (!KV.second.getObjCSelRefsSections().empty() ||
568             !KV.second.getObjCClassListSections().empty())
569           return make_error<StringError>("JITDylib " + KV.first->getName() +
570                                              " contains objc metadata but objc"
571                                              " is not enabled",
572                                          inconvertibleErrorCode());
573       }
574 
575     // Run the initializers.
576     for (auto &KV : *InitSeq) {
577       if (objCRegistrationEnabled()) {
578         KV.second.registerObjCSelectors();
579         if (auto Err = KV.second.registerObjCClasses()) {
580           // FIXME: Roll back registrations on error?
581           return Err;
582         }
583       }
584       KV.second.runModInits();
585     }
586 
587     return Error::success();
588   }
589 
deinitialize(JITDylib & JD)590   Error deinitialize(JITDylib &JD) override {
591     auto &ES = J.getExecutionSession();
592     if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) {
593       for (auto &KV : *DeinitSeq) {
594         auto DSOHandleName = ES.intern("___dso_handle");
595 
596         // FIXME: Run DeInits here.
597         auto Result = ES.lookup(
598             {{KV.first, JITDylibLookupFlags::MatchAllSymbols}},
599             SymbolLookupSet(DSOHandleName,
600                             SymbolLookupFlags::WeaklyReferencedSymbol));
601         if (!Result)
602           return Result.takeError();
603         if (Result->empty())
604           continue;
605         assert(Result->count(DSOHandleName) &&
606                "Result does not contain __dso_handle");
607         auto *DSOHandle = jitTargetAddressToPointer<void *>(
608             Result->begin()->second.getAddress());
609         AtExitMgr.runAtExits(DSOHandle);
610       }
611     } else
612       return DeinitSeq.takeError();
613     return Error::success();
614   }
615 
616 private:
617   template <typename FunctionPtrTy>
hookUpFunction(FunctionPtrTy & Fn,const char * Name)618   static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) {
619     if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) {
620       Fn = reinterpret_cast<FunctionPtrTy>(Fn);
621       return Error::success();
622     }
623 
624     return make_error<StringError>((Twine("Can not enable MachO JIT Platform: "
625                                           "missing function: ") +
626                                     Name)
627                                        .str(),
628                                    inconvertibleErrorCode());
629   }
630 
MachOPlatformSupport(LLJIT & J,JITDylib & PlatformJITDylib,DlFcnValues DlFcn)631   MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn)
632       : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) {
633 
634     SymbolMap HelperSymbols;
635 
636     // platform and atexit helpers.
637     HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] =
638         JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags());
639     HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
640         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
641                            JITSymbolFlags());
642     HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] =
643         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
644                            JITSymbolFlags());
645 
646     // dlfcn helpers.
647     HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] =
648         JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper),
649                            JITSymbolFlags());
650     HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] =
651         JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper),
652                            JITSymbolFlags());
653     HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] =
654         JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper),
655                            JITSymbolFlags());
656     HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] =
657         JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper),
658                            JITSymbolFlags());
659 
660     cantFail(
661         PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols))));
662     cantFail(MP.setupJITDylib(J.getMainJITDylib()));
663     cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule()));
664   }
665 
setupPlatform(LLJIT & J)666   static MachOPlatform &setupPlatform(LLJIT &J) {
667     auto Tmp = std::make_unique<MachOPlatform>(
668         J.getExecutionSession(),
669         static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()),
670         createStandardSymbolsObject(J));
671     auto &MP = *Tmp;
672     J.getExecutionSession().setPlatform(std::move(Tmp));
673     return MP;
674   }
675 
createStandardSymbolsObject(LLJIT & J)676   static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) {
677     LLVMContext Ctx;
678     Module M("__standard_symbols", Ctx);
679     M.setDataLayout(J.getDataLayout());
680 
681     auto *Int64Ty = Type::getInt64Ty(Ctx);
682 
683     auto *DSOHandle =
684         new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage,
685                            ConstantInt::get(Int64Ty, 0), "__dso_handle");
686     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
687 
688     return cantFail(J.getIRCompileLayer().getCompiler()(M));
689   }
690 
createPlatformRuntimeModule()691   ThreadSafeModule createPlatformRuntimeModule() {
692     auto Ctx = std::make_unique<LLVMContext>();
693     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
694     M->setDataLayout(J.getDataLayout());
695 
696     auto *MachOPlatformSupportTy =
697         StructType::create(*Ctx, "lljit.MachOPlatformSupport");
698 
699     auto *PlatformInstanceDecl = new GlobalVariable(
700         *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr,
701         "__lljit.platform_support_instance");
702 
703     auto *Int8Ty = Type::getInt8Ty(*Ctx);
704     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
705     auto *VoidTy = Type::getVoidTy(*Ctx);
706     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
707     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
708     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
709 
710     addHelperAndWrapper(
711         *M, "__cxa_atexit",
712         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
713                           false),
714         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
715         {PlatformInstanceDecl});
716 
717     addHelperAndWrapper(*M, "dlopen",
718                         FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false),
719                         GlobalValue::DefaultVisibility, "__lljit.dlopen_helper",
720                         {PlatformInstanceDecl});
721 
722     addHelperAndWrapper(*M, "dlclose",
723                         FunctionType::get(IntTy, {BytePtrTy}, false),
724                         GlobalValue::DefaultVisibility,
725                         "__lljit.dlclose_helper", {PlatformInstanceDecl});
726 
727     addHelperAndWrapper(
728         *M, "dlsym",
729         FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false),
730         GlobalValue::DefaultVisibility, "__lljit.dlsym_helper",
731         {PlatformInstanceDecl});
732 
733     addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false),
734                         GlobalValue::DefaultVisibility,
735                         "__lljit.dlerror_helper", {PlatformInstanceDecl});
736 
737     return ThreadSafeModule(std::move(M), std::move(Ctx));
738   }
739 
registerAtExitHelper(void * Self,void (* F)(void *),void * Ctx,void * DSOHandle)740   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
741                                    void *DSOHandle) {
742     static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
743         F, Ctx, DSOHandle);
744   }
745 
runAtExitsHelper(void * Self,void * DSOHandle)746   static void runAtExitsHelper(void *Self, void *DSOHandle) {
747     static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle);
748   }
749 
jit_dlopen(const char * Path,int Mode)750   void *jit_dlopen(const char *Path, int Mode) {
751     JITDylib *JDToOpen = nullptr;
752     // FIXME: Do the right thing with Mode flags.
753     {
754       std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
755 
756       // Clear any existing error messages.
757       dlErrorMsgs.erase(std::this_thread::get_id());
758 
759       if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) {
760         auto I = JDRefCounts.find(JD);
761         if (I != JDRefCounts.end()) {
762           ++I->second;
763           return JD;
764         }
765 
766         JDRefCounts[JD] = 1;
767         JDToOpen = JD;
768       }
769     }
770 
771     if (JDToOpen) {
772       if (auto Err = initialize(*JDToOpen)) {
773         recordError(std::move(Err));
774         return 0;
775       }
776     }
777 
778     // Fall through to dlopen if no JITDylib found for Path.
779     return DlFcn.dlopen(Path, Mode);
780   }
781 
dlopenHelper(void * Self,const char * Path,int Mode)782   static void *dlopenHelper(void *Self, const char *Path, int Mode) {
783     return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode);
784   }
785 
jit_dlclose(void * Handle)786   int jit_dlclose(void *Handle) {
787     JITDylib *JDToClose = nullptr;
788 
789     {
790       std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
791 
792       // Clear any existing error messages.
793       dlErrorMsgs.erase(std::this_thread::get_id());
794 
795       auto I = JDRefCounts.find(Handle);
796       if (I != JDRefCounts.end()) {
797         --I->second;
798         if (I->second == 0) {
799           JDRefCounts.erase(I);
800           JDToClose = static_cast<JITDylib *>(Handle);
801         } else
802           return 0;
803       }
804     }
805 
806     if (JDToClose) {
807       if (auto Err = deinitialize(*JDToClose)) {
808         recordError(std::move(Err));
809         return -1;
810       }
811       return 0;
812     }
813 
814     // Fall through to dlclose if no JITDylib found for Path.
815     return DlFcn.dlclose(Handle);
816   }
817 
dlcloseHelper(void * Self,void * Handle)818   static int dlcloseHelper(void *Self, void *Handle) {
819     return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle);
820   }
821 
jit_dlsym(void * Handle,const char * Name)822   void *jit_dlsym(void *Handle, const char *Name) {
823     JITDylibSearchOrder JITSymSearchOrder;
824 
825     // FIXME: RTLD_NEXT, RTLD_SELF not supported.
826     {
827       std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
828 
829       // Clear any existing error messages.
830       dlErrorMsgs.erase(std::this_thread::get_id());
831 
832       if (JDRefCounts.count(Handle)) {
833         JITSymSearchOrder.push_back(
834             {static_cast<JITDylib *>(Handle),
835              JITDylibLookupFlags::MatchExportedSymbolsOnly});
836       } else if (Handle == DlFcn.RTLDDefault) {
837         for (auto &KV : JDRefCounts)
838           JITSymSearchOrder.push_back(
839               {static_cast<JITDylib *>(KV.first),
840                JITDylibLookupFlags::MatchExportedSymbolsOnly});
841       }
842     }
843 
844     if (!JITSymSearchOrder.empty()) {
845       auto MangledName = J.mangleAndIntern(Name);
846       SymbolLookupSet Syms(MangledName,
847                            SymbolLookupFlags::WeaklyReferencedSymbol);
848       if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms,
849                                                        LookupKind::DLSym)) {
850         auto I = Result->find(MangledName);
851         if (I != Result->end())
852           return jitTargetAddressToPointer<void *>(I->second.getAddress());
853       } else {
854         recordError(Result.takeError());
855         return 0;
856       }
857     }
858 
859     // Fall through to dlsym.
860     return DlFcn.dlsym(Handle, Name);
861   }
862 
dlsymHelper(void * Self,void * Handle,const char * Name)863   static void *dlsymHelper(void *Self, void *Handle, const char *Name) {
864     return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name);
865   }
866 
jit_dlerror()867   const char *jit_dlerror() {
868     {
869       std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
870       auto I = dlErrorMsgs.find(std::this_thread::get_id());
871       if (I != dlErrorMsgs.end())
872         return I->second->c_str();
873     }
874     return DlFcn.dlerror();
875   }
876 
dlerrorHelper(void * Self)877   static const char *dlerrorHelper(void *Self) {
878     return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror();
879   }
880 
recordError(Error Err)881   void recordError(Error Err) {
882     std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
883     dlErrorMsgs[std::this_thread::get_id()] =
884         std::make_unique<std::string>(toString(std::move(Err)));
885   }
886 
887   std::mutex PlatformSupportMutex;
888   LLJIT &J;
889   MachOPlatform &MP;
890   DlFcnValues DlFcn;
891   ItaniumCXAAtExitSupport AtExitMgr;
892   DenseMap<void *, unsigned> JDRefCounts;
893   std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs;
894 };
895 
896 /// Inactive Platform Support
897 ///
898 /// Explicitly disables platform support. JITDylibs are not scanned for special
899 /// init/deinit symbols. No runtime API interposes are injected.
900 class InactivePlatformSupport : public LLJIT::PlatformSupport {
901 public:
902   InactivePlatformSupport() = default;
903 
initialize(JITDylib & JD)904   Error initialize(JITDylib &JD) override {
905     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
906                       << JD.getName() << "\n");
907     return Error::success();
908   }
909 
deinitialize(JITDylib & JD)910   Error deinitialize(JITDylib &JD) override {
911     LLVM_DEBUG(
912         dbgs() << "InactivePlatformSupport: no deinitializers running for "
913                << JD.getName() << "\n");
914     return Error::success();
915   }
916 };
917 
918 } // end anonymous namespace
919 
920 namespace llvm {
921 namespace orc {
922 
setInitTransform(LLJIT & J,IRTransformLayer::TransformFunction T)923 void LLJIT::PlatformSupport::setInitTransform(
924     LLJIT &J, IRTransformLayer::TransformFunction T) {
925   J.InitHelperTransformLayer->setTransform(std::move(T));
926 }
927 
~PlatformSupport()928 LLJIT::PlatformSupport::~PlatformSupport() {}
929 
prepareForConstruction()930 Error LLJITBuilderState::prepareForConstruction() {
931 
932   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
933 
934   if (!JTMB) {
935     LLVM_DEBUG({
936       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
937                 "Detecting host...\n";
938     });
939     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
940       JTMB = std::move(*JTMBOrErr);
941     else
942       return JTMBOrErr.takeError();
943   }
944 
945   LLVM_DEBUG({
946     dbgs() << "  JITTargetMachineBuilder is "
947            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
948            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
949            << "\n"
950            << "  DataLayout: ";
951     if (DL)
952       dbgs() << DL->getStringRepresentation() << "\n";
953     else
954       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
955 
956     dbgs() << "  Custom object-linking-layer creator: "
957            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
958            << "  Custom compile-function creator: "
959            << (CreateCompileFunction ? "Yes" : "No") << "\n"
960            << "  Custom platform-setup function: "
961            << (SetUpPlatform ? "Yes" : "No") << "\n"
962            << "  Number of compile threads: " << NumCompileThreads;
963     if (!NumCompileThreads)
964       dbgs() << " (code will be compiled on the execution thread)\n";
965     else
966       dbgs() << "\n";
967   });
968 
969   // If the client didn't configure any linker options then auto-configure the
970   // JIT linker.
971   if (!CreateObjectLinkingLayer) {
972     auto &TT = JTMB->getTargetTriple();
973     if (TT.isOSBinFormatMachO() &&
974         (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
975 
976       JTMB->setRelocationModel(Reloc::PIC_);
977       JTMB->setCodeModel(CodeModel::Small);
978       CreateObjectLinkingLayer =
979           [TPC = this->TPC](
980               ExecutionSession &ES,
981               const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
982         std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer;
983         if (TPC)
984           ObjLinkingLayer =
985               std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr());
986         else
987           ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
988               ES, std::make_unique<jitlink::InProcessMemoryManager>());
989         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
990             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
991         return std::move(ObjLinkingLayer);
992       };
993     }
994   }
995 
996   return Error::success();
997 }
998 
~LLJIT()999 LLJIT::~LLJIT() {
1000   if (CompileThreads)
1001     CompileThreads->wait();
1002   if (auto Err = ES->endSession())
1003     ES->reportError(std::move(Err));
1004 }
1005 
addIRModule(ResourceTrackerSP RT,ThreadSafeModule TSM)1006 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
1007   assert(TSM && "Can not add null module");
1008 
1009   if (auto Err =
1010           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
1011     return Err;
1012 
1013   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
1014 }
1015 
addIRModule(JITDylib & JD,ThreadSafeModule TSM)1016 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1017   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
1018 }
1019 
addObjectFile(ResourceTrackerSP RT,std::unique_ptr<MemoryBuffer> Obj)1020 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
1021                            std::unique_ptr<MemoryBuffer> Obj) {
1022   assert(Obj && "Can not add null object");
1023 
1024   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
1025 }
1026 
addObjectFile(JITDylib & JD,std::unique_ptr<MemoryBuffer> Obj)1027 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
1028   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
1029 }
1030 
lookupLinkerMangled(JITDylib & JD,SymbolStringPtr Name)1031 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
1032                                                         SymbolStringPtr Name) {
1033   return ES->lookup(
1034       makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
1035 }
1036 
1037 Expected<std::unique_ptr<ObjectLayer>>
createObjectLinkingLayer(LLJITBuilderState & S,ExecutionSession & ES)1038 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
1039 
1040   // If the config state provided an ObjectLinkingLayer factory then use it.
1041   if (S.CreateObjectLinkingLayer)
1042     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
1043 
1044   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
1045   // a new SectionMemoryManager for each object.
1046   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
1047   auto Layer =
1048       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
1049 
1050   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
1051     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
1052     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
1053   }
1054 
1055   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
1056   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
1057   //        just return ObjLinkingLayer) once those bots are upgraded.
1058   return std::unique_ptr<ObjectLayer>(std::move(Layer));
1059 }
1060 
1061 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
createCompileFunction(LLJITBuilderState & S,JITTargetMachineBuilder JTMB)1062 LLJIT::createCompileFunction(LLJITBuilderState &S,
1063                              JITTargetMachineBuilder JTMB) {
1064 
1065   /// If there is a custom compile function creator set then use it.
1066   if (S.CreateCompileFunction)
1067     return S.CreateCompileFunction(std::move(JTMB));
1068 
1069   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
1070   // depending on the number of threads requested.
1071   if (S.NumCompileThreads > 0)
1072     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
1073 
1074   auto TM = JTMB.createTargetMachine();
1075   if (!TM)
1076     return TM.takeError();
1077 
1078   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
1079 }
1080 
LLJIT(LLJITBuilderState & S,Error & Err)1081 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1082     : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(),
1083       DL(""), TT(S.JTMB->getTargetTriple()) {
1084 
1085   ErrorAsOutParameter _(&Err);
1086 
1087   if (auto MainOrErr = this->ES->createJITDylib("main"))
1088     Main = &*MainOrErr;
1089   else {
1090     Err = MainOrErr.takeError();
1091     return;
1092   }
1093 
1094   if (S.DL)
1095     DL = std::move(*S.DL);
1096   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
1097     DL = std::move(*DLOrErr);
1098   else {
1099     Err = DLOrErr.takeError();
1100     return;
1101   }
1102 
1103   auto ObjLayer = createObjectLinkingLayer(S, *ES);
1104   if (!ObjLayer) {
1105     Err = ObjLayer.takeError();
1106     return;
1107   }
1108   ObjLinkingLayer = std::move(*ObjLayer);
1109   ObjTransformLayer =
1110       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
1111 
1112   {
1113     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
1114     if (!CompileFunction) {
1115       Err = CompileFunction.takeError();
1116       return;
1117     }
1118     CompileLayer = std::make_unique<IRCompileLayer>(
1119         *ES, *ObjTransformLayer, std::move(*CompileFunction));
1120     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
1121     InitHelperTransformLayer =
1122         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
1123   }
1124 
1125   if (S.NumCompileThreads > 0) {
1126     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1127     CompileThreads =
1128         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
1129     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
1130       // FIXME: We should be able to use move-capture here, but ThreadPool's
1131       // AsyncTaskTys are std::functions rather than unique_functions
1132       // (because MSVC's std::packaged_tasks don't support move-only types).
1133       // Fix this when all the above gets sorted out.
1134       CompileThreads->async([UnownedT = T.release()]() mutable {
1135         std::unique_ptr<Task> T(UnownedT);
1136         T->run();
1137       });
1138     });
1139   }
1140 
1141   if (S.SetUpPlatform)
1142     Err = S.SetUpPlatform(*this);
1143   else
1144     setUpGenericLLVMIRPlatform(*this);
1145 }
1146 
mangle(StringRef UnmangledName) const1147 std::string LLJIT::mangle(StringRef UnmangledName) const {
1148   std::string MangledName;
1149   {
1150     raw_string_ostream MangledNameStream(MangledName);
1151     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
1152   }
1153   return MangledName;
1154 }
1155 
applyDataLayout(Module & M)1156 Error LLJIT::applyDataLayout(Module &M) {
1157   if (M.getDataLayout().isDefault())
1158     M.setDataLayout(DL);
1159 
1160   if (M.getDataLayout() != DL)
1161     return make_error<StringError>(
1162         "Added modules have incompatible data layouts: " +
1163             M.getDataLayout().getStringRepresentation() + " (module) vs " +
1164             DL.getStringRepresentation() + " (jit)",
1165         inconvertibleErrorCode());
1166 
1167   return Error::success();
1168 }
1169 
setUpGenericLLVMIRPlatform(LLJIT & J)1170 void setUpGenericLLVMIRPlatform(LLJIT &J) {
1171   LLVM_DEBUG(
1172       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1173   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
1174 }
1175 
setUpMachOPlatform(LLJIT & J)1176 Error setUpMachOPlatform(LLJIT &J) {
1177   LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; });
1178   auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib());
1179   if (!MP)
1180     return MP.takeError();
1181   J.setPlatformSupport(std::move(*MP));
1182   return Error::success();
1183 }
1184 
setUpInactivePlatform(LLJIT & J)1185 Error setUpInactivePlatform(LLJIT &J) {
1186   LLVM_DEBUG(
1187       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
1188   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
1189   return Error::success();
1190 }
1191 
prepareForConstruction()1192 Error LLLazyJITBuilderState::prepareForConstruction() {
1193   if (auto Err = LLJITBuilderState::prepareForConstruction())
1194     return Err;
1195   TT = JTMB->getTargetTriple();
1196   return Error::success();
1197 }
1198 
addLazyIRModule(JITDylib & JD,ThreadSafeModule TSM)1199 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1200   assert(TSM && "Can not add null module");
1201 
1202   if (auto Err = TSM.withModuleDo(
1203           [&](Module &M) -> Error { return applyDataLayout(M); }))
1204     return Err;
1205 
1206   return CODLayer->add(JD, std::move(TSM));
1207 }
1208 
LLLazyJIT(LLLazyJITBuilderState & S,Error & Err)1209 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1210 
1211   // If LLJIT construction failed then bail out.
1212   if (Err)
1213     return;
1214 
1215   ErrorAsOutParameter _(&Err);
1216 
1217   /// Take/Create the lazy-compile callthrough manager.
1218   if (S.LCTMgr)
1219     LCTMgr = std::move(S.LCTMgr);
1220   else {
1221     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1222             S.TT, *ES, S.LazyCompileFailureAddr))
1223       LCTMgr = std::move(*LCTMgrOrErr);
1224     else {
1225       Err = LCTMgrOrErr.takeError();
1226       return;
1227     }
1228   }
1229 
1230   // Take/Create the indirect stubs manager builder.
1231   auto ISMBuilder = std::move(S.ISMBuilder);
1232 
1233   // If none was provided, try to build one.
1234   if (!ISMBuilder)
1235     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1236 
1237   // No luck. Bail out.
1238   if (!ISMBuilder) {
1239     Err = make_error<StringError>("Could not construct "
1240                                   "IndirectStubsManagerBuilder for target " +
1241                                       S.TT.str(),
1242                                   inconvertibleErrorCode());
1243     return;
1244   }
1245 
1246   // Create the COD layer.
1247   CODLayer = std::make_unique<CompileOnDemandLayer>(
1248       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
1249 
1250   if (S.NumCompileThreads > 0)
1251     CODLayer->setCloneToNewContextOnEmit(true);
1252 }
1253 
1254 } // End namespace orc.
1255 } // End namespace llvm.
1256