1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
13 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
14 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
16 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
17 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Mangler.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Support/DynamicLibrary.h"
24 
25 #include <map>
26 
27 #define DEBUG_TYPE "orc"
28 
29 using namespace llvm;
30 using namespace llvm::orc;
31 
32 namespace {
33 
34 /// Adds helper function decls and wrapper functions that call the helper with
35 /// some additional prefix arguments.
36 ///
37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
38 /// args i32 4 and i16 12345, this function will add:
39 ///
40 /// declare i8 @bar(i32, i16, i8, i64)
41 ///
42 /// define i8 @foo(i8, i64) {
43 /// entry:
44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
45 ///   ret i8 %2
46 /// }
47 ///
48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
49                               FunctionType *WrapperFnType,
50                               GlobalValue::VisibilityTypes WrapperVisibility,
51                               StringRef HelperName,
52                               ArrayRef<Value *> HelperPrefixArgs) {
53   std::vector<Type *> HelperArgTypes;
54   for (auto *Arg : HelperPrefixArgs)
55     HelperArgTypes.push_back(Arg->getType());
56   for (auto *T : WrapperFnType->params())
57     HelperArgTypes.push_back(T);
58   auto *HelperFnType =
59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
61                                     HelperName, M);
62 
63   auto *WrapperFn = Function::Create(
64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
65   WrapperFn->setVisibility(WrapperVisibility);
66 
67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
68   IRBuilder<> IB(EntryBlock);
69 
70   std::vector<Value *> HelperArgs;
71   for (auto *Arg : HelperPrefixArgs)
72     HelperArgs.push_back(Arg);
73   for (auto &Arg : WrapperFn->args())
74     HelperArgs.push_back(&Arg);
75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
76   if (HelperFn->getReturnType()->isVoidTy())
77     IB.CreateRetVoid();
78   else
79     IB.CreateRet(HelperResult);
80 
81   return WrapperFn;
82 }
83 
84 class GenericLLVMIRPlatformSupport;
85 
86 /// orc::Platform component of Generic LLVM IR Platform support.
87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
88 class GenericLLVMIRPlatform : public Platform {
89 public:
90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
91   Error setupJITDylib(JITDylib &JD) override;
92   Error teardownJITDylib(JITDylib &JD) override;
93   Error notifyAdding(ResourceTracker &RT,
94                      const MaterializationUnit &MU) override;
95   Error notifyRemoving(ResourceTracker &RT) override {
96     // Noop -- Nothing to do (yet).
97     return Error::success();
98   }
99 
100 private:
101   GenericLLVMIRPlatformSupport &S;
102 };
103 
104 /// This transform parses llvm.global_ctors to produce a single initialization
105 /// function for the module, records the function, then deletes
106 /// llvm.global_ctors.
107 class GlobalCtorDtorScraper {
108 public:
109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
110                         StringRef InitFunctionPrefix,
111                         StringRef DeInitFunctionPrefix)
112       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
113         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
114   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
115                                         MaterializationResponsibility &R);
116 
117 private:
118   GenericLLVMIRPlatformSupport &PS;
119   StringRef InitFunctionPrefix;
120   StringRef DeInitFunctionPrefix;
121 };
122 
123 /// Generic IR Platform Support
124 ///
125 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
126 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
127 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
128 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
129 public:
130   GenericLLVMIRPlatformSupport(LLJIT &J)
131       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
132         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
133 
134     getExecutionSession().setPlatform(
135         std::make_unique<GenericLLVMIRPlatform>(*this));
136 
137     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
138                                               DeInitFunctionPrefix));
139 
140     SymbolMap StdInterposes;
141 
142     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
143         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
144                            JITSymbolFlags::Exported);
145     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
146         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
147                            JITSymbolFlags());
148 
149     cantFail(
150         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
151     cantFail(setupJITDylib(J.getMainJITDylib()));
152     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
153   }
154 
155   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
156 
157   /// Adds a module that defines the __dso_handle global.
158   Error setupJITDylib(JITDylib &JD) {
159 
160     // Add per-jitdylib standard interposes.
161     SymbolMap PerJDInterposes;
162     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
163         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
164                            JITSymbolFlags());
165     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
166 
167     auto Ctx = std::make_unique<LLVMContext>();
168     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
169     M->setDataLayout(J.getDataLayout());
170 
171     auto *Int64Ty = Type::getInt64Ty(*Ctx);
172     auto *DSOHandle = new GlobalVariable(
173         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
174         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
175         "__dso_handle");
176     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
177     DSOHandle->setInitializer(
178         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
179 
180     auto *GenericIRPlatformSupportTy =
181         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
182 
183     auto *PlatformInstanceDecl = new GlobalVariable(
184         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
185         nullptr, "__lljit.platform_support_instance");
186 
187     auto *VoidTy = Type::getVoidTy(*Ctx);
188     addHelperAndWrapper(
189         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
190         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
191         {PlatformInstanceDecl, DSOHandle});
192 
193     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
194   }
195 
196   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
197     auto &JD = RT.getJITDylib();
198     if (auto &InitSym = MU.getInitializerSymbol())
199       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
200     else {
201       // If there's no identified init symbol attached, but there is a symbol
202       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
203       // an init function. Add the symbol to both the InitSymbols map (which
204       // will trigger a lookup to materialize the module) and the InitFunctions
205       // map (which holds the names of the symbols to execute).
206       for (auto &KV : MU.getSymbols())
207         if ((*KV.first).startswith(InitFunctionPrefix)) {
208           InitSymbols[&JD].add(KV.first,
209                                SymbolLookupFlags::WeaklyReferencedSymbol);
210           InitFunctions[&JD].add(KV.first);
211         } else if ((*KV.first).startswith(DeInitFunctionPrefix)) {
212           DeInitFunctions[&JD].add(KV.first);
213         }
214     }
215     return Error::success();
216   }
217 
218   Error initialize(JITDylib &JD) override {
219     LLVM_DEBUG({
220       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
221     });
222     if (auto Initializers = getInitializers(JD)) {
223       LLVM_DEBUG(
224           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
225       for (auto InitFnAddr : *Initializers) {
226         LLVM_DEBUG({
227           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
228                  << "...\n";
229         });
230         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
231         InitFn();
232       }
233     } else
234       return Initializers.takeError();
235     return Error::success();
236   }
237 
238   Error deinitialize(JITDylib &JD) override {
239     LLVM_DEBUG({
240       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
241     });
242     if (auto Deinitializers = getDeinitializers(JD)) {
243       LLVM_DEBUG({
244         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
245       });
246       for (auto DeinitFnAddr : *Deinitializers) {
247         LLVM_DEBUG({
248           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
249                  << "...\n";
250         });
251         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
252         DeinitFn();
253       }
254     } else
255       return Deinitializers.takeError();
256 
257     return Error::success();
258   }
259 
260   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
261     getExecutionSession().runSessionLocked([&]() {
262         InitFunctions[&JD].add(InitName);
263       });
264   }
265 
266   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
267     getExecutionSession().runSessionLocked(
268         [&]() { DeInitFunctions[&JD].add(DeInitName); });
269   }
270 
271 private:
272 
273   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
274     if (auto Err = issueInitLookups(JD))
275       return std::move(Err);
276 
277     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
278     std::vector<JITDylibSP> DFSLinkOrder;
279 
280     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
281           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
282             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
283           else
284             return DFSLinkOrderOrErr.takeError();
285 
286           for (auto &NextJD : DFSLinkOrder) {
287             auto IFItr = InitFunctions.find(NextJD.get());
288             if (IFItr != InitFunctions.end()) {
289               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
290               InitFunctions.erase(IFItr);
291             }
292           }
293           return Error::success();
294         }))
295       return std::move(Err);
296 
297     LLVM_DEBUG({
298       dbgs() << "JITDylib init order is [ ";
299       for (auto &JD : llvm::reverse(DFSLinkOrder))
300         dbgs() << "\"" << JD->getName() << "\" ";
301       dbgs() << "]\n";
302       dbgs() << "Looking up init functions:\n";
303       for (auto &KV : LookupSymbols)
304         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
305     });
306 
307     auto &ES = getExecutionSession();
308     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
309 
310     if (!LookupResult)
311       return LookupResult.takeError();
312 
313     std::vector<JITTargetAddress> Initializers;
314     while (!DFSLinkOrder.empty()) {
315       auto &NextJD = *DFSLinkOrder.back();
316       DFSLinkOrder.pop_back();
317       auto InitsItr = LookupResult->find(&NextJD);
318       if (InitsItr == LookupResult->end())
319         continue;
320       for (auto &KV : InitsItr->second)
321         Initializers.push_back(KV.second.getAddress());
322     }
323 
324     return Initializers;
325   }
326 
327   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
328     auto &ES = getExecutionSession();
329 
330     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
331 
332     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
333     std::vector<JITDylibSP> DFSLinkOrder;
334 
335     if (auto Err = ES.runSessionLocked([&]() -> Error {
336           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
337             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
338           else
339             return DFSLinkOrderOrErr.takeError();
340 
341           for (auto &NextJD : DFSLinkOrder) {
342             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
343             auto DIFItr = DeInitFunctions.find(NextJD.get());
344             if (DIFItr != DeInitFunctions.end()) {
345               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
346               DeInitFunctions.erase(DIFItr);
347             }
348             JDLookupSymbols.add(LLJITRunAtExits,
349                                 SymbolLookupFlags::WeaklyReferencedSymbol);
350           }
351           return Error::success();
352         }))
353       return std::move(Err);
354 
355     LLVM_DEBUG({
356       dbgs() << "JITDylib deinit order is [ ";
357       for (auto &JD : DFSLinkOrder)
358         dbgs() << "\"" << JD->getName() << "\" ";
359       dbgs() << "]\n";
360       dbgs() << "Looking up deinit functions:\n";
361       for (auto &KV : LookupSymbols)
362         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
363     });
364 
365     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
366 
367     if (!LookupResult)
368       return LookupResult.takeError();
369 
370     std::vector<JITTargetAddress> DeInitializers;
371     for (auto &NextJD : DFSLinkOrder) {
372       auto DeInitsItr = LookupResult->find(NextJD.get());
373       assert(DeInitsItr != LookupResult->end() &&
374              "Every JD should have at least __lljit_run_atexits");
375 
376       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
377       if (RunAtExitsItr != DeInitsItr->second.end())
378         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
379 
380       for (auto &KV : DeInitsItr->second)
381         if (KV.first != LLJITRunAtExits)
382           DeInitializers.push_back(KV.second.getAddress());
383     }
384 
385     return DeInitializers;
386   }
387 
388   /// Issue lookups for all init symbols required to initialize JD (and any
389   /// JITDylibs that it depends on).
390   Error issueInitLookups(JITDylib &JD) {
391     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
392     std::vector<JITDylibSP> DFSLinkOrder;
393 
394     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
395           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
396             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
397           else
398             return DFSLinkOrderOrErr.takeError();
399 
400           for (auto &NextJD : DFSLinkOrder) {
401             auto ISItr = InitSymbols.find(NextJD.get());
402             if (ISItr != InitSymbols.end()) {
403               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
404               InitSymbols.erase(ISItr);
405             }
406           }
407           return Error::success();
408         }))
409       return Err;
410 
411     return Platform::lookupInitSymbols(getExecutionSession(),
412                                        RequiredInitSymbols)
413         .takeError();
414   }
415 
416   static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
417                                    void *DSOHandle) {
418     LLVM_DEBUG({
419       dbgs() << "Registering atexit function " << (void *)F << " for JD "
420              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
421     });
422     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
423         F, Ctx, DSOHandle);
424   }
425 
426   static void runAtExitsHelper(void *Self, void *DSOHandle) {
427     LLVM_DEBUG({
428       dbgs() << "Running atexit functions for JD "
429              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
430     });
431     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
432         DSOHandle);
433   }
434 
435   // Constructs an LLVM IR module containing platform runtime globals,
436   // functions, and interposes.
437   ThreadSafeModule createPlatformRuntimeModule() {
438     auto Ctx = std::make_unique<LLVMContext>();
439     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
440     M->setDataLayout(J.getDataLayout());
441 
442     auto *GenericIRPlatformSupportTy =
443         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
444 
445     auto *PlatformInstanceDecl = new GlobalVariable(
446         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
447         nullptr, "__lljit.platform_support_instance");
448 
449     auto *Int8Ty = Type::getInt8Ty(*Ctx);
450     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
451     auto *VoidTy = Type::getVoidTy(*Ctx);
452     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
453     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
454     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
455 
456     addHelperAndWrapper(
457         *M, "__cxa_atexit",
458         FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
459                           false),
460         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
461         {PlatformInstanceDecl});
462 
463     return ThreadSafeModule(std::move(M), std::move(Ctx));
464   }
465 
466   LLJIT &J;
467   std::string InitFunctionPrefix;
468   std::string DeInitFunctionPrefix;
469   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
470   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
471   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
472   ItaniumCXAAtExitSupport AtExitMgr;
473 };
474 
475 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
476   return S.setupJITDylib(JD);
477 }
478 
479 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
480   return Error::success();
481 }
482 
483 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
484                                           const MaterializationUnit &MU) {
485   return S.notifyAdding(RT, MU);
486 }
487 
488 Expected<ThreadSafeModule>
489 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
490                                   MaterializationResponsibility &R) {
491   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
492     auto &Ctx = M.getContext();
493     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
494     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
495 
496     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
497                                 bool isCtor) -> Error {
498       // If there's no llvm.global_c/dtor or it's just a decl then skip.
499       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
500         return Error::success();
501       std::string InitOrDeInitFunctionName;
502       if (isCtor)
503         raw_string_ostream(InitOrDeInitFunctionName)
504             << InitFunctionPrefix << M.getModuleIdentifier();
505       else
506         raw_string_ostream(InitOrDeInitFunctionName)
507             << DeInitFunctionPrefix << M.getModuleIdentifier();
508 
509       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
510       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
511       if (auto Err = R.defineMaterializing(
512               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
513         return Err;
514 
515       auto *InitOrDeInitFunc = Function::Create(
516           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
517           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
518       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
519       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
520       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
521 
522       for (auto E : COrDtors)
523         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
524       llvm::sort(InitsOrDeInits,
525                  [](const std::pair<Function *, unsigned> &LHS,
526                     const std::pair<Function *, unsigned> &RHS) {
527                    return LHS.first < RHS.first;
528                  });
529 
530       auto *InitOrDeInitFuncEntryBlock =
531           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
532       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
533       for (auto &KV : InitsOrDeInits)
534         IB.CreateCall(KV.first);
535       IB.CreateRetVoid();
536 
537       if (isCtor)
538         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
539       else
540         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
541 
542       GlobalCOrDtors->eraseFromParent();
543       return Error::success();
544     };
545 
546     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
547       return Err;
548     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
549       return Err;
550 
551     return Error::success();
552   });
553 
554   if (Err)
555     return std::move(Err);
556 
557   return std::move(TSM);
558 }
559 
560 /// Inactive Platform Support
561 ///
562 /// Explicitly disables platform support. JITDylibs are not scanned for special
563 /// init/deinit symbols. No runtime API interposes are injected.
564 class InactivePlatformSupport : public LLJIT::PlatformSupport {
565 public:
566   InactivePlatformSupport() = default;
567 
568   Error initialize(JITDylib &JD) override {
569     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
570                       << JD.getName() << "\n");
571     return Error::success();
572   }
573 
574   Error deinitialize(JITDylib &JD) override {
575     LLVM_DEBUG(
576         dbgs() << "InactivePlatformSupport: no deinitializers running for "
577                << JD.getName() << "\n");
578     return Error::success();
579   }
580 };
581 
582 } // end anonymous namespace
583 
584 namespace llvm {
585 namespace orc {
586 
587 void LLJIT::PlatformSupport::setInitTransform(
588     LLJIT &J, IRTransformLayer::TransformFunction T) {
589   J.InitHelperTransformLayer->setTransform(std::move(T));
590 }
591 
592 LLJIT::PlatformSupport::~PlatformSupport() {}
593 
594 Error LLJITBuilderState::prepareForConstruction() {
595 
596   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
597 
598   if (!JTMB) {
599     LLVM_DEBUG({
600       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
601                 "Detecting host...\n";
602     });
603     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
604       JTMB = std::move(*JTMBOrErr);
605     else
606       return JTMBOrErr.takeError();
607   }
608 
609   LLVM_DEBUG({
610     dbgs() << "  JITTargetMachineBuilder is "
611            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
612            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
613            << "\n"
614            << "  DataLayout: ";
615     if (DL)
616       dbgs() << DL->getStringRepresentation() << "\n";
617     else
618       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
619 
620     dbgs() << "  Custom object-linking-layer creator: "
621            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
622            << "  Custom compile-function creator: "
623            << (CreateCompileFunction ? "Yes" : "No") << "\n"
624            << "  Custom platform-setup function: "
625            << (SetUpPlatform ? "Yes" : "No") << "\n"
626            << "  Number of compile threads: " << NumCompileThreads;
627     if (!NumCompileThreads)
628       dbgs() << " (code will be compiled on the execution thread)\n";
629     else
630       dbgs() << "\n";
631   });
632 
633   // If neither ES nor EPC has been set then create an EPC instance.
634   if (!ES && !EPC) {
635     LLVM_DEBUG({
636       dbgs() << "ExecutorProcessControl not specified, "
637                 "Creating SelfExecutorProcessControl instance\n";
638     });
639     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
640       EPC = std::move(*EPCOrErr);
641     else
642       return EPCOrErr.takeError();
643   } else
644     LLVM_DEBUG({
645       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
646              << EPC.get() << "\n";
647     });
648 
649   // If the client didn't configure any linker options then auto-configure the
650   // JIT linker.
651   if (!CreateObjectLinkingLayer) {
652     auto &TT = JTMB->getTargetTriple();
653     if (TT.isOSBinFormatMachO() &&
654         (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
655 
656       JTMB->setRelocationModel(Reloc::PIC_);
657       JTMB->setCodeModel(CodeModel::Small);
658       CreateObjectLinkingLayer =
659           [](ExecutionSession &ES,
660              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
661         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
662         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
663             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
664         return std::move(ObjLinkingLayer);
665       };
666     }
667   }
668 
669   return Error::success();
670 }
671 
672 LLJIT::~LLJIT() {
673   if (CompileThreads)
674     CompileThreads->wait();
675   if (auto Err = ES->endSession())
676     ES->reportError(std::move(Err));
677 }
678 
679 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
680   assert(TSM && "Can not add null module");
681 
682   if (auto Err =
683           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
684     return Err;
685 
686   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
687 }
688 
689 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
690   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
691 }
692 
693 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
694                            std::unique_ptr<MemoryBuffer> Obj) {
695   assert(Obj && "Can not add null object");
696 
697   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
698 }
699 
700 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
701   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
702 }
703 
704 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
705                                                         SymbolStringPtr Name) {
706   return ES->lookup(
707       makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
708 }
709 
710 Expected<std::unique_ptr<ObjectLayer>>
711 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
712 
713   // If the config state provided an ObjectLinkingLayer factory then use it.
714   if (S.CreateObjectLinkingLayer)
715     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
716 
717   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
718   // a new SectionMemoryManager for each object.
719   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
720   auto Layer =
721       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
722 
723   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
724     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
725     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
726   }
727 
728   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
729   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
730   //        just return ObjLinkingLayer) once those bots are upgraded.
731   return std::unique_ptr<ObjectLayer>(std::move(Layer));
732 }
733 
734 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
735 LLJIT::createCompileFunction(LLJITBuilderState &S,
736                              JITTargetMachineBuilder JTMB) {
737 
738   /// If there is a custom compile function creator set then use it.
739   if (S.CreateCompileFunction)
740     return S.CreateCompileFunction(std::move(JTMB));
741 
742   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
743   // depending on the number of threads requested.
744   if (S.NumCompileThreads > 0)
745     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
746 
747   auto TM = JTMB.createTargetMachine();
748   if (!TM)
749     return TM.takeError();
750 
751   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
752 }
753 
754 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
755     : DL(""), TT(S.JTMB->getTargetTriple()) {
756 
757   ErrorAsOutParameter _(&Err);
758 
759   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
760 
761   if (S.EPC) {
762     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
763   } else if (S.ES)
764     ES = std::move(S.ES);
765   else {
766     if (auto EPC = SelfExecutorProcessControl::Create()) {
767       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
768     } else {
769       Err = EPC.takeError();
770       return;
771     }
772   }
773 
774   if (auto MainOrErr = this->ES->createJITDylib("main"))
775     Main = &*MainOrErr;
776   else {
777     Err = MainOrErr.takeError();
778     return;
779   }
780 
781   if (S.DL)
782     DL = std::move(*S.DL);
783   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
784     DL = std::move(*DLOrErr);
785   else {
786     Err = DLOrErr.takeError();
787     return;
788   }
789 
790   auto ObjLayer = createObjectLinkingLayer(S, *ES);
791   if (!ObjLayer) {
792     Err = ObjLayer.takeError();
793     return;
794   }
795   ObjLinkingLayer = std::move(*ObjLayer);
796   ObjTransformLayer =
797       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
798 
799   {
800     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
801     if (!CompileFunction) {
802       Err = CompileFunction.takeError();
803       return;
804     }
805     CompileLayer = std::make_unique<IRCompileLayer>(
806         *ES, *ObjTransformLayer, std::move(*CompileFunction));
807     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
808     InitHelperTransformLayer =
809         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
810   }
811 
812   if (S.NumCompileThreads > 0) {
813     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
814     CompileThreads =
815         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
816     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
817       // FIXME: We should be able to use move-capture here, but ThreadPool's
818       // AsyncTaskTys are std::functions rather than unique_functions
819       // (because MSVC's std::packaged_tasks don't support move-only types).
820       // Fix this when all the above gets sorted out.
821       CompileThreads->async([UnownedT = T.release()]() mutable {
822         std::unique_ptr<Task> T(UnownedT);
823         T->run();
824       });
825     });
826   }
827 
828   if (S.SetUpPlatform)
829     Err = S.SetUpPlatform(*this);
830   else
831     setUpGenericLLVMIRPlatform(*this);
832 }
833 
834 std::string LLJIT::mangle(StringRef UnmangledName) const {
835   std::string MangledName;
836   {
837     raw_string_ostream MangledNameStream(MangledName);
838     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
839   }
840   return MangledName;
841 }
842 
843 Error LLJIT::applyDataLayout(Module &M) {
844   if (M.getDataLayout().isDefault())
845     M.setDataLayout(DL);
846 
847   if (M.getDataLayout() != DL)
848     return make_error<StringError>(
849         "Added modules have incompatible data layouts: " +
850             M.getDataLayout().getStringRepresentation() + " (module) vs " +
851             DL.getStringRepresentation() + " (jit)",
852         inconvertibleErrorCode());
853 
854   return Error::success();
855 }
856 
857 void setUpGenericLLVMIRPlatform(LLJIT &J) {
858   LLVM_DEBUG(
859       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
860   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
861 }
862 
863 Error setUpInactivePlatform(LLJIT &J) {
864   LLVM_DEBUG(
865       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
866   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
867   return Error::success();
868 }
869 
870 Error LLLazyJITBuilderState::prepareForConstruction() {
871   if (auto Err = LLJITBuilderState::prepareForConstruction())
872     return Err;
873   TT = JTMB->getTargetTriple();
874   return Error::success();
875 }
876 
877 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
878   assert(TSM && "Can not add null module");
879 
880   if (auto Err = TSM.withModuleDo(
881           [&](Module &M) -> Error { return applyDataLayout(M); }))
882     return Err;
883 
884   return CODLayer->add(JD, std::move(TSM));
885 }
886 
887 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
888 
889   // If LLJIT construction failed then bail out.
890   if (Err)
891     return;
892 
893   ErrorAsOutParameter _(&Err);
894 
895   /// Take/Create the lazy-compile callthrough manager.
896   if (S.LCTMgr)
897     LCTMgr = std::move(S.LCTMgr);
898   else {
899     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
900             S.TT, *ES, S.LazyCompileFailureAddr))
901       LCTMgr = std::move(*LCTMgrOrErr);
902     else {
903       Err = LCTMgrOrErr.takeError();
904       return;
905     }
906   }
907 
908   // Take/Create the indirect stubs manager builder.
909   auto ISMBuilder = std::move(S.ISMBuilder);
910 
911   // If none was provided, try to build one.
912   if (!ISMBuilder)
913     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
914 
915   // No luck. Bail out.
916   if (!ISMBuilder) {
917     Err = make_error<StringError>("Could not construct "
918                                   "IndirectStubsManagerBuilder for target " +
919                                       S.TT.str(),
920                                   inconvertibleErrorCode());
921     return;
922   }
923 
924   // Create the COD layer.
925   CODLayer = std::make_unique<CompileOnDemandLayer>(
926       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
927 
928   if (S.NumCompileThreads > 0)
929     CODLayer->setCloneToNewContextOnEmit(true);
930 }
931 
932 } // End namespace orc.
933 } // End namespace llvm.
934