1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
13 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
14 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
16 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
17 #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h"
18 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
19 #include "llvm/IR/GlobalVariable.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Mangler.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/Support/DynamicLibrary.h"
24 
25 #include <map>
26 
27 #define DEBUG_TYPE "orc"
28 
29 using namespace llvm;
30 using namespace llvm::orc;
31 
32 namespace {
33 
34 /// Adds helper function decls and wrapper functions that call the helper with
35 /// some additional prefix arguments.
36 ///
37 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
38 /// args i32 4 and i16 12345, this function will add:
39 ///
40 /// declare i8 @bar(i32, i16, i8, i64)
41 ///
42 /// define i8 @foo(i8, i64) {
43 /// entry:
44 ///   %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
45 ///   ret i8 %2
46 /// }
47 ///
48 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
49                               FunctionType *WrapperFnType,
50                               GlobalValue::VisibilityTypes WrapperVisibility,
51                               StringRef HelperName,
52                               ArrayRef<Value *> HelperPrefixArgs) {
53   std::vector<Type *> HelperArgTypes;
54   for (auto *Arg : HelperPrefixArgs)
55     HelperArgTypes.push_back(Arg->getType());
56   for (auto *T : WrapperFnType->params())
57     HelperArgTypes.push_back(T);
58   auto *HelperFnType =
59       FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
60   auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
61                                     HelperName, M);
62 
63   auto *WrapperFn = Function::Create(
64       WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
65   WrapperFn->setVisibility(WrapperVisibility);
66 
67   auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
68   IRBuilder<> IB(EntryBlock);
69 
70   std::vector<Value *> HelperArgs;
71   for (auto *Arg : HelperPrefixArgs)
72     HelperArgs.push_back(Arg);
73   for (auto &Arg : WrapperFn->args())
74     HelperArgs.push_back(&Arg);
75   auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
76   if (HelperFn->getReturnType()->isVoidTy())
77     IB.CreateRetVoid();
78   else
79     IB.CreateRet(HelperResult);
80 
81   return WrapperFn;
82 }
83 
84 class GenericLLVMIRPlatformSupport;
85 
86 /// orc::Platform component of Generic LLVM IR Platform support.
87 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
88 class GenericLLVMIRPlatform : public Platform {
89 public:
90   GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
91   Error setupJITDylib(JITDylib &JD) override;
92   Error teardownJITDylib(JITDylib &JD) override;
93   Error notifyAdding(ResourceTracker &RT,
94                      const MaterializationUnit &MU) override;
95   Error notifyRemoving(ResourceTracker &RT) override {
96     // Noop -- Nothing to do (yet).
97     return Error::success();
98   }
99 
100 private:
101   GenericLLVMIRPlatformSupport &S;
102 };
103 
104 /// This transform parses llvm.global_ctors to produce a single initialization
105 /// function for the module, records the function, then deletes
106 /// llvm.global_ctors.
107 class GlobalCtorDtorScraper {
108 public:
109   GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
110                         StringRef InitFunctionPrefix,
111                         StringRef DeInitFunctionPrefix)
112       : PS(PS), InitFunctionPrefix(InitFunctionPrefix),
113         DeInitFunctionPrefix(DeInitFunctionPrefix) {}
114   Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
115                                         MaterializationResponsibility &R);
116 
117 private:
118   GenericLLVMIRPlatformSupport &PS;
119   StringRef InitFunctionPrefix;
120   StringRef DeInitFunctionPrefix;
121 };
122 
123 /// Generic IR Platform Support
124 ///
125 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
126 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
127 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
128 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
129 public:
130   GenericLLVMIRPlatformSupport(LLJIT &J)
131       : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")),
132         DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) {
133 
134     getExecutionSession().setPlatform(
135         std::make_unique<GenericLLVMIRPlatform>(*this));
136 
137     setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix,
138                                               DeInitFunctionPrefix));
139 
140     SymbolMap StdInterposes;
141 
142     StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
143         JITEvaluatedSymbol(pointerToJITTargetAddress(this),
144                            JITSymbolFlags::Exported);
145     StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
146         JITEvaluatedSymbol(pointerToJITTargetAddress(registerCxaAtExitHelper),
147                            JITSymbolFlags());
148 
149     cantFail(
150         J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
151     cantFail(setupJITDylib(J.getMainJITDylib()));
152     cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
153   }
154 
155   ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
156 
157   /// Adds a module that defines the __dso_handle global.
158   Error setupJITDylib(JITDylib &JD) {
159 
160     // Add per-jitdylib standard interposes.
161     SymbolMap PerJDInterposes;
162     PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
163         JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
164                            JITSymbolFlags());
165     PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] =
166         JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
167                            JITSymbolFlags());
168     cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
169 
170     auto Ctx = std::make_unique<LLVMContext>();
171     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
172     M->setDataLayout(J.getDataLayout());
173 
174     auto *Int64Ty = Type::getInt64Ty(*Ctx);
175     auto *DSOHandle = new GlobalVariable(
176         *M, Int64Ty, true, GlobalValue::ExternalLinkage,
177         ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
178         "__dso_handle");
179     DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
180     DSOHandle->setInitializer(
181         ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
182 
183     auto *GenericIRPlatformSupportTy =
184         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
185 
186     auto *PlatformInstanceDecl = new GlobalVariable(
187         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
188         nullptr, "__lljit.platform_support_instance");
189 
190     auto *VoidTy = Type::getVoidTy(*Ctx);
191     addHelperAndWrapper(
192         *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
193         GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
194         {PlatformInstanceDecl, DSOHandle});
195 
196     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
197     auto *AtExitCallbackTy = FunctionType::get(VoidTy, {}, false);
198     auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
199     addHelperAndWrapper(*M, "atexit",
200                         FunctionType::get(IntTy, {AtExitCallbackPtrTy}, false),
201                         GlobalValue::HiddenVisibility, "__lljit.atexit_helper",
202                         {PlatformInstanceDecl, DSOHandle});
203 
204     return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
205   }
206 
207   Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) {
208     auto &JD = RT.getJITDylib();
209     if (auto &InitSym = MU.getInitializerSymbol())
210       InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
211     else {
212       // If there's no identified init symbol attached, but there is a symbol
213       // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
214       // an init function. Add the symbol to both the InitSymbols map (which
215       // will trigger a lookup to materialize the module) and the InitFunctions
216       // map (which holds the names of the symbols to execute).
217       for (auto &KV : MU.getSymbols())
218         if ((*KV.first).startswith(InitFunctionPrefix)) {
219           InitSymbols[&JD].add(KV.first,
220                                SymbolLookupFlags::WeaklyReferencedSymbol);
221           InitFunctions[&JD].add(KV.first);
222         } else if ((*KV.first).startswith(DeInitFunctionPrefix)) {
223           DeInitFunctions[&JD].add(KV.first);
224         }
225     }
226     return Error::success();
227   }
228 
229   Error initialize(JITDylib &JD) override {
230     LLVM_DEBUG({
231       dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
232     });
233     if (auto Initializers = getInitializers(JD)) {
234       LLVM_DEBUG(
235           { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
236       for (auto InitFnAddr : *Initializers) {
237         LLVM_DEBUG({
238           dbgs() << "  Running init " << formatv("{0:x16}", InitFnAddr)
239                  << "...\n";
240         });
241         auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
242         InitFn();
243       }
244     } else
245       return Initializers.takeError();
246     return Error::success();
247   }
248 
249   Error deinitialize(JITDylib &JD) override {
250     LLVM_DEBUG({
251       dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
252     });
253     if (auto Deinitializers = getDeinitializers(JD)) {
254       LLVM_DEBUG({
255         dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
256       });
257       for (auto DeinitFnAddr : *Deinitializers) {
258         LLVM_DEBUG({
259           dbgs() << "  Running deinit " << formatv("{0:x16}", DeinitFnAddr)
260                  << "...\n";
261         });
262         auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
263         DeinitFn();
264       }
265     } else
266       return Deinitializers.takeError();
267 
268     return Error::success();
269   }
270 
271   void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
272     getExecutionSession().runSessionLocked([&]() {
273         InitFunctions[&JD].add(InitName);
274       });
275   }
276 
277   void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) {
278     getExecutionSession().runSessionLocked(
279         [&]() { DeInitFunctions[&JD].add(DeInitName); });
280   }
281 
282 private:
283 
284   Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
285     if (auto Err = issueInitLookups(JD))
286       return std::move(Err);
287 
288     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
289     std::vector<JITDylibSP> DFSLinkOrder;
290 
291     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
292           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
293             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
294           else
295             return DFSLinkOrderOrErr.takeError();
296 
297           for (auto &NextJD : DFSLinkOrder) {
298             auto IFItr = InitFunctions.find(NextJD.get());
299             if (IFItr != InitFunctions.end()) {
300               LookupSymbols[NextJD.get()] = std::move(IFItr->second);
301               InitFunctions.erase(IFItr);
302             }
303           }
304           return Error::success();
305         }))
306       return std::move(Err);
307 
308     LLVM_DEBUG({
309       dbgs() << "JITDylib init order is [ ";
310       for (auto &JD : llvm::reverse(DFSLinkOrder))
311         dbgs() << "\"" << JD->getName() << "\" ";
312       dbgs() << "]\n";
313       dbgs() << "Looking up init functions:\n";
314       for (auto &KV : LookupSymbols)
315         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
316     });
317 
318     auto &ES = getExecutionSession();
319     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
320 
321     if (!LookupResult)
322       return LookupResult.takeError();
323 
324     std::vector<JITTargetAddress> Initializers;
325     while (!DFSLinkOrder.empty()) {
326       auto &NextJD = *DFSLinkOrder.back();
327       DFSLinkOrder.pop_back();
328       auto InitsItr = LookupResult->find(&NextJD);
329       if (InitsItr == LookupResult->end())
330         continue;
331       for (auto &KV : InitsItr->second)
332         Initializers.push_back(KV.second.getAddress());
333     }
334 
335     return Initializers;
336   }
337 
338   Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
339     auto &ES = getExecutionSession();
340 
341     auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
342 
343     DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
344     std::vector<JITDylibSP> DFSLinkOrder;
345 
346     if (auto Err = ES.runSessionLocked([&]() -> Error {
347           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
348             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
349           else
350             return DFSLinkOrderOrErr.takeError();
351 
352           for (auto &NextJD : DFSLinkOrder) {
353             auto &JDLookupSymbols = LookupSymbols[NextJD.get()];
354             auto DIFItr = DeInitFunctions.find(NextJD.get());
355             if (DIFItr != DeInitFunctions.end()) {
356               LookupSymbols[NextJD.get()] = std::move(DIFItr->second);
357               DeInitFunctions.erase(DIFItr);
358             }
359             JDLookupSymbols.add(LLJITRunAtExits,
360                                 SymbolLookupFlags::WeaklyReferencedSymbol);
361           }
362           return Error::success();
363         }))
364       return std::move(Err);
365 
366     LLVM_DEBUG({
367       dbgs() << "JITDylib deinit order is [ ";
368       for (auto &JD : DFSLinkOrder)
369         dbgs() << "\"" << JD->getName() << "\" ";
370       dbgs() << "]\n";
371       dbgs() << "Looking up deinit functions:\n";
372       for (auto &KV : LookupSymbols)
373         dbgs() << "  \"" << KV.first->getName() << "\": " << KV.second << "\n";
374     });
375 
376     auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
377 
378     if (!LookupResult)
379       return LookupResult.takeError();
380 
381     std::vector<JITTargetAddress> DeInitializers;
382     for (auto &NextJD : DFSLinkOrder) {
383       auto DeInitsItr = LookupResult->find(NextJD.get());
384       assert(DeInitsItr != LookupResult->end() &&
385              "Every JD should have at least __lljit_run_atexits");
386 
387       auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
388       if (RunAtExitsItr != DeInitsItr->second.end())
389         DeInitializers.push_back(RunAtExitsItr->second.getAddress());
390 
391       for (auto &KV : DeInitsItr->second)
392         if (KV.first != LLJITRunAtExits)
393           DeInitializers.push_back(KV.second.getAddress());
394     }
395 
396     return DeInitializers;
397   }
398 
399   /// Issue lookups for all init symbols required to initialize JD (and any
400   /// JITDylibs that it depends on).
401   Error issueInitLookups(JITDylib &JD) {
402     DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
403     std::vector<JITDylibSP> DFSLinkOrder;
404 
405     if (auto Err = getExecutionSession().runSessionLocked([&]() -> Error {
406           if (auto DFSLinkOrderOrErr = JD.getDFSLinkOrder())
407             DFSLinkOrder = std::move(*DFSLinkOrderOrErr);
408           else
409             return DFSLinkOrderOrErr.takeError();
410 
411           for (auto &NextJD : DFSLinkOrder) {
412             auto ISItr = InitSymbols.find(NextJD.get());
413             if (ISItr != InitSymbols.end()) {
414               RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second);
415               InitSymbols.erase(ISItr);
416             }
417           }
418           return Error::success();
419         }))
420       return Err;
421 
422     return Platform::lookupInitSymbols(getExecutionSession(),
423                                        RequiredInitSymbols)
424         .takeError();
425   }
426 
427   static void registerCxaAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
428                                       void *DSOHandle) {
429     LLVM_DEBUG({
430       dbgs() << "Registering cxa atexit function " << (void *)F << " for JD "
431              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
432     });
433     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
434         F, Ctx, DSOHandle);
435   }
436 
437   static void registerAtExitHelper(void *Self, void *DSOHandle, void (*F)()) {
438     LLVM_DEBUG({
439       dbgs() << "Registering atexit function " << (void *)F << " for JD "
440              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
441     });
442     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
443         reinterpret_cast<void (*)(void *)>(F), nullptr, DSOHandle);
444   }
445 
446   static void runAtExitsHelper(void *Self, void *DSOHandle) {
447     LLVM_DEBUG({
448       dbgs() << "Running atexit functions for JD "
449              << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
450     });
451     static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
452         DSOHandle);
453   }
454 
455   // Constructs an LLVM IR module containing platform runtime globals,
456   // functions, and interposes.
457   ThreadSafeModule createPlatformRuntimeModule() {
458     auto Ctx = std::make_unique<LLVMContext>();
459     auto M = std::make_unique<Module>("__standard_lib", *Ctx);
460     M->setDataLayout(J.getDataLayout());
461 
462     auto *GenericIRPlatformSupportTy =
463         StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
464 
465     auto *PlatformInstanceDecl = new GlobalVariable(
466         *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
467         nullptr, "__lljit.platform_support_instance");
468 
469     auto *Int8Ty = Type::getInt8Ty(*Ctx);
470     auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
471     auto *VoidTy = Type::getVoidTy(*Ctx);
472     auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
473     auto *CxaAtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
474     auto *CxaAtExitCallbackPtrTy = PointerType::getUnqual(CxaAtExitCallbackTy);
475 
476     addHelperAndWrapper(
477         *M, "__cxa_atexit",
478         FunctionType::get(IntTy, {CxaAtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
479                           false),
480         GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
481         {PlatformInstanceDecl});
482 
483     return ThreadSafeModule(std::move(M), std::move(Ctx));
484   }
485 
486   LLJIT &J;
487   std::string InitFunctionPrefix;
488   std::string DeInitFunctionPrefix;
489   DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
490   DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
491   DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
492   ItaniumCXAAtExitSupport AtExitMgr;
493 };
494 
495 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
496   return S.setupJITDylib(JD);
497 }
498 
499 Error GenericLLVMIRPlatform::teardownJITDylib(JITDylib &JD) {
500   return Error::success();
501 }
502 
503 Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT,
504                                           const MaterializationUnit &MU) {
505   return S.notifyAdding(RT, MU);
506 }
507 
508 Expected<ThreadSafeModule>
509 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
510                                   MaterializationResponsibility &R) {
511   auto Err = TSM.withModuleDo([&](Module &M) -> Error {
512     auto &Ctx = M.getContext();
513     auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
514     auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors");
515 
516     auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors,
517                                 bool isCtor) -> Error {
518       // If there's no llvm.global_c/dtor or it's just a decl then skip.
519       if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration())
520         return Error::success();
521       std::string InitOrDeInitFunctionName;
522       if (isCtor)
523         raw_string_ostream(InitOrDeInitFunctionName)
524             << InitFunctionPrefix << M.getModuleIdentifier();
525       else
526         raw_string_ostream(InitOrDeInitFunctionName)
527             << DeInitFunctionPrefix << M.getModuleIdentifier();
528 
529       MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
530       auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName);
531       if (auto Err = R.defineMaterializing(
532               {{InternedInitOrDeInitName, JITSymbolFlags::Callable}}))
533         return Err;
534 
535       auto *InitOrDeInitFunc = Function::Create(
536           FunctionType::get(Type::getVoidTy(Ctx), {}, false),
537           GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M);
538       InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility);
539       std::vector<std::pair<Function *, unsigned>> InitsOrDeInits;
540       auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M);
541 
542       for (auto E : COrDtors)
543         InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority));
544       llvm::sort(InitsOrDeInits, llvm::less_second());
545 
546       auto *InitOrDeInitFuncEntryBlock =
547           BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc);
548       IRBuilder<> IB(InitOrDeInitFuncEntryBlock);
549       for (auto &KV : InitsOrDeInits)
550         IB.CreateCall(KV.first);
551       IB.CreateRetVoid();
552 
553       if (isCtor)
554         PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
555       else
556         PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName);
557 
558       GlobalCOrDtors->eraseFromParent();
559       return Error::success();
560     };
561 
562     if (auto Err = RegisterCOrDtors(GlobalCtors, true))
563       return Err;
564     if (auto Err = RegisterCOrDtors(GlobalDtors, false))
565       return Err;
566 
567     return Error::success();
568   });
569 
570   if (Err)
571     return std::move(Err);
572 
573   return std::move(TSM);
574 }
575 
576 /// Inactive Platform Support
577 ///
578 /// Explicitly disables platform support. JITDylibs are not scanned for special
579 /// init/deinit symbols. No runtime API interposes are injected.
580 class InactivePlatformSupport : public LLJIT::PlatformSupport {
581 public:
582   InactivePlatformSupport() = default;
583 
584   Error initialize(JITDylib &JD) override {
585     LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for "
586                       << JD.getName() << "\n");
587     return Error::success();
588   }
589 
590   Error deinitialize(JITDylib &JD) override {
591     LLVM_DEBUG(
592         dbgs() << "InactivePlatformSupport: no deinitializers running for "
593                << JD.getName() << "\n");
594     return Error::success();
595   }
596 };
597 
598 } // end anonymous namespace
599 
600 namespace llvm {
601 namespace orc {
602 
603 void LLJIT::PlatformSupport::setInitTransform(
604     LLJIT &J, IRTransformLayer::TransformFunction T) {
605   J.InitHelperTransformLayer->setTransform(std::move(T));
606 }
607 
608 LLJIT::PlatformSupport::~PlatformSupport() = default;
609 
610 Error LLJITBuilderState::prepareForConstruction() {
611 
612   LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n");
613 
614   if (!JTMB) {
615     LLVM_DEBUG({
616       dbgs() << "  No explicitly set JITTargetMachineBuilder. "
617                 "Detecting host...\n";
618     });
619     if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
620       JTMB = std::move(*JTMBOrErr);
621     else
622       return JTMBOrErr.takeError();
623   }
624 
625   LLVM_DEBUG({
626     dbgs() << "  JITTargetMachineBuilder is "
627            << JITTargetMachineBuilderPrinter(*JTMB, "  ")
628            << "  Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
629            << "\n"
630            << "  DataLayout: ";
631     if (DL)
632       dbgs() << DL->getStringRepresentation() << "\n";
633     else
634       dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
635 
636     dbgs() << "  Custom object-linking-layer creator: "
637            << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
638            << "  Custom compile-function creator: "
639            << (CreateCompileFunction ? "Yes" : "No") << "\n"
640            << "  Custom platform-setup function: "
641            << (SetUpPlatform ? "Yes" : "No") << "\n"
642            << "  Number of compile threads: " << NumCompileThreads;
643     if (!NumCompileThreads)
644       dbgs() << " (code will be compiled on the execution thread)\n";
645     else
646       dbgs() << "\n";
647   });
648 
649   // If neither ES nor EPC has been set then create an EPC instance.
650   if (!ES && !EPC) {
651     LLVM_DEBUG({
652       dbgs() << "ExecutorProcessControl not specified, "
653                 "Creating SelfExecutorProcessControl instance\n";
654     });
655     if (auto EPCOrErr = SelfExecutorProcessControl::Create())
656       EPC = std::move(*EPCOrErr);
657     else
658       return EPCOrErr.takeError();
659   } else
660     LLVM_DEBUG({
661       dbgs() << "Using explicitly specified ExecutorProcessControl instance "
662              << EPC.get() << "\n";
663     });
664 
665   // If the client didn't configure any linker options then auto-configure the
666   // JIT linker.
667   if (!CreateObjectLinkingLayer) {
668     auto &TT = JTMB->getTargetTriple();
669     if (TT.getArch() == Triple::riscv64 ||
670         (TT.isOSBinFormatMachO() &&
671          (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) {
672 
673       JTMB->setRelocationModel(Reloc::PIC_);
674       JTMB->setCodeModel(CodeModel::Small);
675       CreateObjectLinkingLayer =
676           [](ExecutionSession &ES,
677              const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> {
678         auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES);
679         ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
680             ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>()));
681         return std::move(ObjLinkingLayer);
682       };
683     }
684   }
685 
686   return Error::success();
687 }
688 
689 LLJIT::~LLJIT() {
690   if (CompileThreads)
691     CompileThreads->wait();
692   if (auto Err = ES->endSession())
693     ES->reportError(std::move(Err));
694 }
695 
696 Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) {
697   assert(TSM && "Can not add null module");
698 
699   if (auto Err =
700           TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
701     return Err;
702 
703   return InitHelperTransformLayer->add(std::move(RT), std::move(TSM));
704 }
705 
706 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
707   return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM));
708 }
709 
710 Error LLJIT::addObjectFile(ResourceTrackerSP RT,
711                            std::unique_ptr<MemoryBuffer> Obj) {
712   assert(Obj && "Can not add null object");
713 
714   return ObjTransformLayer->add(std::move(RT), std::move(Obj));
715 }
716 
717 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
718   return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj));
719 }
720 
721 Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD,
722                                                   SymbolStringPtr Name) {
723   if (auto Sym = ES->lookup(
724         makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),
725         Name))
726     return ExecutorAddr(Sym->getAddress());
727   else
728     return Sym.takeError();
729 }
730 
731 Expected<std::unique_ptr<ObjectLayer>>
732 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
733 
734   // If the config state provided an ObjectLinkingLayer factory then use it.
735   if (S.CreateObjectLinkingLayer)
736     return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
737 
738   // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
739   // a new SectionMemoryManager for each object.
740   auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
741   auto Layer =
742       std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
743 
744   if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
745     Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);
746     Layer->setAutoClaimResponsibilityForObjectSymbols(true);
747   }
748 
749   // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
750   //        errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
751   //        just return ObjLinkingLayer) once those bots are upgraded.
752   return std::unique_ptr<ObjectLayer>(std::move(Layer));
753 }
754 
755 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
756 LLJIT::createCompileFunction(LLJITBuilderState &S,
757                              JITTargetMachineBuilder JTMB) {
758 
759   /// If there is a custom compile function creator set then use it.
760   if (S.CreateCompileFunction)
761     return S.CreateCompileFunction(std::move(JTMB));
762 
763   // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
764   // depending on the number of threads requested.
765   if (S.NumCompileThreads > 0)
766     return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
767 
768   auto TM = JTMB.createTargetMachine();
769   if (!TM)
770     return TM.takeError();
771 
772   return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
773 }
774 
775 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
776     : DL(""), TT(S.JTMB->getTargetTriple()) {
777 
778   ErrorAsOutParameter _(&Err);
779 
780   assert(!(S.EPC && S.ES) && "EPC and ES should not both be set");
781 
782   if (S.EPC) {
783     ES = std::make_unique<ExecutionSession>(std::move(S.EPC));
784   } else if (S.ES)
785     ES = std::move(S.ES);
786   else {
787     if (auto EPC = SelfExecutorProcessControl::Create()) {
788       ES = std::make_unique<ExecutionSession>(std::move(*EPC));
789     } else {
790       Err = EPC.takeError();
791       return;
792     }
793   }
794 
795   if (auto MainOrErr = this->ES->createJITDylib("main"))
796     Main = &*MainOrErr;
797   else {
798     Err = MainOrErr.takeError();
799     return;
800   }
801 
802   if (S.DL)
803     DL = std::move(*S.DL);
804   else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
805     DL = std::move(*DLOrErr);
806   else {
807     Err = DLOrErr.takeError();
808     return;
809   }
810 
811   auto ObjLayer = createObjectLinkingLayer(S, *ES);
812   if (!ObjLayer) {
813     Err = ObjLayer.takeError();
814     return;
815   }
816   ObjLinkingLayer = std::move(*ObjLayer);
817   ObjTransformLayer =
818       std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer);
819 
820   {
821     auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
822     if (!CompileFunction) {
823       Err = CompileFunction.takeError();
824       return;
825     }
826     CompileLayer = std::make_unique<IRCompileLayer>(
827         *ES, *ObjTransformLayer, std::move(*CompileFunction));
828     TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
829     InitHelperTransformLayer =
830         std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
831   }
832 
833   if (S.NumCompileThreads > 0) {
834     InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
835     CompileThreads =
836         std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
837     ES->setDispatchTask([this](std::unique_ptr<Task> T) {
838       // FIXME: We should be able to use move-capture here, but ThreadPool's
839       // AsyncTaskTys are std::functions rather than unique_functions
840       // (because MSVC's std::packaged_tasks don't support move-only types).
841       // Fix this when all the above gets sorted out.
842       CompileThreads->async([UnownedT = T.release()]() mutable {
843         std::unique_ptr<Task> T(UnownedT);
844         T->run();
845       });
846     });
847   }
848 
849   if (S.SetUpPlatform)
850     Err = S.SetUpPlatform(*this);
851   else
852     setUpGenericLLVMIRPlatform(*this);
853 }
854 
855 std::string LLJIT::mangle(StringRef UnmangledName) const {
856   std::string MangledName;
857   {
858     raw_string_ostream MangledNameStream(MangledName);
859     Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
860   }
861   return MangledName;
862 }
863 
864 Error LLJIT::applyDataLayout(Module &M) {
865   if (M.getDataLayout().isDefault())
866     M.setDataLayout(DL);
867 
868   if (M.getDataLayout() != DL)
869     return make_error<StringError>(
870         "Added modules have incompatible data layouts: " +
871             M.getDataLayout().getStringRepresentation() + " (module) vs " +
872             DL.getStringRepresentation() + " (jit)",
873         inconvertibleErrorCode());
874 
875   return Error::success();
876 }
877 
878 void setUpGenericLLVMIRPlatform(LLJIT &J) {
879   LLVM_DEBUG(
880       { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
881   J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
882 }
883 
884 Error setUpInactivePlatform(LLJIT &J) {
885   LLVM_DEBUG(
886       { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; });
887   J.setPlatformSupport(std::make_unique<InactivePlatformSupport>());
888   return Error::success();
889 }
890 
891 Error LLLazyJITBuilderState::prepareForConstruction() {
892   if (auto Err = LLJITBuilderState::prepareForConstruction())
893     return Err;
894   TT = JTMB->getTargetTriple();
895   return Error::success();
896 }
897 
898 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
899   assert(TSM && "Can not add null module");
900 
901   if (auto Err = TSM.withModuleDo(
902           [&](Module &M) -> Error { return applyDataLayout(M); }))
903     return Err;
904 
905   return CODLayer->add(JD, std::move(TSM));
906 }
907 
908 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
909 
910   // If LLJIT construction failed then bail out.
911   if (Err)
912     return;
913 
914   ErrorAsOutParameter _(&Err);
915 
916   /// Take/Create the lazy-compile callthrough manager.
917   if (S.LCTMgr)
918     LCTMgr = std::move(S.LCTMgr);
919   else {
920     if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
921         S.TT, *ES, S.LazyCompileFailureAddr.getValue()))
922       LCTMgr = std::move(*LCTMgrOrErr);
923     else {
924       Err = LCTMgrOrErr.takeError();
925       return;
926     }
927   }
928 
929   // Take/Create the indirect stubs manager builder.
930   auto ISMBuilder = std::move(S.ISMBuilder);
931 
932   // If none was provided, try to build one.
933   if (!ISMBuilder)
934     ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
935 
936   // No luck. Bail out.
937   if (!ISMBuilder) {
938     Err = make_error<StringError>("Could not construct "
939                                   "IndirectStubsManagerBuilder for target " +
940                                       S.TT.str(),
941                                   inconvertibleErrorCode());
942     return;
943   }
944 
945   // Create the COD layer.
946   CODLayer = std::make_unique<CompileOnDemandLayer>(
947       *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
948 
949   if (S.NumCompileThreads > 0)
950     CODLayer->setCloneToNewContextOnEmit(true);
951 }
952 
953 } // End namespace orc.
954 } // End namespace llvm.
955