1 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"
12 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
13 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
14 #include "llvm/ExecutionEngine/Orc/OrcError.h"
15 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
16 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
17 #include "llvm/IR/GlobalVariable.h"
18 #include "llvm/IR/IRBuilder.h"
19 #include "llvm/IR/Mangler.h"
20 #include "llvm/IR/Module.h"
21 #include "llvm/Support/DynamicLibrary.h"
22
23 #include <map>
24
25 #define DEBUG_TYPE "orc"
26
27 using namespace llvm;
28 using namespace llvm::orc;
29
30 namespace {
31
32 /// Adds helper function decls and wrapper functions that call the helper with
33 /// some additional prefix arguments.
34 ///
35 /// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix
36 /// args i32 4 and i16 12345, this function will add:
37 ///
38 /// declare i8 @bar(i32, i16, i8, i64)
39 ///
40 /// define i8 @foo(i8, i64) {
41 /// entry:
42 /// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1)
43 /// ret i8 %2
44 /// }
45 ///
addHelperAndWrapper(Module & M,StringRef WrapperName,FunctionType * WrapperFnType,GlobalValue::VisibilityTypes WrapperVisibility,StringRef HelperName,ArrayRef<Value * > HelperPrefixArgs)46 Function *addHelperAndWrapper(Module &M, StringRef WrapperName,
47 FunctionType *WrapperFnType,
48 GlobalValue::VisibilityTypes WrapperVisibility,
49 StringRef HelperName,
50 ArrayRef<Value *> HelperPrefixArgs) {
51 std::vector<Type *> HelperArgTypes;
52 for (auto *Arg : HelperPrefixArgs)
53 HelperArgTypes.push_back(Arg->getType());
54 for (auto *T : WrapperFnType->params())
55 HelperArgTypes.push_back(T);
56 auto *HelperFnType =
57 FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false);
58 auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage,
59 HelperName, M);
60
61 auto *WrapperFn = Function::Create(
62 WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M);
63 WrapperFn->setVisibility(WrapperVisibility);
64
65 auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn);
66 IRBuilder<> IB(EntryBlock);
67
68 std::vector<Value *> HelperArgs;
69 for (auto *Arg : HelperPrefixArgs)
70 HelperArgs.push_back(Arg);
71 for (auto &Arg : WrapperFn->args())
72 HelperArgs.push_back(&Arg);
73 auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs);
74 if (HelperFn->getReturnType()->isVoidTy())
75 IB.CreateRetVoid();
76 else
77 IB.CreateRet(HelperResult);
78
79 return WrapperFn;
80 }
81
82 class GenericLLVMIRPlatformSupport;
83
84 /// orc::Platform component of Generic LLVM IR Platform support.
85 /// Just forwards calls to the GenericLLVMIRPlatformSupport class below.
86 class GenericLLVMIRPlatform : public Platform {
87 public:
GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport & S)88 GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {}
89 Error setupJITDylib(JITDylib &JD) override;
90 Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override;
notifyRemoving(JITDylib & JD,VModuleKey K)91 Error notifyRemoving(JITDylib &JD, VModuleKey K) override {
92 // Noop -- Nothing to do (yet).
93 return Error::success();
94 }
95
96 private:
97 GenericLLVMIRPlatformSupport &S;
98 };
99
100 /// This transform parses llvm.global_ctors to produce a single initialization
101 /// function for the module, records the function, then deletes
102 /// llvm.global_ctors.
103 class GlobalCtorDtorScraper {
104 public:
105
GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport & PS,StringRef InitFunctionPrefix)106 GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS,
107 StringRef InitFunctionPrefix)
108 : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {}
109 Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM,
110 MaterializationResponsibility &R);
111
112 private:
113 GenericLLVMIRPlatformSupport &PS;
114 StringRef InitFunctionPrefix;
115 };
116
117 /// Generic IR Platform Support
118 ///
119 /// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with
120 /// specially named 'init' and 'deinit'. Injects definitions / interposes for
121 /// some runtime API, including __cxa_atexit, dlopen, and dlclose.
122 class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport {
123 public:
124 // GenericLLVMIRPlatform &P) : P(P) {
GenericLLVMIRPlatformSupport(LLJIT & J)125 GenericLLVMIRPlatformSupport(LLJIT &J)
126 : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) {
127
128 getExecutionSession().setPlatform(
129 std::make_unique<GenericLLVMIRPlatform>(*this));
130
131 setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix));
132
133 SymbolMap StdInterposes;
134
135 StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] =
136 JITEvaluatedSymbol(pointerToJITTargetAddress(this),
137 JITSymbolFlags::Exported);
138 StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
139 JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
140 JITSymbolFlags());
141
142 cantFail(
143 J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes))));
144 cantFail(setupJITDylib(J.getMainJITDylib()));
145 cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule()));
146 }
147
getExecutionSession()148 ExecutionSession &getExecutionSession() { return J.getExecutionSession(); }
149
150 /// Adds a module that defines the __dso_handle global.
setupJITDylib(JITDylib & JD)151 Error setupJITDylib(JITDylib &JD) {
152
153 // Add per-jitdylib standard interposes.
154 SymbolMap PerJDInterposes;
155 PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] =
156 JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
157 JITSymbolFlags());
158 cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes))));
159
160 auto Ctx = std::make_unique<LLVMContext>();
161 auto M = std::make_unique<Module>("__standard_lib", *Ctx);
162 M->setDataLayout(J.getDataLayout());
163
164 auto *Int64Ty = Type::getInt64Ty(*Ctx);
165 auto *DSOHandle = new GlobalVariable(
166 *M, Int64Ty, true, GlobalValue::ExternalLinkage,
167 ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)),
168 "__dso_handle");
169 DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
170 DSOHandle->setInitializer(
171 ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD)));
172
173 auto *GenericIRPlatformSupportTy =
174 StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
175
176 auto *PlatformInstanceDecl = new GlobalVariable(
177 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
178 nullptr, "__lljit.platform_support_instance");
179
180 auto *VoidTy = Type::getVoidTy(*Ctx);
181 addHelperAndWrapper(
182 *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false),
183 GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper",
184 {PlatformInstanceDecl, DSOHandle});
185
186 return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx)));
187 }
188
notifyAdding(JITDylib & JD,const MaterializationUnit & MU)189 Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) {
190 if (auto &InitSym = MU.getInitializerSymbol())
191 InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol);
192 else {
193 // If there's no identified init symbol attached, but there is a symbol
194 // with the GenericIRPlatform::InitFunctionPrefix, then treat that as
195 // an init function. Add the symbol to both the InitSymbols map (which
196 // will trigger a lookup to materialize the module) and the InitFunctions
197 // map (which holds the names of the symbols to execute).
198 for (auto &KV : MU.getSymbols())
199 if ((*KV.first).startswith(InitFunctionPrefix)) {
200 InitSymbols[&JD].add(KV.first,
201 SymbolLookupFlags::WeaklyReferencedSymbol);
202 InitFunctions[&JD].add(KV.first);
203 }
204 }
205 return Error::success();
206 }
207
initialize(JITDylib & JD)208 Error initialize(JITDylib &JD) override {
209 LLVM_DEBUG({
210 dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n";
211 });
212 if (auto Initializers = getInitializers(JD)) {
213 LLVM_DEBUG(
214 { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; });
215 for (auto InitFnAddr : *Initializers) {
216 LLVM_DEBUG({
217 dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr)
218 << "...\n";
219 });
220 auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr);
221 InitFn();
222 }
223 } else
224 return Initializers.takeError();
225 return Error::success();
226 }
227
deinitialize(JITDylib & JD)228 Error deinitialize(JITDylib &JD) override {
229 LLVM_DEBUG({
230 dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n";
231 });
232 if (auto Deinitializers = getDeinitializers(JD)) {
233 LLVM_DEBUG({
234 dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n";
235 });
236 for (auto DeinitFnAddr : *Deinitializers) {
237 LLVM_DEBUG({
238 dbgs() << " Running init " << formatv("{0:x16}", DeinitFnAddr)
239 << "...\n";
240 });
241 auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr);
242 DeinitFn();
243 }
244 } else
245 return Deinitializers.takeError();
246
247 return Error::success();
248 }
249
registerInitFunc(JITDylib & JD,SymbolStringPtr InitName)250 void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) {
251 getExecutionSession().runSessionLocked([&]() {
252 InitFunctions[&JD].add(InitName);
253 });
254 }
255
256 private:
257
getInitializers(JITDylib & JD)258 Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) {
259 if (auto Err = issueInitLookups(JD))
260 return std::move(Err);
261
262 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
263 std::vector<JITDylib *> DFSLinkOrder;
264
265 getExecutionSession().runSessionLocked([&]() {
266 DFSLinkOrder = getDFSLinkOrder(JD);
267
268 for (auto *NextJD : DFSLinkOrder) {
269 auto IFItr = InitFunctions.find(NextJD);
270 if (IFItr != InitFunctions.end()) {
271 LookupSymbols[NextJD] = std::move(IFItr->second);
272 InitFunctions.erase(IFItr);
273 }
274 }
275 });
276
277 LLVM_DEBUG({
278 dbgs() << "JITDylib init order is [ ";
279 for (auto *JD : llvm::reverse(DFSLinkOrder))
280 dbgs() << "\"" << JD->getName() << "\" ";
281 dbgs() << "]\n";
282 dbgs() << "Looking up init functions:\n";
283 for (auto &KV : LookupSymbols)
284 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
285 });
286
287 auto &ES = getExecutionSession();
288 auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
289
290 if (!LookupResult)
291 return LookupResult.takeError();
292
293 std::vector<JITTargetAddress> Initializers;
294 while (!DFSLinkOrder.empty()) {
295 auto &NextJD = *DFSLinkOrder.back();
296 DFSLinkOrder.pop_back();
297 auto InitsItr = LookupResult->find(&NextJD);
298 if (InitsItr == LookupResult->end())
299 continue;
300 for (auto &KV : InitsItr->second)
301 Initializers.push_back(KV.second.getAddress());
302 }
303
304 return Initializers;
305 }
306
getDeinitializers(JITDylib & JD)307 Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) {
308 auto &ES = getExecutionSession();
309
310 auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits");
311
312 DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols;
313 std::vector<JITDylib *> DFSLinkOrder;
314
315 ES.runSessionLocked([&]() {
316 DFSLinkOrder = getDFSLinkOrder(JD);
317
318 for (auto *NextJD : DFSLinkOrder) {
319 auto &JDLookupSymbols = LookupSymbols[NextJD];
320 auto DIFItr = DeInitFunctions.find(NextJD);
321 if (DIFItr != DeInitFunctions.end()) {
322 LookupSymbols[NextJD] = std::move(DIFItr->second);
323 DeInitFunctions.erase(DIFItr);
324 }
325 JDLookupSymbols.add(LLJITRunAtExits,
326 SymbolLookupFlags::WeaklyReferencedSymbol);
327 }
328 });
329
330 LLVM_DEBUG({
331 dbgs() << "JITDylib deinit order is [ ";
332 for (auto *JD : DFSLinkOrder)
333 dbgs() << "\"" << JD->getName() << "\" ";
334 dbgs() << "]\n";
335 dbgs() << "Looking up deinit functions:\n";
336 for (auto &KV : LookupSymbols)
337 dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n";
338 });
339
340 auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols);
341
342 if (!LookupResult)
343 return LookupResult.takeError();
344
345 std::vector<JITTargetAddress> DeInitializers;
346 for (auto *NextJD : DFSLinkOrder) {
347 auto DeInitsItr = LookupResult->find(NextJD);
348 assert(DeInitsItr != LookupResult->end() &&
349 "Every JD should have at least __lljit_run_atexits");
350
351 auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits);
352 if (RunAtExitsItr != DeInitsItr->second.end())
353 DeInitializers.push_back(RunAtExitsItr->second.getAddress());
354
355 for (auto &KV : DeInitsItr->second)
356 if (KV.first != LLJITRunAtExits)
357 DeInitializers.push_back(KV.second.getAddress());
358 }
359
360 return DeInitializers;
361 }
362
363 // Returns a DFS traversal order of the JITDylibs reachable (via
364 // links-against edges) from JD, starting with JD itself.
getDFSLinkOrder(JITDylib & JD)365 static std::vector<JITDylib *> getDFSLinkOrder(JITDylib &JD) {
366 std::vector<JITDylib *> DFSLinkOrder;
367 std::vector<JITDylib *> WorkStack({&JD});
368 DenseSet<JITDylib *> Visited;
369
370 while (!WorkStack.empty()) {
371 auto &NextJD = *WorkStack.back();
372 WorkStack.pop_back();
373 if (Visited.count(&NextJD))
374 continue;
375 Visited.insert(&NextJD);
376 DFSLinkOrder.push_back(&NextJD);
377 NextJD.withLinkOrderDo([&](const JITDylibSearchOrder &LinkOrder) {
378 for (auto &KV : LinkOrder)
379 WorkStack.push_back(KV.first);
380 });
381 }
382
383 return DFSLinkOrder;
384 }
385
386 /// Issue lookups for all init symbols required to initialize JD (and any
387 /// JITDylibs that it depends on).
issueInitLookups(JITDylib & JD)388 Error issueInitLookups(JITDylib &JD) {
389 DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols;
390 std::vector<JITDylib *> DFSLinkOrder;
391
392 getExecutionSession().runSessionLocked([&]() {
393 DFSLinkOrder = getDFSLinkOrder(JD);
394
395 for (auto *NextJD : DFSLinkOrder) {
396 auto ISItr = InitSymbols.find(NextJD);
397 if (ISItr != InitSymbols.end()) {
398 RequiredInitSymbols[NextJD] = std::move(ISItr->second);
399 InitSymbols.erase(ISItr);
400 }
401 }
402 });
403
404 return Platform::lookupInitSymbols(getExecutionSession(),
405 RequiredInitSymbols)
406 .takeError();
407 }
408
registerAtExitHelper(void * Self,void (* F)(void *),void * Ctx,void * DSOHandle)409 static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
410 void *DSOHandle) {
411 LLVM_DEBUG({
412 dbgs() << "Registering atexit function " << (void *)F << " for JD "
413 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
414 });
415 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
416 F, Ctx, DSOHandle);
417 }
418
runAtExitsHelper(void * Self,void * DSOHandle)419 static void runAtExitsHelper(void *Self, void *DSOHandle) {
420 LLVM_DEBUG({
421 dbgs() << "Running atexit functions for JD "
422 << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n";
423 });
424 static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits(
425 DSOHandle);
426 }
427
428 // Constructs an LLVM IR module containing platform runtime globals,
429 // functions, and interposes.
createPlatformRuntimeModule()430 ThreadSafeModule createPlatformRuntimeModule() {
431 auto Ctx = std::make_unique<LLVMContext>();
432 auto M = std::make_unique<Module>("__standard_lib", *Ctx);
433 M->setDataLayout(J.getDataLayout());
434
435 auto *GenericIRPlatformSupportTy =
436 StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport");
437
438 auto *PlatformInstanceDecl = new GlobalVariable(
439 *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage,
440 nullptr, "__lljit.platform_support_instance");
441
442 auto *Int8Ty = Type::getInt8Ty(*Ctx);
443 auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
444 auto *VoidTy = Type::getVoidTy(*Ctx);
445 auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
446 auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
447 auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
448
449 addHelperAndWrapper(
450 *M, "__cxa_atexit",
451 FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
452 false),
453 GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
454 {PlatformInstanceDecl});
455
456 return ThreadSafeModule(std::move(M), std::move(Ctx));
457 }
458
459 LLJIT &J;
460 std::string InitFunctionPrefix;
461 DenseMap<JITDylib *, SymbolLookupSet> InitSymbols;
462 DenseMap<JITDylib *, SymbolLookupSet> InitFunctions;
463 DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions;
464 ItaniumCXAAtExitSupport AtExitMgr;
465 };
466
setupJITDylib(JITDylib & JD)467 Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) {
468 return S.setupJITDylib(JD);
469 }
470
notifyAdding(JITDylib & JD,const MaterializationUnit & MU)471 Error GenericLLVMIRPlatform::notifyAdding(JITDylib &JD,
472 const MaterializationUnit &MU) {
473 return S.notifyAdding(JD, MU);
474 }
475
476 Expected<ThreadSafeModule>
operator ()(ThreadSafeModule TSM,MaterializationResponsibility & R)477 GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM,
478 MaterializationResponsibility &R) {
479 auto Err = TSM.withModuleDo([&](Module &M) -> Error {
480 auto &Ctx = M.getContext();
481 auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors");
482
483 // If there's no llvm.global_ctors or it's just a decl then skip.
484 if (!GlobalCtors || GlobalCtors->isDeclaration())
485 return Error::success();
486
487 std::string InitFunctionName;
488 raw_string_ostream(InitFunctionName)
489 << InitFunctionPrefix << M.getModuleIdentifier();
490
491 MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout());
492 auto InternedName = Mangle(InitFunctionName);
493 if (auto Err =
494 R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}}))
495 return Err;
496
497 auto *InitFunc =
498 Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false),
499 GlobalValue::ExternalLinkage, InitFunctionName, &M);
500 InitFunc->setVisibility(GlobalValue::HiddenVisibility);
501 std::vector<std::pair<Function *, unsigned>> Inits;
502 for (auto E : getConstructors(M))
503 Inits.push_back(std::make_pair(E.Func, E.Priority));
504 llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS,
505 const std::pair<Function *, unsigned> &RHS) {
506 return LHS.first < RHS.first;
507 });
508 auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc);
509 IRBuilder<> IB(EntryBlock);
510 for (auto &KV : Inits)
511 IB.CreateCall(KV.first);
512 IB.CreateRetVoid();
513
514 PS.registerInitFunc(R.getTargetJITDylib(), InternedName);
515 GlobalCtors->eraseFromParent();
516 return Error::success();
517 });
518
519 if (Err)
520 return std::move(Err);
521
522 return std::move(TSM);
523 }
524
525 class MachOPlatformSupport : public LLJIT::PlatformSupport {
526 public:
527 using DLOpenType = void *(*)(const char *Name, int Mode);
528 using DLCloseType = int (*)(void *Handle);
529 using DLSymType = void *(*)(void *Handle, const char *Name);
530 using DLErrorType = const char *(*)();
531
532 struct DlFcnValues {
533 Optional<void *> RTLDDefault;
534 DLOpenType dlopen = nullptr;
535 DLCloseType dlclose = nullptr;
536 DLSymType dlsym = nullptr;
537 DLErrorType dlerror = nullptr;
538 };
539
540 static Expected<std::unique_ptr<MachOPlatformSupport>>
Create(LLJIT & J,JITDylib & PlatformJITDylib)541 Create(LLJIT &J, JITDylib &PlatformJITDylib) {
542
543 // Make process symbols visible.
544 {
545 std::string ErrMsg;
546 auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg);
547 if (!Lib.isValid())
548 return make_error<StringError>(std::move(ErrMsg),
549 inconvertibleErrorCode());
550 }
551
552 DlFcnValues DlFcn;
553
554 // Add support for RTLDDefault on known platforms.
555 #ifdef __APPLE__
556 DlFcn.RTLDDefault = reinterpret_cast<void *>(-2);
557 #endif // __APPLE__
558
559 if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen"))
560 return std::move(Err);
561 if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose"))
562 return std::move(Err);
563 if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym"))
564 return std::move(Err);
565 if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror"))
566 return std::move(Err);
567
568 std::unique_ptr<MachOPlatformSupport> MP(
569 new MachOPlatformSupport(J, PlatformJITDylib, DlFcn));
570 return std::move(MP);
571 }
572
initialize(JITDylib & JD)573 Error initialize(JITDylib &JD) override {
574 LLVM_DEBUG({
575 dbgs() << "MachOPlatformSupport initializing \"" << JD.getName()
576 << "\"\n";
577 });
578
579 auto InitSeq = MP.getInitializerSequence(JD);
580 if (!InitSeq)
581 return InitSeq.takeError();
582
583 // If ObjC is not enabled but there are JIT'd ObjC inits then return
584 // an error.
585 if (!objCRegistrationEnabled())
586 for (auto &KV : *InitSeq) {
587 if (!KV.second.getObjCSelRefsSections().empty() ||
588 !KV.second.getObjCClassListSections().empty())
589 return make_error<StringError>("JITDylib " + KV.first->getName() +
590 " contains objc metadata but objc"
591 " is not enabled",
592 inconvertibleErrorCode());
593 }
594
595 // Run the initializers.
596 for (auto &KV : *InitSeq) {
597 if (objCRegistrationEnabled()) {
598 KV.second.registerObjCSelectors();
599 if (auto Err = KV.second.registerObjCClasses()) {
600 // FIXME: Roll back registrations on error?
601 return Err;
602 }
603 }
604 KV.second.runModInits();
605 }
606
607 return Error::success();
608 }
609
deinitialize(JITDylib & JD)610 Error deinitialize(JITDylib &JD) override {
611 auto &ES = J.getExecutionSession();
612 if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) {
613 for (auto &KV : *DeinitSeq) {
614 auto DSOHandleName = ES.intern("___dso_handle");
615
616 // FIXME: Run DeInits here.
617 auto Result = ES.lookup(
618 {{KV.first, JITDylibLookupFlags::MatchAllSymbols}},
619 SymbolLookupSet(DSOHandleName,
620 SymbolLookupFlags::WeaklyReferencedSymbol));
621 if (!Result)
622 return Result.takeError();
623 if (Result->empty())
624 continue;
625 assert(Result->count(DSOHandleName) &&
626 "Result does not contain __dso_handle");
627 auto *DSOHandle = jitTargetAddressToPointer<void *>(
628 Result->begin()->second.getAddress());
629 AtExitMgr.runAtExits(DSOHandle);
630 }
631 } else
632 return DeinitSeq.takeError();
633 return Error::success();
634 }
635
636 private:
637 template <typename FunctionPtrTy>
hookUpFunction(FunctionPtrTy & Fn,const char * Name)638 static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) {
639 if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) {
640 Fn = reinterpret_cast<FunctionPtrTy>(Fn);
641 return Error::success();
642 }
643
644 return make_error<StringError>((Twine("Can not enable MachO JIT Platform: "
645 "missing function: ") +
646 Name)
647 .str(),
648 inconvertibleErrorCode());
649 }
650
MachOPlatformSupport(LLJIT & J,JITDylib & PlatformJITDylib,DlFcnValues DlFcn)651 MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn)
652 : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) {
653
654 SymbolMap HelperSymbols;
655
656 // platform and atexit helpers.
657 HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] =
658 JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags());
659 HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] =
660 JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper),
661 JITSymbolFlags());
662 HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] =
663 JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper),
664 JITSymbolFlags());
665
666 // dlfcn helpers.
667 HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] =
668 JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper),
669 JITSymbolFlags());
670 HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] =
671 JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper),
672 JITSymbolFlags());
673 HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] =
674 JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper),
675 JITSymbolFlags());
676 HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] =
677 JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper),
678 JITSymbolFlags());
679
680 cantFail(
681 PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols))));
682 cantFail(MP.setupJITDylib(J.getMainJITDylib()));
683 cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule()));
684 }
685
setupPlatform(LLJIT & J)686 static MachOPlatform &setupPlatform(LLJIT &J) {
687 auto Tmp = std::make_unique<MachOPlatform>(
688 J.getExecutionSession(),
689 static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()),
690 createStandardSymbolsObject(J));
691 auto &MP = *Tmp;
692 J.getExecutionSession().setPlatform(std::move(Tmp));
693 return MP;
694 }
695
createStandardSymbolsObject(LLJIT & J)696 static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) {
697 LLVMContext Ctx;
698 Module M("__standard_symbols", Ctx);
699 M.setDataLayout(J.getDataLayout());
700
701 auto *Int64Ty = Type::getInt64Ty(Ctx);
702
703 auto *DSOHandle =
704 new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage,
705 ConstantInt::get(Int64Ty, 0), "__dso_handle");
706 DSOHandle->setVisibility(GlobalValue::DefaultVisibility);
707
708 return cantFail(J.getIRCompileLayer().getCompiler()(M));
709 }
710
createPlatformRuntimeModule()711 ThreadSafeModule createPlatformRuntimeModule() {
712 auto Ctx = std::make_unique<LLVMContext>();
713 auto M = std::make_unique<Module>("__standard_lib", *Ctx);
714 M->setDataLayout(J.getDataLayout());
715
716 auto *MachOPlatformSupportTy =
717 StructType::create(*Ctx, "lljit.MachOPlatformSupport");
718
719 auto *PlatformInstanceDecl = new GlobalVariable(
720 *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr,
721 "__lljit.platform_support_instance");
722
723 auto *Int8Ty = Type::getInt8Ty(*Ctx);
724 auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT);
725 auto *VoidTy = Type::getVoidTy(*Ctx);
726 auto *BytePtrTy = PointerType::getUnqual(Int8Ty);
727 auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false);
728 auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy);
729
730 addHelperAndWrapper(
731 *M, "__cxa_atexit",
732 FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy},
733 false),
734 GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper",
735 {PlatformInstanceDecl});
736
737 addHelperAndWrapper(*M, "dlopen",
738 FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false),
739 GlobalValue::DefaultVisibility, "__lljit.dlopen_helper",
740 {PlatformInstanceDecl});
741
742 addHelperAndWrapper(*M, "dlclose",
743 FunctionType::get(IntTy, {BytePtrTy}, false),
744 GlobalValue::DefaultVisibility,
745 "__lljit.dlclose_helper", {PlatformInstanceDecl});
746
747 addHelperAndWrapper(
748 *M, "dlsym",
749 FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false),
750 GlobalValue::DefaultVisibility, "__lljit.dlsym_helper",
751 {PlatformInstanceDecl});
752
753 addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false),
754 GlobalValue::DefaultVisibility,
755 "__lljit.dlerror_helper", {PlatformInstanceDecl});
756
757 return ThreadSafeModule(std::move(M), std::move(Ctx));
758 }
759
registerAtExitHelper(void * Self,void (* F)(void *),void * Ctx,void * DSOHandle)760 static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx,
761 void *DSOHandle) {
762 static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit(
763 F, Ctx, DSOHandle);
764 }
765
runAtExitsHelper(void * Self,void * DSOHandle)766 static void runAtExitsHelper(void *Self, void *DSOHandle) {
767 static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle);
768 }
769
jit_dlopen(const char * Path,int Mode)770 void *jit_dlopen(const char *Path, int Mode) {
771 JITDylib *JDToOpen = nullptr;
772 // FIXME: Do the right thing with Mode flags.
773 {
774 std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
775
776 // Clear any existing error messages.
777 dlErrorMsgs.erase(std::this_thread::get_id());
778
779 if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) {
780 auto I = JDRefCounts.find(JD);
781 if (I != JDRefCounts.end()) {
782 ++I->second;
783 return JD;
784 }
785
786 JDRefCounts[JD] = 1;
787 JDToOpen = JD;
788 }
789 }
790
791 if (JDToOpen) {
792 if (auto Err = initialize(*JDToOpen)) {
793 recordError(std::move(Err));
794 return 0;
795 }
796 }
797
798 // Fall through to dlopen if no JITDylib found for Path.
799 return DlFcn.dlopen(Path, Mode);
800 }
801
dlopenHelper(void * Self,const char * Path,int Mode)802 static void *dlopenHelper(void *Self, const char *Path, int Mode) {
803 return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode);
804 }
805
jit_dlclose(void * Handle)806 int jit_dlclose(void *Handle) {
807 JITDylib *JDToClose = nullptr;
808
809 {
810 std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
811
812 // Clear any existing error messages.
813 dlErrorMsgs.erase(std::this_thread::get_id());
814
815 auto I = JDRefCounts.find(Handle);
816 if (I != JDRefCounts.end()) {
817 --I->second;
818 if (I->second == 0) {
819 JDRefCounts.erase(I);
820 JDToClose = static_cast<JITDylib *>(Handle);
821 } else
822 return 0;
823 }
824 }
825
826 if (JDToClose) {
827 if (auto Err = deinitialize(*JDToClose)) {
828 recordError(std::move(Err));
829 return -1;
830 }
831 return 0;
832 }
833
834 // Fall through to dlclose if no JITDylib found for Path.
835 return DlFcn.dlclose(Handle);
836 }
837
dlcloseHelper(void * Self,void * Handle)838 static int dlcloseHelper(void *Self, void *Handle) {
839 return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle);
840 }
841
jit_dlsym(void * Handle,const char * Name)842 void *jit_dlsym(void *Handle, const char *Name) {
843 JITDylibSearchOrder JITSymSearchOrder;
844
845 // FIXME: RTLD_NEXT, RTLD_SELF not supported.
846 {
847 std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
848
849 // Clear any existing error messages.
850 dlErrorMsgs.erase(std::this_thread::get_id());
851
852 if (JDRefCounts.count(Handle)) {
853 JITSymSearchOrder.push_back(
854 {static_cast<JITDylib *>(Handle),
855 JITDylibLookupFlags::MatchExportedSymbolsOnly});
856 } else if (Handle == DlFcn.RTLDDefault) {
857 for (auto &KV : JDRefCounts)
858 JITSymSearchOrder.push_back(
859 {static_cast<JITDylib *>(KV.first),
860 JITDylibLookupFlags::MatchExportedSymbolsOnly});
861 }
862 }
863
864 if (!JITSymSearchOrder.empty()) {
865 auto MangledName = J.mangleAndIntern(Name);
866 SymbolLookupSet Syms(MangledName,
867 SymbolLookupFlags::WeaklyReferencedSymbol);
868 if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms,
869 LookupKind::DLSym)) {
870 auto I = Result->find(MangledName);
871 if (I != Result->end())
872 return jitTargetAddressToPointer<void *>(I->second.getAddress());
873 } else {
874 recordError(Result.takeError());
875 return 0;
876 }
877 }
878
879 // Fall through to dlsym.
880 return DlFcn.dlsym(Handle, Name);
881 }
882
dlsymHelper(void * Self,void * Handle,const char * Name)883 static void *dlsymHelper(void *Self, void *Handle, const char *Name) {
884 return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name);
885 }
886
jit_dlerror()887 const char *jit_dlerror() {
888 {
889 std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
890 auto I = dlErrorMsgs.find(std::this_thread::get_id());
891 if (I != dlErrorMsgs.end())
892 return I->second->c_str();
893 }
894 return DlFcn.dlerror();
895 }
896
dlerrorHelper(void * Self)897 static const char *dlerrorHelper(void *Self) {
898 return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror();
899 }
900
recordError(Error Err)901 void recordError(Error Err) {
902 std::lock_guard<std::mutex> Lock(PlatformSupportMutex);
903 dlErrorMsgs[std::this_thread::get_id()] =
904 std::make_unique<std::string>(toString(std::move(Err)));
905 }
906
907 std::mutex PlatformSupportMutex;
908 LLJIT &J;
909 MachOPlatform &MP;
910 DlFcnValues DlFcn;
911 ItaniumCXAAtExitSupport AtExitMgr;
912 DenseMap<void *, unsigned> JDRefCounts;
913 std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs;
914 };
915
916 } // end anonymous namespace
917
918 namespace llvm {
919 namespace orc {
920
setInitTransform(LLJIT & J,IRTransformLayer::TransformFunction T)921 void LLJIT::PlatformSupport::setInitTransform(
922 LLJIT &J, IRTransformLayer::TransformFunction T) {
923 J.InitHelperTransformLayer->setTransform(std::move(T));
924 }
925
~PlatformSupport()926 LLJIT::PlatformSupport::~PlatformSupport() {}
927
prepareForConstruction()928 Error LLJITBuilderState::prepareForConstruction() {
929
930 LLVM_DEBUG(dbgs() << "Preparing to create LLIT instance...\n");
931
932 if (!JTMB) {
933 LLVM_DEBUG({
934 dbgs() << " No explicitly set JITTargetMachineBuilder. "
935 "Detecting host...\n";
936 });
937 if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost())
938 JTMB = std::move(*JTMBOrErr);
939 else
940 return JTMBOrErr.takeError();
941 }
942
943 LLVM_DEBUG({
944 dbgs() << " JITTargetMachineBuilder is " << JTMB << "\n"
945 << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No")
946 << "\n"
947 << " DataLayout: ";
948 if (DL)
949 dbgs() << DL->getStringRepresentation() << "\n";
950 else
951 dbgs() << "None (will be created by JITTargetMachineBuilder)\n";
952
953 dbgs() << " Custom object-linking-layer creator: "
954 << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n"
955 << " Custom compile-function creator: "
956 << (CreateCompileFunction ? "Yes" : "No") << "\n"
957 << " Custom platform-setup function: "
958 << (SetUpPlatform ? "Yes" : "No") << "\n"
959 << " Number of compile threads: " << NumCompileThreads;
960 if (!NumCompileThreads)
961 dbgs() << " (code will be compiled on the execution thread)\n";
962 else
963 dbgs() << "\n";
964 });
965
966 // If the client didn't configure any linker options then auto-configure the
967 // JIT linker.
968 if (!CreateObjectLinkingLayer) {
969 auto &TT = JTMB->getTargetTriple();
970 if (TT.isOSBinFormatMachO() &&
971 (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) {
972
973 JTMB->setRelocationModel(Reloc::PIC_);
974 JTMB->setCodeModel(CodeModel::Small);
975 CreateObjectLinkingLayer =
976 [](ExecutionSession &ES,
977 const Triple &) -> std::unique_ptr<ObjectLayer> {
978 auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(
979 ES, std::make_unique<jitlink::InProcessMemoryManager>());
980 ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>(
981 jitlink::InProcessEHFrameRegistrar::getInstance()));
982 return std::move(ObjLinkingLayer);
983 };
984 }
985 }
986
987 return Error::success();
988 }
989
~LLJIT()990 LLJIT::~LLJIT() {
991 if (CompileThreads)
992 CompileThreads->wait();
993 }
994
addIRModule(JITDylib & JD,ThreadSafeModule TSM)995 Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
996 assert(TSM && "Can not add null module");
997
998 if (auto Err =
999 TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
1000 return Err;
1001
1002 return InitHelperTransformLayer->add(JD, std::move(TSM),
1003 ES->allocateVModule());
1004 }
1005
addObjectFile(JITDylib & JD,std::unique_ptr<MemoryBuffer> Obj)1006 Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) {
1007 assert(Obj && "Can not add null object");
1008
1009 return ObjTransformLayer.add(JD, std::move(Obj), ES->allocateVModule());
1010 }
1011
lookupLinkerMangled(JITDylib & JD,SymbolStringPtr Name)1012 Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
1013 SymbolStringPtr Name) {
1014 return ES->lookup(
1015 makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name);
1016 }
1017
1018 std::unique_ptr<ObjectLayer>
createObjectLinkingLayer(LLJITBuilderState & S,ExecutionSession & ES)1019 LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
1020
1021 // If the config state provided an ObjectLinkingLayer factory then use it.
1022 if (S.CreateObjectLinkingLayer)
1023 return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
1024
1025 // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
1026 // a new SectionMemoryManager for each object.
1027 auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
1028 auto ObjLinkingLayer =
1029 std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
1030
1031 if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) {
1032 ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
1033 ObjLinkingLayer->setAutoClaimResponsibilityForObjectSymbols(true);
1034 }
1035
1036 // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
1037 // errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
1038 // just return ObjLinkingLayer) once those bots are upgraded.
1039 return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer));
1040 }
1041
1042 Expected<std::unique_ptr<IRCompileLayer::IRCompiler>>
createCompileFunction(LLJITBuilderState & S,JITTargetMachineBuilder JTMB)1043 LLJIT::createCompileFunction(LLJITBuilderState &S,
1044 JITTargetMachineBuilder JTMB) {
1045
1046 /// If there is a custom compile function creator set then use it.
1047 if (S.CreateCompileFunction)
1048 return S.CreateCompileFunction(std::move(JTMB));
1049
1050 // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler,
1051 // depending on the number of threads requested.
1052 if (S.NumCompileThreads > 0)
1053 return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB));
1054
1055 auto TM = JTMB.createTargetMachine();
1056 if (!TM)
1057 return TM.takeError();
1058
1059 return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM));
1060 }
1061
LLJIT(LLJITBuilderState & S,Error & Err)1062 LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
1063 : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(),
1064 DL(""), TT(S.JTMB->getTargetTriple()),
1065 ObjLinkingLayer(createObjectLinkingLayer(S, *ES)),
1066 ObjTransformLayer(*this->ES, *ObjLinkingLayer) {
1067
1068 ErrorAsOutParameter _(&Err);
1069
1070 if (auto MainOrErr = this->ES->createJITDylib("main"))
1071 Main = &*MainOrErr;
1072 else {
1073 Err = MainOrErr.takeError();
1074 return;
1075 }
1076
1077 if (S.DL)
1078 DL = std::move(*S.DL);
1079 else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget())
1080 DL = std::move(*DLOrErr);
1081 else {
1082 Err = DLOrErr.takeError();
1083 return;
1084 }
1085
1086 {
1087 auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB));
1088 if (!CompileFunction) {
1089 Err = CompileFunction.takeError();
1090 return;
1091 }
1092 CompileLayer = std::make_unique<IRCompileLayer>(
1093 *ES, ObjTransformLayer, std::move(*CompileFunction));
1094 TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
1095 InitHelperTransformLayer =
1096 std::make_unique<IRTransformLayer>(*ES, *TransformLayer);
1097 }
1098
1099 if (S.NumCompileThreads > 0) {
1100 InitHelperTransformLayer->setCloneToNewContextOnEmit(true);
1101 CompileThreads =
1102 std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads));
1103 ES->setDispatchMaterialization(
1104 [this](std::unique_ptr<MaterializationUnit> MU,
1105 MaterializationResponsibility MR) {
1106 // FIXME: Switch to move capture once ThreadPool uses unique_function.
1107 auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU));
1108 auto SharedMR =
1109 std::make_shared<MaterializationResponsibility>(std::move(MR));
1110 auto Work = [SharedMU, SharedMR]() mutable {
1111 SharedMU->materialize(std::move(*SharedMR));
1112 };
1113 CompileThreads->async(std::move(Work));
1114 });
1115 }
1116
1117 if (S.SetUpPlatform)
1118 Err = S.SetUpPlatform(*this);
1119 else
1120 setUpGenericLLVMIRPlatform(*this);
1121 }
1122
mangle(StringRef UnmangledName) const1123 std::string LLJIT::mangle(StringRef UnmangledName) const {
1124 std::string MangledName;
1125 {
1126 raw_string_ostream MangledNameStream(MangledName);
1127 Mangler::getNameWithPrefix(MangledNameStream, UnmangledName, DL);
1128 }
1129 return MangledName;
1130 }
1131
applyDataLayout(Module & M)1132 Error LLJIT::applyDataLayout(Module &M) {
1133 if (M.getDataLayout().isDefault())
1134 M.setDataLayout(DL);
1135
1136 if (M.getDataLayout() != DL)
1137 return make_error<StringError>(
1138 "Added modules have incompatible data layouts: " +
1139 M.getDataLayout().getStringRepresentation() + " (module) vs " +
1140 DL.getStringRepresentation() + " (jit)",
1141 inconvertibleErrorCode());
1142
1143 return Error::success();
1144 }
1145
setUpGenericLLVMIRPlatform(LLJIT & J)1146 void setUpGenericLLVMIRPlatform(LLJIT &J) {
1147 LLVM_DEBUG(
1148 { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; });
1149 J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J));
1150 }
1151
setUpMachOPlatform(LLJIT & J)1152 Error setUpMachOPlatform(LLJIT &J) {
1153 LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; });
1154 auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib());
1155 if (!MP)
1156 return MP.takeError();
1157 J.setPlatformSupport(std::move(*MP));
1158 return Error::success();
1159 }
1160
prepareForConstruction()1161 Error LLLazyJITBuilderState::prepareForConstruction() {
1162 if (auto Err = LLJITBuilderState::prepareForConstruction())
1163 return Err;
1164 TT = JTMB->getTargetTriple();
1165 return Error::success();
1166 }
1167
addLazyIRModule(JITDylib & JD,ThreadSafeModule TSM)1168 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
1169 assert(TSM && "Can not add null module");
1170
1171 if (auto Err = TSM.withModuleDo(
1172 [&](Module &M) -> Error { return applyDataLayout(M); }))
1173 return Err;
1174
1175 return CODLayer->add(JD, std::move(TSM), ES->allocateVModule());
1176 }
1177
LLLazyJIT(LLLazyJITBuilderState & S,Error & Err)1178 LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
1179
1180 // If LLJIT construction failed then bail out.
1181 if (Err)
1182 return;
1183
1184 ErrorAsOutParameter _(&Err);
1185
1186 /// Take/Create the lazy-compile callthrough manager.
1187 if (S.LCTMgr)
1188 LCTMgr = std::move(S.LCTMgr);
1189 else {
1190 if (auto LCTMgrOrErr = createLocalLazyCallThroughManager(
1191 S.TT, *ES, S.LazyCompileFailureAddr))
1192 LCTMgr = std::move(*LCTMgrOrErr);
1193 else {
1194 Err = LCTMgrOrErr.takeError();
1195 return;
1196 }
1197 }
1198
1199 // Take/Create the indirect stubs manager builder.
1200 auto ISMBuilder = std::move(S.ISMBuilder);
1201
1202 // If none was provided, try to build one.
1203 if (!ISMBuilder)
1204 ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT);
1205
1206 // No luck. Bail out.
1207 if (!ISMBuilder) {
1208 Err = make_error<StringError>("Could not construct "
1209 "IndirectStubsManagerBuilder for target " +
1210 S.TT.str(),
1211 inconvertibleErrorCode());
1212 return;
1213 }
1214
1215 // Create the COD layer.
1216 CODLayer = std::make_unique<CompileOnDemandLayer>(
1217 *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder));
1218
1219 if (S.NumCompileThreads > 0)
1220 CODLayer->setCloneToNewContextOnEmit(true);
1221 }
1222
1223 } // End namespace orc.
1224 } // End namespace llvm.
1225