1 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
10 #define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
11
12 #include "llvm-c/OrcBindings.h"
13 #include "llvm-c/TargetMachine.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/ADT/StringRef.h"
16 #include "llvm/ExecutionEngine/JITSymbol.h"
17 #include "llvm/ExecutionEngine/JITEventListener.h"
18 #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h"
19 #include "llvm/ExecutionEngine/Orc/CompileUtils.h"
20 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
21 #include "llvm/ExecutionEngine/Orc/IRCompileLayer.h"
22 #include "llvm/ExecutionEngine/Orc/LambdaResolver.h"
23 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
24 #include "llvm/ExecutionEngine/RuntimeDyld.h"
25 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
26 #include "llvm/IR/DataLayout.h"
27 #include "llvm/IR/Mangler.h"
28 #include "llvm/IR/Module.h"
29 #include "llvm/Support/CBindingWrapping.h"
30 #include "llvm/Support/Error.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include "llvm/Target/TargetMachine.h"
33 #include <algorithm>
34 #include <cstdint>
35 #include <functional>
36 #include <map>
37 #include <memory>
38 #include <set>
39 #include <string>
40 #include <vector>
41
42 namespace llvm {
43
44 class OrcCBindingsStack;
45
DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack,LLVMOrcJITStackRef)46 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef)
47 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
48
49 namespace detail {
50
51 // FIXME: Kill this off once the Layer concept becomes an interface.
52 class GenericLayer {
53 public:
54 virtual ~GenericLayer() = default;
55
56 virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
57 bool ExportedSymbolsOnly) = 0;
58 virtual Error removeModule(orc::VModuleKey K) = 0;
59 };
60
61 template <typename LayerT> class GenericLayerImpl : public GenericLayer {
62 public:
63 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
64
65 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
66 bool ExportedSymbolsOnly) override {
67 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
68 }
69
70 Error removeModule(orc::VModuleKey K) override {
71 return Layer.removeModule(K);
72 }
73
74 private:
75 LayerT &Layer;
76 };
77
78 template <>
79 class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer {
80 private:
81 using LayerT = orc::LegacyRTDyldObjectLinkingLayer;
82 public:
83 GenericLayerImpl(LayerT &Layer) : Layer(Layer) {}
84
85 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
86 bool ExportedSymbolsOnly) override {
87 return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly);
88 }
89
90 Error removeModule(orc::VModuleKey K) override {
91 return Layer.removeObject(K);
92 }
93
94 private:
95 LayerT &Layer;
96 };
97
98 template <typename LayerT>
99 std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
100 return std::make_unique<GenericLayerImpl<LayerT>>(Layer);
101 }
102
103 } // end namespace detail
104
105 class OrcCBindingsStack {
106 public:
107
108 using CompileCallbackMgr = orc::JITCompileCallbackManager;
109 using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer;
110 using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>;
111 using CODLayerT =
112 orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>;
113
114 using CallbackManagerBuilder =
115 std::function<std::unique_ptr<CompileCallbackMgr>()>;
116
117 using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT;
118
119 private:
120
121 using OwningObject = object::OwningBinary<object::ObjectFile>;
122
123 class CBindingsResolver : public orc::SymbolResolver {
124 public:
CBindingsResolver(OrcCBindingsStack & Stack,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)125 CBindingsResolver(OrcCBindingsStack &Stack,
126 LLVMOrcSymbolResolverFn ExternalResolver,
127 void *ExternalResolverCtx)
128 : Stack(Stack), ExternalResolver(std::move(ExternalResolver)),
129 ExternalResolverCtx(std::move(ExternalResolverCtx)) {}
130
131 orc::SymbolNameSet
getResponsibilitySet(const orc::SymbolNameSet & Symbols)132 getResponsibilitySet(const orc::SymbolNameSet &Symbols) override {
133 orc::SymbolNameSet Result;
134
135 for (auto &S : Symbols) {
136 if (auto Sym = findSymbol(std::string(*S))) {
137 if (!Sym.getFlags().isStrong())
138 Result.insert(S);
139 } else if (auto Err = Sym.takeError()) {
140 Stack.reportError(std::move(Err));
141 return orc::SymbolNameSet();
142 }
143 }
144
145 return Result;
146 }
147
148 orc::SymbolNameSet
lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,orc::SymbolNameSet Symbols)149 lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query,
150 orc::SymbolNameSet Symbols) override {
151 orc::SymbolNameSet UnresolvedSymbols;
152
153 for (auto &S : Symbols) {
154 if (auto Sym = findSymbol(std::string(*S))) {
155 if (auto Addr = Sym.getAddress()) {
156 Query->notifySymbolMetRequiredState(
157 S, JITEvaluatedSymbol(*Addr, Sym.getFlags()));
158 } else {
159 Stack.ES.legacyFailQuery(*Query, Addr.takeError());
160 return orc::SymbolNameSet();
161 }
162 } else if (auto Err = Sym.takeError()) {
163 Stack.ES.legacyFailQuery(*Query, std::move(Err));
164 return orc::SymbolNameSet();
165 } else
166 UnresolvedSymbols.insert(S);
167 }
168
169 if (Query->isComplete())
170 Query->handleComplete();
171
172 return UnresolvedSymbols;
173 }
174
175 private:
findSymbol(const std::string & Name)176 JITSymbol findSymbol(const std::string &Name) {
177 // Search order:
178 // 1. JIT'd symbols.
179 // 2. Runtime overrides.
180 // 3. External resolver (if present).
181
182 if (Stack.CODLayer) {
183 if (auto Sym = Stack.CODLayer->findSymbol(Name, true))
184 return Sym;
185 else if (auto Err = Sym.takeError())
186 return Sym.takeError();
187 } else {
188 if (auto Sym = Stack.CompileLayer.findSymbol(Name, true))
189 return Sym;
190 else if (auto Err = Sym.takeError())
191 return Sym.takeError();
192 }
193
194 if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name))
195 return Sym;
196
197 if (ExternalResolver)
198 return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx),
199 JITSymbolFlags::Exported);
200
201 return JITSymbol(nullptr);
202 }
203
204 OrcCBindingsStack &Stack;
205 LLVMOrcSymbolResolverFn ExternalResolver;
206 void *ExternalResolverCtx = nullptr;
207 };
208
209 public:
OrcCBindingsStack(TargetMachine & TM,IndirectStubsManagerBuilder IndirectStubsMgrBuilder)210 OrcCBindingsStack(TargetMachine &TM,
211 IndirectStubsManagerBuilder IndirectStubsMgrBuilder)
212 : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()),
213 IndirectStubsMgr(IndirectStubsMgrBuilder()),
214 ObjectLayer(
215 AcknowledgeORCv1Deprecation, ES,
216 [this](orc::VModuleKey K) {
217 auto ResolverI = Resolvers.find(K);
218 assert(ResolverI != Resolvers.end() &&
219 "No resolver for module K");
220 auto Resolver = std::move(ResolverI->second);
221 Resolvers.erase(ResolverI);
222 return ObjLayerT::Resources{
223 std::make_shared<SectionMemoryManager>(), Resolver};
224 },
225 nullptr,
226 [this](orc::VModuleKey K, const object::ObjectFile &Obj,
227 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
228 this->notifyFinalized(K, Obj, LoadedObjInfo);
229 },
230 [this](orc::VModuleKey K, const object::ObjectFile &Obj) {
231 this->notifyFreed(K, Obj);
232 }),
233 CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer,
234 orc::SimpleCompiler(TM)),
235 CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(),
236 std::move(IndirectStubsMgrBuilder), Resolvers)),
237 CXXRuntimeOverrides(
238 AcknowledgeORCv1Deprecation,
239 [this](const std::string &S) { return mangle(S); }) {}
240
shutdown()241 Error shutdown() {
242 // Run any destructors registered with __cxa_atexit.
243 CXXRuntimeOverrides.runDestructors();
244 // Run any IR destructors.
245 for (auto &DtorRunner : IRStaticDestructorRunners)
246 if (auto Err = DtorRunner.runViaLayer(*this))
247 return Err;
248 return Error::success();
249 }
250
mangle(StringRef Name)251 std::string mangle(StringRef Name) {
252 std::string MangledName;
253 {
254 raw_string_ostream MangledNameStream(MangledName);
255 Mangler::getNameWithPrefix(MangledNameStream, Name, DL);
256 }
257 return MangledName;
258 }
259
260 template <typename PtrTy>
fromTargetAddress(JITTargetAddress Addr)261 static PtrTy fromTargetAddress(JITTargetAddress Addr) {
262 return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr));
263 }
264
265 Expected<JITTargetAddress>
createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,void * CallbackCtx)266 createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback,
267 void *CallbackCtx) {
268 auto WrappedCallback = [=]() -> JITTargetAddress {
269 return Callback(wrap(this), CallbackCtx);
270 };
271
272 return CCMgr->getCompileCallback(std::move(WrappedCallback));
273 }
274
createIndirectStub(StringRef StubName,JITTargetAddress Addr)275 Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) {
276 return IndirectStubsMgr->createStub(StubName, Addr,
277 JITSymbolFlags::Exported);
278 }
279
setIndirectStubPointer(StringRef Name,JITTargetAddress Addr)280 Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) {
281 return IndirectStubsMgr->updatePointer(Name, Addr);
282 }
283
284 template <typename LayerT>
285 Expected<orc::VModuleKey>
addIRModule(LayerT & Layer,std::unique_ptr<Module> M,std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)286 addIRModule(LayerT &Layer, std::unique_ptr<Module> M,
287 std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr,
288 LLVMOrcSymbolResolverFn ExternalResolver,
289 void *ExternalResolverCtx) {
290
291 // Attach a data-layout if one isn't already present.
292 if (M->getDataLayout().isDefault())
293 M->setDataLayout(DL);
294
295 // Record the static constructors and destructors. We have to do this before
296 // we hand over ownership of the module to the JIT.
297 std::vector<std::string> CtorNames, DtorNames;
298 for (auto Ctor : orc::getConstructors(*M))
299 CtorNames.push_back(mangle(Ctor.Func->getName()));
300 for (auto Dtor : orc::getDestructors(*M))
301 DtorNames.push_back(mangle(Dtor.Func->getName()));
302
303 // Add the module to the JIT.
304 auto K = ES.allocateVModule();
305 Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver,
306 ExternalResolverCtx);
307 if (auto Err = Layer.addModule(K, std::move(M)))
308 return std::move(Err);
309
310 KeyLayers[K] = detail::createGenericLayer(Layer);
311
312 // Run the static constructors, and save the static destructor runner for
313 // execution when the JIT is torn down.
314 orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(
315 AcknowledgeORCv1Deprecation, std::move(CtorNames), K);
316 if (auto Err = CtorRunner.runViaLayer(*this))
317 return std::move(Err);
318
319 IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation,
320 std::move(DtorNames), K);
321
322 return K;
323 }
324
325 Expected<orc::VModuleKey>
addIRModuleEager(std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)326 addIRModuleEager(std::unique_ptr<Module> M,
327 LLVMOrcSymbolResolverFn ExternalResolver,
328 void *ExternalResolverCtx) {
329 return addIRModule(CompileLayer, std::move(M),
330 std::make_unique<SectionMemoryManager>(),
331 std::move(ExternalResolver), ExternalResolverCtx);
332 }
333
334 Expected<orc::VModuleKey>
addIRModuleLazy(std::unique_ptr<Module> M,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)335 addIRModuleLazy(std::unique_ptr<Module> M,
336 LLVMOrcSymbolResolverFn ExternalResolver,
337 void *ExternalResolverCtx) {
338 if (!CODLayer)
339 return make_error<StringError>("Can not add lazy module: No compile "
340 "callback manager available",
341 inconvertibleErrorCode());
342
343 return addIRModule(*CODLayer, std::move(M),
344 std::make_unique<SectionMemoryManager>(),
345 std::move(ExternalResolver), ExternalResolverCtx);
346 }
347
removeModule(orc::VModuleKey K)348 Error removeModule(orc::VModuleKey K) {
349 // FIXME: Should error release the module key?
350 if (auto Err = KeyLayers[K]->removeModule(K))
351 return Err;
352 ES.releaseVModule(K);
353 KeyLayers.erase(K);
354 return Error::success();
355 }
356
addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,LLVMOrcSymbolResolverFn ExternalResolver,void * ExternalResolverCtx)357 Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer,
358 LLVMOrcSymbolResolverFn ExternalResolver,
359 void *ExternalResolverCtx) {
360 if (auto Obj = object::ObjectFile::createObjectFile(
361 ObjBuffer->getMemBufferRef())) {
362
363 auto K = ES.allocateVModule();
364 Resolvers[K] = std::make_shared<CBindingsResolver>(
365 *this, ExternalResolver, ExternalResolverCtx);
366
367 if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer)))
368 return std::move(Err);
369
370 KeyLayers[K] = detail::createGenericLayer(ObjectLayer);
371
372 return K;
373 } else
374 return Obj.takeError();
375 }
376
findSymbol(const std::string & Name,bool ExportedSymbolsOnly)377 JITSymbol findSymbol(const std::string &Name,
378 bool ExportedSymbolsOnly) {
379 if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly))
380 return Sym;
381 if (CODLayer)
382 return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly);
383 return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly);
384 }
385
findSymbolIn(orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)386 JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name,
387 bool ExportedSymbolsOnly) {
388 assert(KeyLayers.count(K) && "looking up symbol in unknown module");
389 return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly);
390 }
391
findSymbolAddress(const std::string & Name,bool ExportedSymbolsOnly)392 Expected<JITTargetAddress> findSymbolAddress(const std::string &Name,
393 bool ExportedSymbolsOnly) {
394 if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) {
395 // Successful lookup, non-null symbol:
396 if (auto AddrOrErr = Sym.getAddress())
397 return *AddrOrErr;
398 else
399 return AddrOrErr.takeError();
400 } else if (auto Err = Sym.takeError()) {
401 // Lookup failure - report error.
402 return std::move(Err);
403 }
404
405 // No symbol not found. Return 0.
406 return 0;
407 }
408
findSymbolAddressIn(orc::VModuleKey K,const std::string & Name,bool ExportedSymbolsOnly)409 Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K,
410 const std::string &Name,
411 bool ExportedSymbolsOnly) {
412 if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) {
413 // Successful lookup, non-null symbol:
414 if (auto AddrOrErr = Sym.getAddress())
415 return *AddrOrErr;
416 else
417 return AddrOrErr.takeError();
418 } else if (auto Err = Sym.takeError()) {
419 // Lookup failure - report error.
420 return std::move(Err);
421 }
422
423 // Symbol not found. Return 0.
424 return 0;
425 }
426
getErrorMessage()427 const std::string &getErrorMessage() const { return ErrMsg; }
428
RegisterJITEventListener(JITEventListener * L)429 void RegisterJITEventListener(JITEventListener *L) {
430 if (!L)
431 return;
432 EventListeners.push_back(L);
433 }
434
UnregisterJITEventListener(JITEventListener * L)435 void UnregisterJITEventListener(JITEventListener *L) {
436 if (!L)
437 return;
438
439 auto I = find(reverse(EventListeners), L);
440 if (I != EventListeners.rend()) {
441 std::swap(*I, EventListeners.back());
442 EventListeners.pop_back();
443 }
444 }
445
446 private:
447 using ResolverMap =
448 std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>;
449
450 static std::unique_ptr<CompileCallbackMgr>
createCompileCallbackManager(TargetMachine & TM,orc::ExecutionSession & ES)451 createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) {
452 auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0);
453 if (!CCMgr) {
454 // FIXME: It would be good if we could report this somewhere, but we do
455 // have an instance yet.
456 logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: ");
457 return nullptr;
458 }
459 return std::move(*CCMgr);
460 }
461
462 static std::unique_ptr<CODLayerT>
createCODLayer(orc::ExecutionSession & ES,CompileLayerT & CompileLayer,CompileCallbackMgr * CCMgr,IndirectStubsManagerBuilder IndirectStubsMgrBuilder,ResolverMap & Resolvers)463 createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer,
464 CompileCallbackMgr *CCMgr,
465 IndirectStubsManagerBuilder IndirectStubsMgrBuilder,
466 ResolverMap &Resolvers) {
467 // If there is no compile callback manager available we can not create a
468 // compile on demand layer.
469 if (!CCMgr)
470 return nullptr;
471
472 return std::make_unique<CODLayerT>(
473 AcknowledgeORCv1Deprecation, ES, CompileLayer,
474 [&Resolvers](orc::VModuleKey K) {
475 auto ResolverI = Resolvers.find(K);
476 assert(ResolverI != Resolvers.end() && "No resolver for module K");
477 return ResolverI->second;
478 },
479 [&Resolvers](orc::VModuleKey K,
480 std::shared_ptr<orc::SymbolResolver> Resolver) {
481 assert(!Resolvers.count(K) && "Resolver already present");
482 Resolvers[K] = std::move(Resolver);
483 },
484 [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr,
485 std::move(IndirectStubsMgrBuilder), false);
486 }
487
reportError(Error Err)488 void reportError(Error Err) {
489 // FIXME: Report errors on the execution session.
490 logAllUnhandledErrors(std::move(Err), errs(), "ORC error: ");
491 };
492
notifyFinalized(orc::VModuleKey K,const object::ObjectFile & Obj,const RuntimeDyld::LoadedObjectInfo & LoadedObjInfo)493 void notifyFinalized(orc::VModuleKey K,
494 const object::ObjectFile &Obj,
495 const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) {
496 uint64_t Key = static_cast<uint64_t>(
497 reinterpret_cast<uintptr_t>(Obj.getData().data()));
498 for (auto &Listener : EventListeners)
499 Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo);
500 }
501
notifyFreed(orc::VModuleKey K,const object::ObjectFile & Obj)502 void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) {
503 uint64_t Key = static_cast<uint64_t>(
504 reinterpret_cast<uintptr_t>(Obj.getData().data()));
505 for (auto &Listener : EventListeners)
506 Listener->notifyFreeingObject(Key);
507 }
508
509 orc::ExecutionSession ES;
510 std::unique_ptr<CompileCallbackMgr> CCMgr;
511
512 std::vector<JITEventListener *> EventListeners;
513
514 DataLayout DL;
515 SectionMemoryManager CCMgrMemMgr;
516
517 std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr;
518
519 ObjLayerT ObjectLayer;
520 CompileLayerT CompileLayer;
521 std::unique_ptr<CODLayerT> CODLayer;
522
523 std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers;
524
525 orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides;
526 std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners;
527 std::string ErrMsg;
528
529 ResolverMap Resolvers;
530 };
531
532 } // end namespace llvm
533
534 #endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H
535