1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
10
11 #include "llvm/ADT/Optional.h"
12 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
13 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
14 #include "llvm/Support/MemoryBuffer.h"
15
16 #include <vector>
17
18 #define DEBUG_TYPE "orc"
19
20 using namespace llvm;
21 using namespace llvm::jitlink;
22 using namespace llvm::orc;
23
24 namespace {
25
26 class LinkGraphMaterializationUnit : public MaterializationUnit {
27 private:
28 struct LinkGraphInterface {
29 SymbolFlagsMap SymbolFlags;
30 SymbolStringPtr InitSymbol;
31 };
32
33 public:
34 static std::unique_ptr<LinkGraphMaterializationUnit>
Create(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G)35 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) {
36 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G);
37 return std::unique_ptr<LinkGraphMaterializationUnit>(
38 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G),
39 std::move(LGI)));
40 }
41
getName() const42 StringRef getName() const override { return G->getName(); }
materialize(std::unique_ptr<MaterializationResponsibility> MR)43 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override {
44 ObjLinkingLayer.emit(std::move(MR), std::move(G));
45 }
46
47 private:
scanLinkGraph(ExecutionSession & ES,LinkGraph & G)48 static LinkGraphInterface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
49
50 LinkGraphInterface LGI;
51
52 for (auto *Sym : G.defined_symbols()) {
53 // Skip local symbols.
54 if (Sym->getScope() == Scope::Local)
55 continue;
56 assert(Sym->hasName() && "Anonymous non-local symbol?");
57
58 JITSymbolFlags Flags;
59 if (Sym->getScope() == Scope::Default)
60 Flags |= JITSymbolFlags::Exported;
61
62 if (Sym->isCallable())
63 Flags |= JITSymbolFlags::Callable;
64
65 LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags;
66 }
67
68 if (G.getTargetTriple().isOSBinFormatMachO())
69 if (hasMachOInitSection(G))
70 LGI.InitSymbol = makeInitSymbol(ES, G);
71
72 return LGI;
73 }
74
hasMachOInitSection(LinkGraph & G)75 static bool hasMachOInitSection(LinkGraph &G) {
76 for (auto &Sec : G.sections())
77 if (Sec.getName() == "__DATA,__obj_selrefs" ||
78 Sec.getName() == "__DATA,__objc_classlist" ||
79 Sec.getName() == "__TEXT,__swift5_protos" ||
80 Sec.getName() == "__TEXT,__swift5_proto" ||
81 Sec.getName() == "__DATA,__mod_init_func")
82 return true;
83 return false;
84 }
85
makeInitSymbol(ExecutionSession & ES,LinkGraph & G)86 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) {
87 std::string InitSymString;
88 raw_string_ostream(InitSymString)
89 << "$." << G.getName() << ".__inits" << Counter++;
90 return ES.intern(InitSymString);
91 }
92
LinkGraphMaterializationUnit(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G,LinkGraphInterface LGI)93 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer,
94 std::unique_ptr<LinkGraph> G,
95 LinkGraphInterface LGI)
96 : MaterializationUnit(std::move(LGI.SymbolFlags),
97 std::move(LGI.InitSymbol)),
98 ObjLinkingLayer(ObjLinkingLayer), G(std::move(G)) {}
99
discard(const JITDylib & JD,const SymbolStringPtr & Name)100 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
101 for (auto *Sym : G->defined_symbols())
102 if (Sym->getName() == *Name) {
103 assert(Sym->getLinkage() == Linkage::Weak &&
104 "Discarding non-weak definition");
105 G->makeExternal(*Sym);
106 break;
107 }
108 }
109
110 ObjectLinkingLayer &ObjLinkingLayer;
111 std::unique_ptr<LinkGraph> G;
112 static std::atomic<uint64_t> Counter;
113 };
114
115 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0};
116
117 } // end anonymous namespace
118
119 namespace llvm {
120 namespace orc {
121
122 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
123 public:
ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer & Layer,std::unique_ptr<MaterializationResponsibility> MR,std::unique_ptr<MemoryBuffer> ObjBuffer)124 ObjectLinkingLayerJITLinkContext(
125 ObjectLinkingLayer &Layer,
126 std::unique_ptr<MaterializationResponsibility> MR,
127 std::unique_ptr<MemoryBuffer> ObjBuffer)
128 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer),
129 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
130
~ObjectLinkingLayerJITLinkContext()131 ~ObjectLinkingLayerJITLinkContext() {
132 // If there is an object buffer return function then use it to
133 // return ownership of the buffer.
134 if (Layer.ReturnObjectBuffer && ObjBuffer)
135 Layer.ReturnObjectBuffer(std::move(ObjBuffer));
136 }
137
getMemoryManager()138 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
139
notifyMaterializing(LinkGraph & G)140 void notifyMaterializing(LinkGraph &G) {
141 for (auto &P : Layer.Plugins)
142 P->notifyMaterializing(*MR, G, *this,
143 ObjBuffer ? ObjBuffer->getMemBufferRef()
144 : MemoryBufferRef());
145 }
146
notifyFailed(Error Err)147 void notifyFailed(Error Err) override {
148 for (auto &P : Layer.Plugins)
149 Err = joinErrors(std::move(Err), P->notifyFailed(*MR));
150 Layer.getExecutionSession().reportError(std::move(Err));
151 MR->failMaterialization();
152 }
153
lookup(const LookupMap & Symbols,std::unique_ptr<JITLinkAsyncLookupContinuation> LC)154 void lookup(const LookupMap &Symbols,
155 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
156
157 JITDylibSearchOrder LinkOrder;
158 MR->getTargetJITDylib().withLinkOrderDo(
159 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; });
160
161 auto &ES = Layer.getExecutionSession();
162
163 SymbolLookupSet LookupSet;
164 for (auto &KV : Symbols) {
165 orc::SymbolLookupFlags LookupFlags;
166 switch (KV.second) {
167 case jitlink::SymbolLookupFlags::RequiredSymbol:
168 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol;
169 break;
170 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol:
171 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol;
172 break;
173 }
174 LookupSet.add(ES.intern(KV.first), LookupFlags);
175 }
176
177 // OnResolve -- De-intern the symbols and pass the result to the linker.
178 auto OnResolve = [LookupContinuation =
179 std::move(LC)](Expected<SymbolMap> Result) mutable {
180 if (!Result)
181 LookupContinuation->run(Result.takeError());
182 else {
183 AsyncLookupResult LR;
184 for (auto &KV : *Result)
185 LR[*KV.first] = KV.second;
186 LookupContinuation->run(std::move(LR));
187 }
188 };
189
190 for (auto &KV : InternalNamedSymbolDeps) {
191 SymbolDependenceMap InternalDeps;
192 InternalDeps[&MR->getTargetJITDylib()] = std::move(KV.second);
193 MR->addDependencies(KV.first, InternalDeps);
194 }
195
196 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet),
197 SymbolState::Resolved, std::move(OnResolve),
198 [this](const SymbolDependenceMap &Deps) {
199 registerDependencies(Deps);
200 });
201 }
202
notifyResolved(LinkGraph & G)203 Error notifyResolved(LinkGraph &G) override {
204 auto &ES = Layer.getExecutionSession();
205
206 SymbolFlagsMap ExtraSymbolsToClaim;
207 bool AutoClaim = Layer.AutoClaimObjectSymbols;
208
209 SymbolMap InternedResult;
210 for (auto *Sym : G.defined_symbols())
211 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
212 auto InternedName = ES.intern(Sym->getName());
213 JITSymbolFlags Flags;
214
215 if (Sym->isCallable())
216 Flags |= JITSymbolFlags::Callable;
217 if (Sym->getScope() == Scope::Default)
218 Flags |= JITSymbolFlags::Exported;
219
220 InternedResult[InternedName] =
221 JITEvaluatedSymbol(Sym->getAddress(), Flags);
222 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
223 assert(!ExtraSymbolsToClaim.count(InternedName) &&
224 "Duplicate symbol to claim?");
225 ExtraSymbolsToClaim[InternedName] = Flags;
226 }
227 }
228
229 for (auto *Sym : G.absolute_symbols())
230 if (Sym->hasName()) {
231 auto InternedName = ES.intern(Sym->getName());
232 JITSymbolFlags Flags;
233 Flags |= JITSymbolFlags::Absolute;
234 if (Sym->isCallable())
235 Flags |= JITSymbolFlags::Callable;
236 if (Sym->getLinkage() == Linkage::Weak)
237 Flags |= JITSymbolFlags::Weak;
238 InternedResult[InternedName] =
239 JITEvaluatedSymbol(Sym->getAddress(), Flags);
240 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
241 assert(!ExtraSymbolsToClaim.count(InternedName) &&
242 "Duplicate symbol to claim?");
243 ExtraSymbolsToClaim[InternedName] = Flags;
244 }
245 }
246
247 if (!ExtraSymbolsToClaim.empty())
248 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
249 return Err;
250
251 {
252
253 // Check that InternedResult matches up with MR->getSymbols().
254 // This guards against faulty transformations / compilers / object caches.
255
256 // First check that there aren't any missing symbols.
257 size_t NumMaterializationSideEffectsOnlySymbols = 0;
258 SymbolNameVector ExtraSymbols;
259 SymbolNameVector MissingSymbols;
260 for (auto &KV : MR->getSymbols()) {
261
262 // If this is a materialization-side-effects only symbol then bump
263 // the counter and make sure it's *not* defined, otherwise make
264 // sure that it is defined.
265 if (KV.second.hasMaterializationSideEffectsOnly()) {
266 ++NumMaterializationSideEffectsOnlySymbols;
267 if (InternedResult.count(KV.first))
268 ExtraSymbols.push_back(KV.first);
269 continue;
270 } else if (!InternedResult.count(KV.first))
271 MissingSymbols.push_back(KV.first);
272 }
273
274 // If there were missing symbols then report the error.
275 if (!MissingSymbols.empty())
276 return make_error<MissingSymbolDefinitions>(G.getName(),
277 std::move(MissingSymbols));
278
279 // If there are more definitions than expected, add them to the
280 // ExtraSymbols vector.
281 if (InternedResult.size() >
282 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
283 for (auto &KV : InternedResult)
284 if (!MR->getSymbols().count(KV.first))
285 ExtraSymbols.push_back(KV.first);
286 }
287
288 // If there were extra definitions then report the error.
289 if (!ExtraSymbols.empty())
290 return make_error<UnexpectedSymbolDefinitions>(G.getName(),
291 std::move(ExtraSymbols));
292 }
293
294 if (auto Err = MR->notifyResolved(InternedResult))
295 return Err;
296
297 Layer.notifyLoaded(*MR);
298 return Error::success();
299 }
300
notifyFinalized(std::unique_ptr<JITLinkMemoryManager::Allocation> A)301 void notifyFinalized(
302 std::unique_ptr<JITLinkMemoryManager::Allocation> A) override {
303 if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) {
304 Layer.getExecutionSession().reportError(std::move(Err));
305 MR->failMaterialization();
306 return;
307 }
308 if (auto Err = MR->notifyEmitted()) {
309 Layer.getExecutionSession().reportError(std::move(Err));
310 MR->failMaterialization();
311 }
312 }
313
getMarkLivePass(const Triple & TT) const314 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
315 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
316 }
317
modifyPassConfig(LinkGraph & LG,PassConfiguration & Config)318 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override {
319 // Add passes to mark duplicate defs as should-discard, and to walk the
320 // link graph to build the symbol dependence graph.
321 Config.PrePrunePasses.push_back([this](LinkGraph &G) {
322 return claimOrExternalizeWeakAndCommonSymbols(G);
323 });
324
325 Layer.modifyPassConfig(*MR, LG, Config);
326
327 Config.PostPrunePasses.push_back(
328 [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
329
330 return Error::success();
331 }
332
333 private:
334 struct LocalSymbolNamedDependencies {
335 SymbolNameSet Internal, External;
336 };
337
338 using LocalSymbolNamedDependenciesMap =
339 DenseMap<const Symbol *, LocalSymbolNamedDependencies>;
340
claimOrExternalizeWeakAndCommonSymbols(LinkGraph & G)341 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) {
342 auto &ES = Layer.getExecutionSession();
343
344 SymbolFlagsMap NewSymbolsToClaim;
345 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
346
347 auto ProcessSymbol = [&](Symbol *Sym) {
348 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) {
349 auto Name = ES.intern(Sym->getName());
350 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) {
351 JITSymbolFlags SF = JITSymbolFlags::Weak;
352 if (Sym->getScope() == Scope::Default)
353 SF |= JITSymbolFlags::Exported;
354 NewSymbolsToClaim[Name] = SF;
355 NameToSym.push_back(std::make_pair(std::move(Name), Sym));
356 }
357 }
358 };
359
360 for (auto *Sym : G.defined_symbols())
361 ProcessSymbol(Sym);
362 for (auto *Sym : G.absolute_symbols())
363 ProcessSymbol(Sym);
364
365 // Attempt to claim all weak defs that we're not already responsible for.
366 // This cannot fail -- any clashes will just result in rejection of our
367 // claim, at which point we'll externalize that symbol.
368 cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim)));
369
370 for (auto &KV : NameToSym)
371 if (!MR->getSymbols().count(KV.first))
372 G.makeExternal(*KV.second);
373
374 return Error::success();
375 }
376
markResponsibilitySymbolsLive(LinkGraph & G) const377 Error markResponsibilitySymbolsLive(LinkGraph &G) const {
378 auto &ES = Layer.getExecutionSession();
379 for (auto *Sym : G.defined_symbols())
380 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName())))
381 Sym->setLive(true);
382 return Error::success();
383 }
384
computeNamedSymbolDependencies(LinkGraph & G)385 Error computeNamedSymbolDependencies(LinkGraph &G) {
386 auto &ES = MR->getTargetJITDylib().getExecutionSession();
387 auto LocalDeps = computeLocalDeps(G);
388
389 // Compute dependencies for symbols defined in the JITLink graph.
390 for (auto *Sym : G.defined_symbols()) {
391
392 // Skip local symbols: we do not track dependencies for these.
393 if (Sym->getScope() == Scope::Local)
394 continue;
395 assert(Sym->hasName() &&
396 "Defined non-local jitlink::Symbol should have a name");
397
398 SymbolNameSet ExternalSymDeps, InternalSymDeps;
399
400 // Find internal and external named symbol dependencies.
401 for (auto &E : Sym->getBlock().edges()) {
402 auto &TargetSym = E.getTarget();
403
404 if (TargetSym.getScope() != Scope::Local) {
405 if (TargetSym.isExternal())
406 ExternalSymDeps.insert(ES.intern(TargetSym.getName()));
407 else if (&TargetSym != Sym)
408 InternalSymDeps.insert(ES.intern(TargetSym.getName()));
409 } else {
410 assert(TargetSym.isDefined() &&
411 "local symbols must be defined");
412 auto I = LocalDeps.find(&TargetSym);
413 if (I != LocalDeps.end()) {
414 for (auto &S : I->second.External)
415 ExternalSymDeps.insert(S);
416 for (auto &S : I->second.Internal)
417 InternalSymDeps.insert(S);
418 }
419 }
420 }
421
422 if (ExternalSymDeps.empty() && InternalSymDeps.empty())
423 continue;
424
425 auto SymName = ES.intern(Sym->getName());
426 if (!ExternalSymDeps.empty())
427 ExternalNamedSymbolDeps[SymName] = std::move(ExternalSymDeps);
428 if (!InternalSymDeps.empty())
429 InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps);
430 }
431
432 for (auto &P : Layer.Plugins) {
433 auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(*MR);
434 if (SyntheticLocalDeps.empty())
435 continue;
436
437 for (auto &KV : SyntheticLocalDeps) {
438 auto &Name = KV.first;
439 auto &LocalDepsForName = KV.second;
440 for (auto *Local : LocalDepsForName) {
441 assert(Local->getScope() == Scope::Local &&
442 "Dependence on non-local symbol");
443 auto LocalNamedDepsItr = LocalDeps.find(Local);
444 if (LocalNamedDepsItr == LocalDeps.end())
445 continue;
446 for (auto &S : LocalNamedDepsItr->second.Internal)
447 InternalNamedSymbolDeps[Name].insert(S);
448 for (auto &S : LocalNamedDepsItr->second.External)
449 ExternalNamedSymbolDeps[Name].insert(S);
450 }
451 }
452 }
453
454 return Error::success();
455 }
456
computeLocalDeps(LinkGraph & G)457 LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) {
458 DenseMap<jitlink::Symbol *, DenseSet<jitlink::Symbol *>> DepMap;
459
460 // For all local symbols:
461 // (1) Add their named dependencies.
462 // (2) Add them to the worklist for further iteration if they have any
463 // depend on any other local symbols.
464 struct WorklistEntry {
465 WorklistEntry(Symbol *Sym, DenseSet<Symbol *> LocalDeps)
466 : Sym(Sym), LocalDeps(std::move(LocalDeps)) {}
467
468 Symbol *Sym = nullptr;
469 DenseSet<Symbol *> LocalDeps;
470 };
471 std::vector<WorklistEntry> Worklist;
472 for (auto *Sym : G.defined_symbols())
473 if (Sym->getScope() == Scope::Local) {
474 auto &SymNamedDeps = DepMap[Sym];
475 DenseSet<Symbol *> LocalDeps;
476
477 for (auto &E : Sym->getBlock().edges()) {
478 auto &TargetSym = E.getTarget();
479 if (TargetSym.getScope() != Scope::Local)
480 SymNamedDeps.insert(&TargetSym);
481 else {
482 assert(TargetSym.isDefined() &&
483 "local symbols must be defined");
484 LocalDeps.insert(&TargetSym);
485 }
486 }
487
488 if (!LocalDeps.empty())
489 Worklist.push_back(WorklistEntry(Sym, std::move(LocalDeps)));
490 }
491
492 // Loop over all local symbols with local dependencies, propagating
493 // their respective non-local dependencies. Iterate until we hit a stable
494 // state.
495 bool Changed;
496 do {
497 Changed = false;
498 for (auto &WLEntry : Worklist) {
499 auto *Sym = WLEntry.Sym;
500 auto &NamedDeps = DepMap[Sym];
501 auto &LocalDeps = WLEntry.LocalDeps;
502
503 for (auto *TargetSym : LocalDeps) {
504 auto I = DepMap.find(TargetSym);
505 if (I != DepMap.end())
506 for (const auto &S : I->second)
507 Changed |= NamedDeps.insert(S).second;
508 }
509 }
510 } while (Changed);
511
512 // Intern the results to produce a mapping of jitlink::Symbol* to internal
513 // and external symbol names.
514 auto &ES = Layer.getExecutionSession();
515 LocalSymbolNamedDependenciesMap Result;
516 for (auto &KV : DepMap) {
517 auto *Local = KV.first;
518 assert(Local->getScope() == Scope::Local &&
519 "DepMap keys should all be local symbols");
520 auto &LocalNamedDeps = Result[Local];
521 for (auto *Named : KV.second) {
522 assert(Named->getScope() != Scope::Local &&
523 "DepMap values should all be non-local symbol sets");
524 if (Named->isExternal())
525 LocalNamedDeps.External.insert(ES.intern(Named->getName()));
526 else
527 LocalNamedDeps.Internal.insert(ES.intern(Named->getName()));
528 }
529 }
530
531 return Result;
532 }
533
registerDependencies(const SymbolDependenceMap & QueryDeps)534 void registerDependencies(const SymbolDependenceMap &QueryDeps) {
535 for (auto &NamedDepsEntry : ExternalNamedSymbolDeps) {
536 auto &Name = NamedDepsEntry.first;
537 auto &NameDeps = NamedDepsEntry.second;
538 SymbolDependenceMap SymbolDeps;
539
540 for (const auto &QueryDepsEntry : QueryDeps) {
541 JITDylib &SourceJD = *QueryDepsEntry.first;
542 const SymbolNameSet &Symbols = QueryDepsEntry.second;
543 auto &DepsForJD = SymbolDeps[&SourceJD];
544
545 for (const auto &S : Symbols)
546 if (NameDeps.count(S))
547 DepsForJD.insert(S);
548
549 if (DepsForJD.empty())
550 SymbolDeps.erase(&SourceJD);
551 }
552
553 MR->addDependencies(Name, SymbolDeps);
554 }
555 }
556
557 ObjectLinkingLayer &Layer;
558 std::unique_ptr<MaterializationResponsibility> MR;
559 std::unique_ptr<MemoryBuffer> ObjBuffer;
560 DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps;
561 DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps;
562 };
563
~Plugin()564 ObjectLinkingLayer::Plugin::~Plugin() {}
565
566 char ObjectLinkingLayer::ID;
567
568 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>;
569
ObjectLinkingLayer(ExecutionSession & ES,JITLinkMemoryManager & MemMgr)570 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
571 JITLinkMemoryManager &MemMgr)
572 : BaseT(ES), MemMgr(MemMgr) {
573 ES.registerResourceManager(*this);
574 }
575
ObjectLinkingLayer(ExecutionSession & ES,std::unique_ptr<JITLinkMemoryManager> MemMgr)576 ObjectLinkingLayer::ObjectLinkingLayer(
577 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr)
578 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) {
579 ES.registerResourceManager(*this);
580 }
581
~ObjectLinkingLayer()582 ObjectLinkingLayer::~ObjectLinkingLayer() {
583 assert(Allocs.empty() && "Layer destroyed with resources still attached");
584 getExecutionSession().deregisterResourceManager(*this);
585 }
586
add(ResourceTrackerSP RT,std::unique_ptr<LinkGraph> G)587 Error ObjectLinkingLayer::add(ResourceTrackerSP RT,
588 std::unique_ptr<LinkGraph> G) {
589 auto &JD = RT->getJITDylib();
590 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)),
591 std::move(RT));
592 }
593
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<MemoryBuffer> O)594 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
595 std::unique_ptr<MemoryBuffer> O) {
596 assert(O && "Object must not be null");
597 MemoryBufferRef ObjBuffer = O->getMemBufferRef();
598
599 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
600 *this, std::move(R), std::move(O));
601 if (auto G = createLinkGraphFromObject(ObjBuffer)) {
602 Ctx->notifyMaterializing(**G);
603 link(std::move(*G), std::move(Ctx));
604 } else {
605 Ctx->notifyFailed(G.takeError());
606 }
607 }
608
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<LinkGraph> G)609 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
610 std::unique_ptr<LinkGraph> G) {
611 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
612 *this, std::move(R), nullptr);
613 Ctx->notifyMaterializing(*G);
614 link(std::move(G), std::move(Ctx));
615 }
616
modifyPassConfig(MaterializationResponsibility & MR,LinkGraph & G,PassConfiguration & PassConfig)617 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR,
618 LinkGraph &G,
619 PassConfiguration &PassConfig) {
620 for (auto &P : Plugins)
621 P->modifyPassConfig(MR, G, PassConfig);
622 }
623
notifyLoaded(MaterializationResponsibility & MR)624 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
625 for (auto &P : Plugins)
626 P->notifyLoaded(MR);
627 }
628
notifyEmitted(MaterializationResponsibility & MR,AllocPtr Alloc)629 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
630 AllocPtr Alloc) {
631 Error Err = Error::success();
632 for (auto &P : Plugins)
633 Err = joinErrors(std::move(Err), P->notifyEmitted(MR));
634
635 if (Err)
636 return Err;
637
638 return MR.withResourceKeyDo(
639 [&](ResourceKey K) { Allocs[K].push_back(std::move(Alloc)); });
640 }
641
handleRemoveResources(ResourceKey K)642 Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) {
643
644 Error Err = Error::success();
645
646 for (auto &P : Plugins)
647 Err = joinErrors(std::move(Err), P->notifyRemovingResources(K));
648
649 std::vector<AllocPtr> AllocsToRemove;
650 getExecutionSession().runSessionLocked([&] {
651 auto I = Allocs.find(K);
652 if (I != Allocs.end()) {
653 std::swap(AllocsToRemove, I->second);
654 Allocs.erase(I);
655 }
656 });
657
658 while (!AllocsToRemove.empty()) {
659 Err = joinErrors(std::move(Err), AllocsToRemove.back()->deallocate());
660 AllocsToRemove.pop_back();
661 }
662
663 return Err;
664 }
665
handleTransferResources(ResourceKey DstKey,ResourceKey SrcKey)666 void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey,
667 ResourceKey SrcKey) {
668 auto I = Allocs.find(SrcKey);
669 if (I != Allocs.end()) {
670 auto &SrcAllocs = I->second;
671 auto &DstAllocs = Allocs[DstKey];
672 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
673 for (auto &Alloc : SrcAllocs)
674 DstAllocs.push_back(std::move(Alloc));
675
676 // Erase SrcKey entry using value rather than iterator I: I may have been
677 // invalidated when we looked up DstKey.
678 Allocs.erase(SrcKey);
679 }
680
681 for (auto &P : Plugins)
682 P->notifyTransferringResources(DstKey, SrcKey);
683 }
684
EHFrameRegistrationPlugin(ExecutionSession & ES,std::unique_ptr<EHFrameRegistrar> Registrar)685 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
686 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar)
687 : ES(ES), Registrar(std::move(Registrar)) {}
688
modifyPassConfig(MaterializationResponsibility & MR,LinkGraph & G,PassConfiguration & PassConfig)689 void EHFrameRegistrationPlugin::modifyPassConfig(
690 MaterializationResponsibility &MR, LinkGraph &G,
691 PassConfiguration &PassConfig) {
692
693 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass(
694 G.getTargetTriple(), [this, &MR](JITTargetAddress Addr, size_t Size) {
695 if (Addr) {
696 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
697 assert(!InProcessLinks.count(&MR) &&
698 "Link for MR already being tracked?");
699 InProcessLinks[&MR] = {Addr, Size};
700 }
701 }));
702 }
703
notifyEmitted(MaterializationResponsibility & MR)704 Error EHFrameRegistrationPlugin::notifyEmitted(
705 MaterializationResponsibility &MR) {
706
707 EHFrameRange EmittedRange;
708 {
709 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
710
711 auto EHFrameRangeItr = InProcessLinks.find(&MR);
712 if (EHFrameRangeItr == InProcessLinks.end())
713 return Error::success();
714
715 EmittedRange = EHFrameRangeItr->second;
716 assert(EmittedRange.Addr && "eh-frame addr to register can not be null");
717 InProcessLinks.erase(EHFrameRangeItr);
718 }
719
720 if (auto Err = MR.withResourceKeyDo(
721 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
722 return Err;
723
724 return Registrar->registerEHFrames(EmittedRange.Addr, EmittedRange.Size);
725 }
726
notifyFailed(MaterializationResponsibility & MR)727 Error EHFrameRegistrationPlugin::notifyFailed(
728 MaterializationResponsibility &MR) {
729 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
730 InProcessLinks.erase(&MR);
731 return Error::success();
732 }
733
notifyRemovingResources(ResourceKey K)734 Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) {
735 std::vector<EHFrameRange> RangesToRemove;
736
737 ES.runSessionLocked([&] {
738 auto I = EHFrameRanges.find(K);
739 if (I != EHFrameRanges.end()) {
740 RangesToRemove = std::move(I->second);
741 EHFrameRanges.erase(I);
742 }
743 });
744
745 Error Err = Error::success();
746 while (!RangesToRemove.empty()) {
747 auto RangeToRemove = RangesToRemove.back();
748 RangesToRemove.pop_back();
749 assert(RangeToRemove.Addr && "Untracked eh-frame range must not be null");
750 Err = joinErrors(
751 std::move(Err),
752 Registrar->deregisterEHFrames(RangeToRemove.Addr, RangeToRemove.Size));
753 }
754
755 return Err;
756 }
757
notifyTransferringResources(ResourceKey DstKey,ResourceKey SrcKey)758 void EHFrameRegistrationPlugin::notifyTransferringResources(
759 ResourceKey DstKey, ResourceKey SrcKey) {
760 auto SI = EHFrameRanges.find(SrcKey);
761 if (SI == EHFrameRanges.end())
762 return;
763
764 auto DI = EHFrameRanges.find(DstKey);
765 if (DI != EHFrameRanges.end()) {
766 auto &SrcRanges = SI->second;
767 auto &DstRanges = DI->second;
768 DstRanges.reserve(DstRanges.size() + SrcRanges.size());
769 for (auto &SrcRange : SrcRanges)
770 DstRanges.push_back(std::move(SrcRange));
771 EHFrameRanges.erase(SI);
772 } else {
773 // We need to move SrcKey's ranges over without invalidating the SI
774 // iterator.
775 auto Tmp = std::move(SI->second);
776 EHFrameRanges.erase(SI);
777 EHFrameRanges[DstKey] = std::move(Tmp);
778 }
779 }
780
781 } // End namespace orc.
782 } // End namespace llvm.
783