1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/aarch32.h"
12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
13 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
14 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
15 #include "llvm/Support/MemoryBuffer.h"
16 #include <string>
17 #include <vector>
18
19 #define DEBUG_TYPE "orc"
20
21 using namespace llvm;
22 using namespace llvm::jitlink;
23 using namespace llvm::orc;
24
25 namespace {
26
hasInitializerSection(jitlink::LinkGraph & G)27 bool hasInitializerSection(jitlink::LinkGraph &G) {
28 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO();
29 bool IsElf = G.getTargetTriple().isOSBinFormatELF();
30 if (!IsMachO && !IsElf)
31 return false;
32
33 for (auto &Sec : G.sections()) {
34 if (IsMachO && isMachOInitializerSection(Sec.getName()))
35 return true;
36 if (IsElf && isELFInitializerSection(Sec.getName()))
37 return true;
38 }
39
40 return false;
41 }
42
getJITSymbolPtrForSymbol(Symbol & Sym,const Triple & TT)43 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) {
44 switch (TT.getArch()) {
45 case Triple::arm:
46 case Triple::armeb:
47 case Triple::thumb:
48 case Triple::thumbeb:
49 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) {
50 // Set LSB to indicate thumb target
51 assert(Sym.isCallable() && "Only callable symbols can have thumb flag");
52 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear");
53 return Sym.getAddress() + 0x01;
54 }
55 return Sym.getAddress();
56 default:
57 return Sym.getAddress();
58 }
59 }
60
getJITSymbolFlagsForSymbol(Symbol & Sym)61 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) {
62 JITSymbolFlags Flags;
63
64 if (Sym.getLinkage() == Linkage::Weak)
65 Flags |= JITSymbolFlags::Weak;
66
67 if (Sym.getScope() == Scope::Default)
68 Flags |= JITSymbolFlags::Exported;
69
70 if (Sym.isCallable())
71 Flags |= JITSymbolFlags::Callable;
72
73 return Flags;
74 }
75
76 class LinkGraphMaterializationUnit : public MaterializationUnit {
77 public:
78 static std::unique_ptr<LinkGraphMaterializationUnit>
Create(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G)79 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) {
80 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G);
81 return std::unique_ptr<LinkGraphMaterializationUnit>(
82 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G),
83 std::move(LGI)));
84 }
85
getName() const86 StringRef getName() const override { return G->getName(); }
materialize(std::unique_ptr<MaterializationResponsibility> MR)87 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override {
88 ObjLinkingLayer.emit(std::move(MR), std::move(G));
89 }
90
91 private:
scanLinkGraph(ExecutionSession & ES,LinkGraph & G)92 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
93
94 Interface LGI;
95
96 auto AddSymbol = [&](Symbol *Sym) {
97 // Skip local symbols.
98 if (Sym->getScope() == Scope::Local)
99 return;
100 assert(Sym->hasName() && "Anonymous non-local symbol?");
101
102 LGI.SymbolFlags[ES.intern(Sym->getName())] =
103 getJITSymbolFlagsForSymbol(*Sym);
104 };
105
106 for (auto *Sym : G.defined_symbols())
107 AddSymbol(Sym);
108 for (auto *Sym : G.absolute_symbols())
109 AddSymbol(Sym);
110
111 if (hasInitializerSection(G))
112 LGI.InitSymbol = makeInitSymbol(ES, G);
113
114 return LGI;
115 }
116
makeInitSymbol(ExecutionSession & ES,LinkGraph & G)117 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) {
118 std::string InitSymString;
119 raw_string_ostream(InitSymString)
120 << "$." << G.getName() << ".__inits" << Counter++;
121 return ES.intern(InitSymString);
122 }
123
LinkGraphMaterializationUnit(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G,Interface LGI)124 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer,
125 std::unique_ptr<LinkGraph> G, Interface LGI)
126 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer),
127 G(std::move(G)) {}
128
discard(const JITDylib & JD,const SymbolStringPtr & Name)129 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
130 for (auto *Sym : G->defined_symbols())
131 if (Sym->getName() == *Name) {
132 assert(Sym->getLinkage() == Linkage::Weak &&
133 "Discarding non-weak definition");
134 G->makeExternal(*Sym);
135 break;
136 }
137 }
138
139 ObjectLinkingLayer &ObjLinkingLayer;
140 std::unique_ptr<LinkGraph> G;
141 static std::atomic<uint64_t> Counter;
142 };
143
144 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0};
145
146 } // end anonymous namespace
147
148 namespace llvm {
149 namespace orc {
150
151 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
152 public:
ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer & Layer,std::unique_ptr<MaterializationResponsibility> MR,std::unique_ptr<MemoryBuffer> ObjBuffer)153 ObjectLinkingLayerJITLinkContext(
154 ObjectLinkingLayer &Layer,
155 std::unique_ptr<MaterializationResponsibility> MR,
156 std::unique_ptr<MemoryBuffer> ObjBuffer)
157 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer),
158 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
159
~ObjectLinkingLayerJITLinkContext()160 ~ObjectLinkingLayerJITLinkContext() {
161 // If there is an object buffer return function then use it to
162 // return ownership of the buffer.
163 if (Layer.ReturnObjectBuffer && ObjBuffer)
164 Layer.ReturnObjectBuffer(std::move(ObjBuffer));
165 }
166
getMemoryManager()167 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
168
notifyMaterializing(LinkGraph & G)169 void notifyMaterializing(LinkGraph &G) {
170 for (auto &P : Layer.Plugins)
171 P->notifyMaterializing(*MR, G, *this,
172 ObjBuffer ? ObjBuffer->getMemBufferRef()
173 : MemoryBufferRef());
174 }
175
notifyFailed(Error Err)176 void notifyFailed(Error Err) override {
177 for (auto &P : Layer.Plugins)
178 Err = joinErrors(std::move(Err), P->notifyFailed(*MR));
179 Layer.getExecutionSession().reportError(std::move(Err));
180 MR->failMaterialization();
181 }
182
lookup(const LookupMap & Symbols,std::unique_ptr<JITLinkAsyncLookupContinuation> LC)183 void lookup(const LookupMap &Symbols,
184 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
185
186 JITDylibSearchOrder LinkOrder;
187 MR->getTargetJITDylib().withLinkOrderDo(
188 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; });
189
190 auto &ES = Layer.getExecutionSession();
191
192 SymbolLookupSet LookupSet;
193 for (auto &KV : Symbols) {
194 orc::SymbolLookupFlags LookupFlags;
195 switch (KV.second) {
196 case jitlink::SymbolLookupFlags::RequiredSymbol:
197 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol;
198 break;
199 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol:
200 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol;
201 break;
202 }
203 LookupSet.add(ES.intern(KV.first), LookupFlags);
204 }
205
206 // OnResolve -- De-intern the symbols and pass the result to the linker.
207 auto OnResolve = [LookupContinuation =
208 std::move(LC)](Expected<SymbolMap> Result) mutable {
209 if (!Result)
210 LookupContinuation->run(Result.takeError());
211 else {
212 AsyncLookupResult LR;
213 for (auto &KV : *Result)
214 LR[*KV.first] = KV.second;
215 LookupContinuation->run(std::move(LR));
216 }
217 };
218
219 for (auto &KV : InternalNamedSymbolDeps) {
220 SymbolDependenceMap InternalDeps;
221 InternalDeps[&MR->getTargetJITDylib()] = std::move(KV.second);
222 MR->addDependencies(KV.first, InternalDeps);
223 }
224
225 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet),
226 SymbolState::Resolved, std::move(OnResolve),
227 [this](const SymbolDependenceMap &Deps) {
228 registerDependencies(Deps);
229 });
230 }
231
notifyResolved(LinkGraph & G)232 Error notifyResolved(LinkGraph &G) override {
233 auto &ES = Layer.getExecutionSession();
234
235 SymbolFlagsMap ExtraSymbolsToClaim;
236 bool AutoClaim = Layer.AutoClaimObjectSymbols;
237
238 SymbolMap InternedResult;
239 for (auto *Sym : G.defined_symbols())
240 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
241 auto InternedName = ES.intern(Sym->getName());
242 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
243 auto Flags = getJITSymbolFlagsForSymbol(*Sym);
244 InternedResult[InternedName] = {Ptr, Flags};
245 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
246 assert(!ExtraSymbolsToClaim.count(InternedName) &&
247 "Duplicate symbol to claim?");
248 ExtraSymbolsToClaim[InternedName] = Flags;
249 }
250 }
251
252 for (auto *Sym : G.absolute_symbols())
253 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
254 auto InternedName = ES.intern(Sym->getName());
255 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
256 auto Flags = getJITSymbolFlagsForSymbol(*Sym);
257 InternedResult[InternedName] = {Ptr, Flags};
258 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
259 assert(!ExtraSymbolsToClaim.count(InternedName) &&
260 "Duplicate symbol to claim?");
261 ExtraSymbolsToClaim[InternedName] = Flags;
262 }
263 }
264
265 if (!ExtraSymbolsToClaim.empty())
266 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
267 return Err;
268
269 {
270
271 // Check that InternedResult matches up with MR->getSymbols(), overriding
272 // flags if requested.
273 // This guards against faulty transformations / compilers / object caches.
274
275 // First check that there aren't any missing symbols.
276 size_t NumMaterializationSideEffectsOnlySymbols = 0;
277 SymbolNameVector ExtraSymbols;
278 SymbolNameVector MissingSymbols;
279 for (auto &KV : MR->getSymbols()) {
280
281 auto I = InternedResult.find(KV.first);
282
283 // If this is a materialization-side-effects only symbol then bump
284 // the counter and make sure it's *not* defined, otherwise make
285 // sure that it is defined.
286 if (KV.second.hasMaterializationSideEffectsOnly()) {
287 ++NumMaterializationSideEffectsOnlySymbols;
288 if (I != InternedResult.end())
289 ExtraSymbols.push_back(KV.first);
290 continue;
291 } else if (I == InternedResult.end())
292 MissingSymbols.push_back(KV.first);
293 else if (Layer.OverrideObjectFlags)
294 I->second.setFlags(KV.second);
295 }
296
297 // If there were missing symbols then report the error.
298 if (!MissingSymbols.empty())
299 return make_error<MissingSymbolDefinitions>(
300 Layer.getExecutionSession().getSymbolStringPool(), G.getName(),
301 std::move(MissingSymbols));
302
303 // If there are more definitions than expected, add them to the
304 // ExtraSymbols vector.
305 if (InternedResult.size() >
306 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
307 for (auto &KV : InternedResult)
308 if (!MR->getSymbols().count(KV.first))
309 ExtraSymbols.push_back(KV.first);
310 }
311
312 // If there were extra definitions then report the error.
313 if (!ExtraSymbols.empty())
314 return make_error<UnexpectedSymbolDefinitions>(
315 Layer.getExecutionSession().getSymbolStringPool(), G.getName(),
316 std::move(ExtraSymbols));
317 }
318
319 if (auto Err = MR->notifyResolved(InternedResult))
320 return Err;
321
322 Layer.notifyLoaded(*MR);
323 return Error::success();
324 }
325
notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A)326 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override {
327 if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) {
328 Layer.getExecutionSession().reportError(std::move(Err));
329 MR->failMaterialization();
330 return;
331 }
332 if (auto Err = MR->notifyEmitted()) {
333 Layer.getExecutionSession().reportError(std::move(Err));
334 MR->failMaterialization();
335 }
336 }
337
getMarkLivePass(const Triple & TT) const338 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
339 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
340 }
341
modifyPassConfig(LinkGraph & LG,PassConfiguration & Config)342 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override {
343 // Add passes to mark duplicate defs as should-discard, and to walk the
344 // link graph to build the symbol dependence graph.
345 Config.PrePrunePasses.push_back([this](LinkGraph &G) {
346 return claimOrExternalizeWeakAndCommonSymbols(G);
347 });
348
349 Layer.modifyPassConfig(*MR, LG, Config);
350
351 Config.PostPrunePasses.push_back(
352 [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
353
354 return Error::success();
355 }
356
357 private:
358 // Symbol name dependencies:
359 // Internal: Defined in this graph.
360 // External: Defined externally.
361 struct BlockSymbolDependencies {
362 SymbolNameSet Internal, External;
363 };
364
365 // Lazily populated map of blocks to BlockSymbolDependencies values.
366 class BlockDependenciesMap {
367 public:
BlockDependenciesMap(ExecutionSession & ES,DenseMap<const Block *,DenseSet<Block * >> BlockDeps)368 BlockDependenciesMap(ExecutionSession &ES,
369 DenseMap<const Block *, DenseSet<Block *>> BlockDeps)
370 : ES(ES), BlockDeps(std::move(BlockDeps)) {}
371
operator [](const Block & B)372 const BlockSymbolDependencies &operator[](const Block &B) {
373 // Check the cache first.
374 auto I = BlockTransitiveDepsCache.find(&B);
375 if (I != BlockTransitiveDepsCache.end())
376 return I->second;
377
378 // No value. Populate the cache.
379 BlockSymbolDependencies BTDCacheVal;
380 auto BDI = BlockDeps.find(&B);
381 assert(BDI != BlockDeps.end() && "No block dependencies");
382
383 for (auto *BDep : BDI->second) {
384 auto &BID = getBlockImmediateDeps(*BDep);
385 for (auto &ExternalDep : BID.External)
386 BTDCacheVal.External.insert(ExternalDep);
387 for (auto &InternalDep : BID.Internal)
388 BTDCacheVal.Internal.insert(InternalDep);
389 }
390
391 return BlockTransitiveDepsCache
392 .insert(std::make_pair(&B, std::move(BTDCacheVal)))
393 .first->second;
394 }
395
getInternedName(Symbol & Sym)396 SymbolStringPtr &getInternedName(Symbol &Sym) {
397 auto I = NameCache.find(&Sym);
398 if (I != NameCache.end())
399 return I->second;
400
401 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName())))
402 .first->second;
403 }
404
405 private:
getBlockImmediateDeps(Block & B)406 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) {
407 // Check the cache first.
408 auto I = BlockImmediateDepsCache.find(&B);
409 if (I != BlockImmediateDepsCache.end())
410 return I->second;
411
412 BlockSymbolDependencies BIDCacheVal;
413 for (auto &E : B.edges()) {
414 auto &Tgt = E.getTarget();
415 if (Tgt.getScope() != Scope::Local) {
416 if (Tgt.isExternal())
417 BIDCacheVal.External.insert(getInternedName(Tgt));
418 else
419 BIDCacheVal.Internal.insert(getInternedName(Tgt));
420 }
421 }
422
423 return BlockImmediateDepsCache
424 .insert(std::make_pair(&B, std::move(BIDCacheVal)))
425 .first->second;
426 }
427
428 ExecutionSession &ES;
429 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
430 DenseMap<const Symbol *, SymbolStringPtr> NameCache;
431 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache;
432 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache;
433 };
434
claimOrExternalizeWeakAndCommonSymbols(LinkGraph & G)435 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) {
436 auto &ES = Layer.getExecutionSession();
437
438 SymbolFlagsMap NewSymbolsToClaim;
439 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
440
441 auto ProcessSymbol = [&](Symbol *Sym) {
442 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak &&
443 Sym->getScope() != Scope::Local) {
444 auto Name = ES.intern(Sym->getName());
445 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) {
446 NewSymbolsToClaim[Name] =
447 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak;
448 NameToSym.push_back(std::make_pair(std::move(Name), Sym));
449 }
450 }
451 };
452
453 for (auto *Sym : G.defined_symbols())
454 ProcessSymbol(Sym);
455 for (auto *Sym : G.absolute_symbols())
456 ProcessSymbol(Sym);
457
458 // Attempt to claim all weak defs that we're not already responsible for.
459 // This cannot fail -- any clashes will just result in rejection of our
460 // claim, at which point we'll externalize that symbol.
461 cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim)));
462
463 // Walk the list of symbols that we just tried to claim. Symbols that we're
464 // responsible for are marked live. Symbols that we're not responsible for
465 // are turned into external references.
466 for (auto &KV : NameToSym) {
467 if (MR->getSymbols().count(KV.first))
468 KV.second->setLive(true);
469 else
470 G.makeExternal(*KV.second);
471 }
472
473 return Error::success();
474 }
475
markResponsibilitySymbolsLive(LinkGraph & G) const476 Error markResponsibilitySymbolsLive(LinkGraph &G) const {
477 auto &ES = Layer.getExecutionSession();
478 for (auto *Sym : G.defined_symbols())
479 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName())))
480 Sym->setLive(true);
481 return Error::success();
482 }
483
computeNamedSymbolDependencies(LinkGraph & G)484 Error computeNamedSymbolDependencies(LinkGraph &G) {
485 auto &ES = MR->getTargetJITDylib().getExecutionSession();
486 auto BlockDeps = computeBlockNonLocalDeps(G);
487
488 // Compute dependencies for symbols defined in the JITLink graph.
489 for (auto *Sym : G.defined_symbols()) {
490
491 // Skip local symbols: we do not track dependencies for these.
492 if (Sym->getScope() == Scope::Local)
493 continue;
494 assert(Sym->hasName() &&
495 "Defined non-local jitlink::Symbol should have a name");
496
497 auto &SymDeps = BlockDeps[Sym->getBlock()];
498 if (SymDeps.External.empty() && SymDeps.Internal.empty())
499 continue;
500
501 auto SymName = ES.intern(Sym->getName());
502 if (!SymDeps.External.empty())
503 ExternalNamedSymbolDeps[SymName] = SymDeps.External;
504 if (!SymDeps.Internal.empty())
505 InternalNamedSymbolDeps[SymName] = SymDeps.Internal;
506 }
507
508 for (auto &P : Layer.Plugins) {
509 auto SynthDeps = P->getSyntheticSymbolDependencies(*MR);
510 if (SynthDeps.empty())
511 continue;
512
513 DenseSet<Block *> BlockVisited;
514 for (auto &KV : SynthDeps) {
515 auto &Name = KV.first;
516 auto &DepsForName = KV.second;
517 for (auto *Sym : DepsForName) {
518 if (Sym->getScope() == Scope::Local) {
519 auto &BDeps = BlockDeps[Sym->getBlock()];
520 for (auto &S : BDeps.Internal)
521 InternalNamedSymbolDeps[Name].insert(S);
522 for (auto &S : BDeps.External)
523 ExternalNamedSymbolDeps[Name].insert(S);
524 } else {
525 if (Sym->isExternal())
526 ExternalNamedSymbolDeps[Name].insert(
527 BlockDeps.getInternedName(*Sym));
528 else
529 InternalNamedSymbolDeps[Name].insert(
530 BlockDeps.getInternedName(*Sym));
531 }
532 }
533 }
534 }
535
536 return Error::success();
537 }
538
computeBlockNonLocalDeps(LinkGraph & G)539 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) {
540 // First calculate the reachable-via-non-local-symbol blocks for each block.
541 struct BlockInfo {
542 DenseSet<Block *> Dependencies;
543 DenseSet<Block *> Dependants;
544 bool DependenciesChanged = true;
545 };
546 DenseMap<Block *, BlockInfo> BlockInfos;
547 SmallVector<Block *> WorkList;
548
549 // Pre-allocate map entries. This prevents any iterator/reference
550 // invalidation in the next loop.
551 for (auto *B : G.blocks())
552 (void)BlockInfos[B];
553
554 // Build initial worklist, record block dependencies/dependants and
555 // non-local symbol dependencies.
556 for (auto *B : G.blocks()) {
557 auto &BI = BlockInfos[B];
558 for (auto &E : B->edges()) {
559 if (E.getTarget().getScope() == Scope::Local &&
560 !E.getTarget().isAbsolute()) {
561 auto &TgtB = E.getTarget().getBlock();
562 if (&TgtB != B) {
563 BI.Dependencies.insert(&TgtB);
564 BlockInfos[&TgtB].Dependants.insert(B);
565 }
566 }
567 }
568
569 // If this node has both dependants and dependencies then add it to the
570 // worklist to propagate the dependencies to the dependants.
571 if (!BI.Dependants.empty() && !BI.Dependencies.empty())
572 WorkList.push_back(B);
573 }
574
575 // Propagate block-level dependencies through the block-dependence graph.
576 while (!WorkList.empty()) {
577 auto *B = WorkList.pop_back_val();
578
579 auto &BI = BlockInfos[B];
580 assert(BI.DependenciesChanged &&
581 "Block in worklist has unchanged dependencies");
582 BI.DependenciesChanged = false;
583 for (auto *Dependant : BI.Dependants) {
584 auto &DependantBI = BlockInfos[Dependant];
585 for (auto *Dependency : BI.Dependencies) {
586 if (Dependant != Dependency &&
587 DependantBI.Dependencies.insert(Dependency).second)
588 if (!DependantBI.DependenciesChanged) {
589 DependantBI.DependenciesChanged = true;
590 WorkList.push_back(Dependant);
591 }
592 }
593 }
594 }
595
596 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
597 for (auto &KV : BlockInfos)
598 BlockDeps[KV.first] = std::move(KV.second.Dependencies);
599
600 return BlockDependenciesMap(Layer.getExecutionSession(),
601 std::move(BlockDeps));
602 }
603
registerDependencies(const SymbolDependenceMap & QueryDeps)604 void registerDependencies(const SymbolDependenceMap &QueryDeps) {
605 for (auto &NamedDepsEntry : ExternalNamedSymbolDeps) {
606 auto &Name = NamedDepsEntry.first;
607 auto &NameDeps = NamedDepsEntry.second;
608 SymbolDependenceMap SymbolDeps;
609
610 for (const auto &QueryDepsEntry : QueryDeps) {
611 JITDylib &SourceJD = *QueryDepsEntry.first;
612 const SymbolNameSet &Symbols = QueryDepsEntry.second;
613 auto &DepsForJD = SymbolDeps[&SourceJD];
614
615 for (const auto &S : Symbols)
616 if (NameDeps.count(S))
617 DepsForJD.insert(S);
618
619 if (DepsForJD.empty())
620 SymbolDeps.erase(&SourceJD);
621 }
622
623 MR->addDependencies(Name, SymbolDeps);
624 }
625 }
626
627 ObjectLinkingLayer &Layer;
628 std::unique_ptr<MaterializationResponsibility> MR;
629 std::unique_ptr<MemoryBuffer> ObjBuffer;
630 DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps;
631 DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps;
632 };
633
634 ObjectLinkingLayer::Plugin::~Plugin() = default;
635
636 char ObjectLinkingLayer::ID;
637
638 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>;
639
ObjectLinkingLayer(ExecutionSession & ES)640 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES)
641 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) {
642 ES.registerResourceManager(*this);
643 }
644
ObjectLinkingLayer(ExecutionSession & ES,JITLinkMemoryManager & MemMgr)645 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
646 JITLinkMemoryManager &MemMgr)
647 : BaseT(ES), MemMgr(MemMgr) {
648 ES.registerResourceManager(*this);
649 }
650
ObjectLinkingLayer(ExecutionSession & ES,std::unique_ptr<JITLinkMemoryManager> MemMgr)651 ObjectLinkingLayer::ObjectLinkingLayer(
652 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr)
653 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) {
654 ES.registerResourceManager(*this);
655 }
656
~ObjectLinkingLayer()657 ObjectLinkingLayer::~ObjectLinkingLayer() {
658 assert(Allocs.empty() && "Layer destroyed with resources still attached");
659 getExecutionSession().deregisterResourceManager(*this);
660 }
661
add(ResourceTrackerSP RT,std::unique_ptr<LinkGraph> G)662 Error ObjectLinkingLayer::add(ResourceTrackerSP RT,
663 std::unique_ptr<LinkGraph> G) {
664 auto &JD = RT->getJITDylib();
665 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)),
666 std::move(RT));
667 }
668
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<MemoryBuffer> O)669 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
670 std::unique_ptr<MemoryBuffer> O) {
671 assert(O && "Object must not be null");
672 MemoryBufferRef ObjBuffer = O->getMemBufferRef();
673
674 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
675 *this, std::move(R), std::move(O));
676 if (auto G = createLinkGraphFromObject(ObjBuffer)) {
677 Ctx->notifyMaterializing(**G);
678 link(std::move(*G), std::move(Ctx));
679 } else {
680 Ctx->notifyFailed(G.takeError());
681 }
682 }
683
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<LinkGraph> G)684 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
685 std::unique_ptr<LinkGraph> G) {
686 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
687 *this, std::move(R), nullptr);
688 Ctx->notifyMaterializing(*G);
689 link(std::move(G), std::move(Ctx));
690 }
691
modifyPassConfig(MaterializationResponsibility & MR,LinkGraph & G,PassConfiguration & PassConfig)692 void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR,
693 LinkGraph &G,
694 PassConfiguration &PassConfig) {
695 for (auto &P : Plugins)
696 P->modifyPassConfig(MR, G, PassConfig);
697 }
698
notifyLoaded(MaterializationResponsibility & MR)699 void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) {
700 for (auto &P : Plugins)
701 P->notifyLoaded(MR);
702 }
703
notifyEmitted(MaterializationResponsibility & MR,FinalizedAlloc FA)704 Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR,
705 FinalizedAlloc FA) {
706 Error Err = Error::success();
707 for (auto &P : Plugins)
708 Err = joinErrors(std::move(Err), P->notifyEmitted(MR));
709
710 if (Err)
711 return Err;
712
713 if (!FA)
714 return Error::success();
715
716 return MR.withResourceKeyDo(
717 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); });
718 }
719
handleRemoveResources(JITDylib & JD,ResourceKey K)720 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
721
722 {
723 Error Err = Error::success();
724 for (auto &P : Plugins)
725 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K));
726 if (Err)
727 return Err;
728 }
729
730 std::vector<FinalizedAlloc> AllocsToRemove;
731 getExecutionSession().runSessionLocked([&] {
732 auto I = Allocs.find(K);
733 if (I != Allocs.end()) {
734 std::swap(AllocsToRemove, I->second);
735 Allocs.erase(I);
736 }
737 });
738
739 if (AllocsToRemove.empty())
740 return Error::success();
741
742 return MemMgr.deallocate(std::move(AllocsToRemove));
743 }
744
handleTransferResources(JITDylib & JD,ResourceKey DstKey,ResourceKey SrcKey)745 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD,
746 ResourceKey DstKey,
747 ResourceKey SrcKey) {
748 auto I = Allocs.find(SrcKey);
749 if (I != Allocs.end()) {
750 auto &SrcAllocs = I->second;
751 auto &DstAllocs = Allocs[DstKey];
752 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
753 for (auto &Alloc : SrcAllocs)
754 DstAllocs.push_back(std::move(Alloc));
755
756 // Erase SrcKey entry using value rather than iterator I: I may have been
757 // invalidated when we looked up DstKey.
758 Allocs.erase(SrcKey);
759 }
760
761 for (auto &P : Plugins)
762 P->notifyTransferringResources(JD, DstKey, SrcKey);
763 }
764
EHFrameRegistrationPlugin(ExecutionSession & ES,std::unique_ptr<EHFrameRegistrar> Registrar)765 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
766 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar)
767 : ES(ES), Registrar(std::move(Registrar)) {}
768
modifyPassConfig(MaterializationResponsibility & MR,LinkGraph & G,PassConfiguration & PassConfig)769 void EHFrameRegistrationPlugin::modifyPassConfig(
770 MaterializationResponsibility &MR, LinkGraph &G,
771 PassConfiguration &PassConfig) {
772
773 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass(
774 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) {
775 if (Addr) {
776 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
777 assert(!InProcessLinks.count(&MR) &&
778 "Link for MR already being tracked?");
779 InProcessLinks[&MR] = {Addr, Size};
780 }
781 }));
782 }
783
notifyEmitted(MaterializationResponsibility & MR)784 Error EHFrameRegistrationPlugin::notifyEmitted(
785 MaterializationResponsibility &MR) {
786
787 ExecutorAddrRange EmittedRange;
788 {
789 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
790
791 auto EHFrameRangeItr = InProcessLinks.find(&MR);
792 if (EHFrameRangeItr == InProcessLinks.end())
793 return Error::success();
794
795 EmittedRange = EHFrameRangeItr->second;
796 assert(EmittedRange.Start && "eh-frame addr to register can not be null");
797 InProcessLinks.erase(EHFrameRangeItr);
798 }
799
800 if (auto Err = MR.withResourceKeyDo(
801 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
802 return Err;
803
804 return Registrar->registerEHFrames(EmittedRange);
805 }
806
notifyFailed(MaterializationResponsibility & MR)807 Error EHFrameRegistrationPlugin::notifyFailed(
808 MaterializationResponsibility &MR) {
809 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
810 InProcessLinks.erase(&MR);
811 return Error::success();
812 }
813
notifyRemovingResources(JITDylib & JD,ResourceKey K)814 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD,
815 ResourceKey K) {
816 std::vector<ExecutorAddrRange> RangesToRemove;
817
818 ES.runSessionLocked([&] {
819 auto I = EHFrameRanges.find(K);
820 if (I != EHFrameRanges.end()) {
821 RangesToRemove = std::move(I->second);
822 EHFrameRanges.erase(I);
823 }
824 });
825
826 Error Err = Error::success();
827 while (!RangesToRemove.empty()) {
828 auto RangeToRemove = RangesToRemove.back();
829 RangesToRemove.pop_back();
830 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null");
831 Err = joinErrors(std::move(Err),
832 Registrar->deregisterEHFrames(RangeToRemove));
833 }
834
835 return Err;
836 }
837
notifyTransferringResources(JITDylib & JD,ResourceKey DstKey,ResourceKey SrcKey)838 void EHFrameRegistrationPlugin::notifyTransferringResources(
839 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {
840 auto SI = EHFrameRanges.find(SrcKey);
841 if (SI == EHFrameRanges.end())
842 return;
843
844 auto DI = EHFrameRanges.find(DstKey);
845 if (DI != EHFrameRanges.end()) {
846 auto &SrcRanges = SI->second;
847 auto &DstRanges = DI->second;
848 DstRanges.reserve(DstRanges.size() + SrcRanges.size());
849 for (auto &SrcRange : SrcRanges)
850 DstRanges.push_back(std::move(SrcRange));
851 EHFrameRanges.erase(SI);
852 } else {
853 // We need to move SrcKey's ranges over without invalidating the SI
854 // iterator.
855 auto Tmp = std::move(SI->second);
856 EHFrameRanges.erase(SI);
857 EHFrameRanges[DstKey] = std::move(Tmp);
858 }
859 }
860
861 } // End namespace orc.
862 } // End namespace llvm.
863