1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10 
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13 
14 #include <future>
15 
16 using namespace llvm;
17 using namespace llvm::orc;
18 
19 namespace llvm {
20 namespace orc {
21 
22 class EPCIndirectionUtilsAccess {
23 public:
24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26 
27   static Expected<IndirectStubInfoVector>
getIndirectStubs(EPCIndirectionUtils & EPCIU,unsigned NumStubs)28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29     return EPCIU.getIndirectStubs(NumStubs);
30   };
31 };
32 
33 } // end namespace orc
34 } // end namespace llvm
35 
36 namespace {
37 
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41   Error deallocatePool();
42 
43 protected:
44   Error grow() override;
45 
46   using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
47 
48   EPCIndirectionUtils &EPCIU;
49   unsigned TrampolineSize = 0;
50   unsigned TrampolinesPerPage = 0;
51   std::vector<FinalizedAlloc> TrampolineBlocks;
52 };
53 
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55                                 private EPCIndirectionUtilsAccess {
56 public:
EPCIndirectStubsManager(EPCIndirectionUtils & EPCIU)57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58 
59   Error deallocateStubs();
60 
61   Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62                    JITSymbolFlags StubFlags) override;
63 
64   Error createStubs(const StubInitsMap &StubInits) override;
65 
66   JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67 
68   JITEvaluatedSymbol findPointer(StringRef Name) override;
69 
70   Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
71 
72 private:
73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74 
75   std::mutex ISMMutex;
76   EPCIndirectionUtils &EPCIU;
77   StringMap<StubInfo> StubInfos;
78 };
79 
EPCTrampolinePool(EPCIndirectionUtils & EPCIU)80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81     : EPCIU(EPCIU) {
82   auto &EPC = EPCIU.getExecutorProcessControl();
83   auto &ABI = EPCIU.getABISupport();
84 
85   TrampolineSize = ABI.getTrampolineSize();
86   TrampolinesPerPage =
87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89 
deallocatePool()90 Error EPCTrampolinePool::deallocatePool() {
91   std::promise<MSVCPError> DeallocResultP;
92   auto DeallocResultF = DeallocResultP.get_future();
93 
94   EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
95       std::move(TrampolineBlocks),
96       [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97 
98   return DeallocResultF.get();
99 }
100 
grow()101 Error EPCTrampolinePool::grow() {
102   using namespace jitlink;
103 
104   assert(AvailableTrampolines.empty() &&
105          "Grow called with trampolines still available");
106 
107   auto ResolverAddress = EPCIU.getResolverBlockAddress();
108   assert(ResolverAddress && "Resolver address can not be null");
109 
110   auto &EPC = EPCIU.getExecutorProcessControl();
111   auto PageSize = EPC.getPageSize();
112   auto Alloc = SimpleSegmentAlloc::Create(
113       EPC.getMemMgr(), nullptr,
114       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115   if (!Alloc)
116     return Alloc.takeError();
117 
118   unsigned NumTrampolines = TrampolinesPerPage;
119 
120   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121   EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(),
122                                          SegInfo.Addr.getValue(),
123                                          ResolverAddress, NumTrampolines);
124   for (unsigned I = 0; I < NumTrampolines; ++I)
125     AvailableTrampolines.push_back(SegInfo.Addr.getValue() +
126                                    (I * TrampolineSize));
127 
128   auto FA = Alloc->finalize();
129   if (!FA)
130     return FA.takeError();
131 
132   TrampolineBlocks.push_back(std::move(*FA));
133 
134   return Error::success();
135 }
136 
createStub(StringRef StubName,JITTargetAddress StubAddr,JITSymbolFlags StubFlags)137 Error EPCIndirectStubsManager::createStub(StringRef StubName,
138                                           JITTargetAddress StubAddr,
139                                           JITSymbolFlags StubFlags) {
140   StubInitsMap SIM;
141   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142   return createStubs(SIM);
143 }
144 
createStubs(const StubInitsMap & StubInits)145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
147   if (!AvailableStubInfos)
148     return AvailableStubInfos.takeError();
149 
150   {
151     std::lock_guard<std::mutex> Lock(ISMMutex);
152     unsigned ASIdx = 0;
153     for (auto &SI : StubInits) {
154       auto &A = (*AvailableStubInfos)[ASIdx++];
155       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156     }
157   }
158 
159   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
160   switch (EPCIU.getABISupport().getPointerSize()) {
161   case 4: {
162     unsigned ASIdx = 0;
163     std::vector<tpctypes::UInt32Write> PtrUpdates;
164     for (auto &SI : StubInits)
165       PtrUpdates.push_back(
166           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
167            static_cast<uint32_t>(SI.second.first)});
168     return MemAccess.writeUInt32s(PtrUpdates);
169   }
170   case 8: {
171     unsigned ASIdx = 0;
172     std::vector<tpctypes::UInt64Write> PtrUpdates;
173     for (auto &SI : StubInits)
174       PtrUpdates.push_back(
175           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
176            static_cast<uint64_t>(SI.second.first)});
177     return MemAccess.writeUInt64s(PtrUpdates);
178   }
179   default:
180     return make_error<StringError>("Unsupported pointer size",
181                                    inconvertibleErrorCode());
182   }
183 }
184 
findStub(StringRef Name,bool ExportedStubsOnly)185 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
186                                                      bool ExportedStubsOnly) {
187   std::lock_guard<std::mutex> Lock(ISMMutex);
188   auto I = StubInfos.find(Name);
189   if (I == StubInfos.end())
190     return nullptr;
191   return {I->second.first.StubAddress, I->second.second};
192 }
193 
findPointer(StringRef Name)194 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
195   std::lock_guard<std::mutex> Lock(ISMMutex);
196   auto I = StubInfos.find(Name);
197   if (I == StubInfos.end())
198     return nullptr;
199   return {I->second.first.PointerAddress, I->second.second};
200 }
201 
updatePointer(StringRef Name,JITTargetAddress NewAddr)202 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
203                                              JITTargetAddress NewAddr) {
204 
205   JITTargetAddress PtrAddr = 0;
206   {
207     std::lock_guard<std::mutex> Lock(ISMMutex);
208     auto I = StubInfos.find(Name);
209     if (I == StubInfos.end())
210       return make_error<StringError>("Unknown stub name",
211                                      inconvertibleErrorCode());
212     PtrAddr = I->second.first.PointerAddress;
213   }
214 
215   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
216   switch (EPCIU.getABISupport().getPointerSize()) {
217   case 4: {
218     tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
219     return MemAccess.writeUInt32s(PUpdate);
220   }
221   case 8: {
222     tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
223     return MemAccess.writeUInt64s(PUpdate);
224   }
225   default:
226     return make_error<StringError>("Unsupported pointer size",
227                                    inconvertibleErrorCode());
228   }
229 }
230 
231 } // end anonymous namespace.
232 
233 namespace llvm {
234 namespace orc {
235 
236 EPCIndirectionUtils::ABISupport::~ABISupport() = default;
237 
238 Expected<std::unique_ptr<EPCIndirectionUtils>>
Create(ExecutorProcessControl & EPC)239 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
240   const auto &TT = EPC.getTargetTriple();
241   switch (TT.getArch()) {
242   default:
243     return make_error<StringError>(
244         std::string("No EPCIndirectionUtils available for ") + TT.str(),
245         inconvertibleErrorCode());
246   case Triple::aarch64:
247   case Triple::aarch64_32:
248     return CreateWithABI<OrcAArch64>(EPC);
249 
250   case Triple::x86:
251     return CreateWithABI<OrcI386>(EPC);
252 
253   case Triple::loongarch64:
254     return CreateWithABI<OrcLoongArch64>(EPC);
255 
256   case Triple::mips:
257     return CreateWithABI<OrcMips32Be>(EPC);
258 
259   case Triple::mipsel:
260     return CreateWithABI<OrcMips32Le>(EPC);
261 
262   case Triple::mips64:
263   case Triple::mips64el:
264     return CreateWithABI<OrcMips64>(EPC);
265 
266   case Triple::riscv64:
267     return CreateWithABI<OrcRiscv64>(EPC);
268 
269   case Triple::x86_64:
270     if (TT.getOS() == Triple::OSType::Win32)
271       return CreateWithABI<OrcX86_64_Win32>(EPC);
272     else
273       return CreateWithABI<OrcX86_64_SysV>(EPC);
274   }
275 }
276 
cleanup()277 Error EPCIndirectionUtils::cleanup() {
278 
279   auto &MemMgr = EPC.getMemMgr();
280   auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
281 
282   if (TP)
283     Err = joinErrors(std::move(Err),
284                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
285 
286   if (ResolverBlock)
287     Err =
288         joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
289 
290   return Err;
291 }
292 
293 Expected<JITTargetAddress>
writeResolverBlock(JITTargetAddress ReentryFnAddr,JITTargetAddress ReentryCtxAddr)294 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
295                                         JITTargetAddress ReentryCtxAddr) {
296   using namespace jitlink;
297 
298   assert(ABI && "ABI can not be null");
299   auto ResolverSize = ABI->getResolverCodeSize();
300 
301   auto Alloc =
302       SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
303                                  {{MemProt::Read | MemProt::Exec,
304                                    {ResolverSize, Align(EPC.getPageSize())}}});
305 
306   if (!Alloc)
307     return Alloc.takeError();
308 
309   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
310   ResolverBlockAddr = SegInfo.Addr.getValue();
311   ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
312                          ReentryFnAddr, ReentryCtxAddr);
313 
314   auto FA = Alloc->finalize();
315   if (!FA)
316     return FA.takeError();
317 
318   ResolverBlock = std::move(*FA);
319   return ResolverBlockAddr;
320 }
321 
322 std::unique_ptr<IndirectStubsManager>
createIndirectStubsManager()323 EPCIndirectionUtils::createIndirectStubsManager() {
324   return std::make_unique<EPCIndirectStubsManager>(*this);
325 }
326 
getTrampolinePool()327 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
328   if (!TP)
329     TP = std::make_unique<EPCTrampolinePool>(*this);
330   return *TP;
331 }
332 
createLazyCallThroughManager(ExecutionSession & ES,JITTargetAddress ErrorHandlerAddr)333 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
334     ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
335   assert(!LCTM &&
336          "createLazyCallThroughManager can not have been called before");
337   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
338                                                   &getTrampolinePool());
339   return *LCTM;
340 }
341 
EPCIndirectionUtils(ExecutorProcessControl & EPC,std::unique_ptr<ABISupport> ABI)342 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
343                                          std::unique_ptr<ABISupport> ABI)
344     : EPC(EPC), ABI(std::move(ABI)) {
345   assert(this->ABI && "ABI can not be null");
346 
347   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
348          "Stubs larger than one page are not supported");
349 }
350 
351 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
getIndirectStubs(unsigned NumStubs)352 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
353   using namespace jitlink;
354 
355   std::lock_guard<std::mutex> Lock(EPCUIMutex);
356 
357   // If there aren't enough stubs available then allocate some more.
358   if (NumStubs > AvailableIndirectStubs.size()) {
359     auto NumStubsToAllocate = NumStubs;
360     auto PageSize = EPC.getPageSize();
361     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
362     NumStubsToAllocate = StubBytes / ABI->getStubSize();
363     auto PtrBytes =
364         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
365 
366     auto StubProt = MemProt::Read | MemProt::Exec;
367     auto PtrProt = MemProt::Read | MemProt::Write;
368 
369     auto Alloc = SimpleSegmentAlloc::Create(
370         EPC.getMemMgr(), nullptr,
371         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
372          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
373 
374     if (!Alloc)
375       return Alloc.takeError();
376 
377     auto StubSeg = Alloc->getSegInfo(StubProt);
378     auto PtrSeg = Alloc->getSegInfo(PtrProt);
379 
380     ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(),
381                                  StubSeg.Addr.getValue(),
382                                  PtrSeg.Addr.getValue(), NumStubsToAllocate);
383 
384     auto FA = Alloc->finalize();
385     if (!FA)
386       return FA.takeError();
387 
388     IndirectStubAllocs.push_back(std::move(*FA));
389 
390     auto StubExecutorAddr = StubSeg.Addr;
391     auto PtrExecutorAddr = PtrSeg.Addr;
392     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
393       AvailableIndirectStubs.push_back(IndirectStubInfo(
394           StubExecutorAddr.getValue(), PtrExecutorAddr.getValue()));
395       StubExecutorAddr += ABI->getStubSize();
396       PtrExecutorAddr += ABI->getPointerSize();
397     }
398   }
399 
400   assert(NumStubs <= AvailableIndirectStubs.size() &&
401          "Sufficient stubs should have been allocated above");
402 
403   IndirectStubInfoVector Result;
404   while (NumStubs--) {
405     Result.push_back(AvailableIndirectStubs.back());
406     AvailableIndirectStubs.pop_back();
407   }
408 
409   return std::move(Result);
410 }
411 
reentry(JITTargetAddress LCTMAddr,JITTargetAddress TrampolineAddr)412 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
413                                 JITTargetAddress TrampolineAddr) {
414   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
415   std::promise<JITTargetAddress> LandingAddrP;
416   auto LandingAddrF = LandingAddrP.get_future();
417   LCTM.resolveTrampolineLandingAddress(
418       TrampolineAddr,
419       [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
420   return LandingAddrF.get();
421 }
422 
setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils & EPCIU)423 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
424   auto &LCTM = EPCIU.getLazyCallThroughManager();
425   return EPCIU
426       .writeResolverBlock(pointerToJITTargetAddress(&reentry),
427                           pointerToJITTargetAddress(&LCTM))
428       .takeError();
429 }
430 
431 } // end namespace orc
432 } // end namespace llvm
433