1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10 
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13 
14 #include <future>
15 
16 using namespace llvm;
17 using namespace llvm::orc;
18 
19 namespace llvm {
20 namespace orc {
21 
22 class EPCIndirectionUtilsAccess {
23 public:
24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26 
27   static Expected<IndirectStubInfoVector>
28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29     return EPCIU.getIndirectStubs(NumStubs);
30   };
31 };
32 
33 } // end namespace orc
34 } // end namespace llvm
35 
36 namespace {
37 
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41   Error deallocatePool();
42 
43 protected:
44   Error grow() override;
45 
46   using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
47 
48   EPCIndirectionUtils &EPCIU;
49   unsigned TrampolineSize = 0;
50   unsigned TrampolinesPerPage = 0;
51   std::vector<FinalizedAlloc> TrampolineBlocks;
52 };
53 
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55                                 private EPCIndirectionUtilsAccess {
56 public:
57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58 
59   Error deallocateStubs();
60 
61   Error createStub(StringRef StubName, ExecutorAddr StubAddr,
62                    JITSymbolFlags StubFlags) override;
63 
64   Error createStubs(const StubInitsMap &StubInits) override;
65 
66   ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;
67 
68   ExecutorSymbolDef findPointer(StringRef Name) override;
69 
70   Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;
71 
72 private:
73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74 
75   std::mutex ISMMutex;
76   EPCIndirectionUtils &EPCIU;
77   StringMap<StubInfo> StubInfos;
78 };
79 
80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81     : EPCIU(EPCIU) {
82   auto &EPC = EPCIU.getExecutorProcessControl();
83   auto &ABI = EPCIU.getABISupport();
84 
85   TrampolineSize = ABI.getTrampolineSize();
86   TrampolinesPerPage =
87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89 
90 Error EPCTrampolinePool::deallocatePool() {
91   std::promise<MSVCPError> DeallocResultP;
92   auto DeallocResultF = DeallocResultP.get_future();
93 
94   EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
95       std::move(TrampolineBlocks),
96       [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97 
98   return DeallocResultF.get();
99 }
100 
101 Error EPCTrampolinePool::grow() {
102   using namespace jitlink;
103 
104   assert(AvailableTrampolines.empty() &&
105          "Grow called with trampolines still available");
106 
107   auto ResolverAddress = EPCIU.getResolverBlockAddress();
108   assert(ResolverAddress && "Resolver address can not be null");
109 
110   auto &EPC = EPCIU.getExecutorProcessControl();
111   auto PageSize = EPC.getPageSize();
112   auto Alloc = SimpleSegmentAlloc::Create(
113       EPC.getMemMgr(), nullptr,
114       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115   if (!Alloc)
116     return Alloc.takeError();
117 
118   unsigned NumTrampolines = TrampolinesPerPage;
119 
120   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121   EPCIU.getABISupport().writeTrampolines(
122       SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);
123   for (unsigned I = 0; I < NumTrampolines; ++I)
124     AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));
125 
126   auto FA = Alloc->finalize();
127   if (!FA)
128     return FA.takeError();
129 
130   TrampolineBlocks.push_back(std::move(*FA));
131 
132   return Error::success();
133 }
134 
135 Error EPCIndirectStubsManager::createStub(StringRef StubName,
136                                           ExecutorAddr StubAddr,
137                                           JITSymbolFlags StubFlags) {
138   StubInitsMap SIM;
139   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
140   return createStubs(SIM);
141 }
142 
143 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
144   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
145   if (!AvailableStubInfos)
146     return AvailableStubInfos.takeError();
147 
148   {
149     std::lock_guard<std::mutex> Lock(ISMMutex);
150     unsigned ASIdx = 0;
151     for (auto &SI : StubInits) {
152       auto &A = (*AvailableStubInfos)[ASIdx++];
153       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
154     }
155   }
156 
157   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
158   switch (EPCIU.getABISupport().getPointerSize()) {
159   case 4: {
160     unsigned ASIdx = 0;
161     std::vector<tpctypes::UInt32Write> PtrUpdates;
162     for (auto &SI : StubInits)
163       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
164                             static_cast<uint32_t>(SI.second.first.getValue())});
165     return MemAccess.writeUInt32s(PtrUpdates);
166   }
167   case 8: {
168     unsigned ASIdx = 0;
169     std::vector<tpctypes::UInt64Write> PtrUpdates;
170     for (auto &SI : StubInits)
171       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
172                             static_cast<uint64_t>(SI.second.first.getValue())});
173     return MemAccess.writeUInt64s(PtrUpdates);
174   }
175   default:
176     return make_error<StringError>("Unsupported pointer size",
177                                    inconvertibleErrorCode());
178   }
179 }
180 
181 ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,
182                                                     bool ExportedStubsOnly) {
183   std::lock_guard<std::mutex> Lock(ISMMutex);
184   auto I = StubInfos.find(Name);
185   if (I == StubInfos.end())
186     return ExecutorSymbolDef();
187   return {I->second.first.StubAddress, I->second.second};
188 }
189 
190 ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {
191   std::lock_guard<std::mutex> Lock(ISMMutex);
192   auto I = StubInfos.find(Name);
193   if (I == StubInfos.end())
194     return ExecutorSymbolDef();
195   return {I->second.first.PointerAddress, I->second.second};
196 }
197 
198 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
199                                              ExecutorAddr NewAddr) {
200 
201   ExecutorAddr PtrAddr;
202   {
203     std::lock_guard<std::mutex> Lock(ISMMutex);
204     auto I = StubInfos.find(Name);
205     if (I == StubInfos.end())
206       return make_error<StringError>("Unknown stub name",
207                                      inconvertibleErrorCode());
208     PtrAddr = I->second.first.PointerAddress;
209   }
210 
211   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
212   switch (EPCIU.getABISupport().getPointerSize()) {
213   case 4: {
214     tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());
215     return MemAccess.writeUInt32s(PUpdate);
216   }
217   case 8: {
218     tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());
219     return MemAccess.writeUInt64s(PUpdate);
220   }
221   default:
222     return make_error<StringError>("Unsupported pointer size",
223                                    inconvertibleErrorCode());
224   }
225 }
226 
227 } // end anonymous namespace.
228 
229 namespace llvm {
230 namespace orc {
231 
232 EPCIndirectionUtils::ABISupport::~ABISupport() = default;
233 
234 Expected<std::unique_ptr<EPCIndirectionUtils>>
235 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
236   const auto &TT = EPC.getTargetTriple();
237   switch (TT.getArch()) {
238   default:
239     return make_error<StringError>(
240         std::string("No EPCIndirectionUtils available for ") + TT.str(),
241         inconvertibleErrorCode());
242   case Triple::aarch64:
243   case Triple::aarch64_32:
244     return CreateWithABI<OrcAArch64>(EPC);
245 
246   case Triple::x86:
247     return CreateWithABI<OrcI386>(EPC);
248 
249   case Triple::loongarch64:
250     return CreateWithABI<OrcLoongArch64>(EPC);
251 
252   case Triple::mips:
253     return CreateWithABI<OrcMips32Be>(EPC);
254 
255   case Triple::mipsel:
256     return CreateWithABI<OrcMips32Le>(EPC);
257 
258   case Triple::mips64:
259   case Triple::mips64el:
260     return CreateWithABI<OrcMips64>(EPC);
261 
262   case Triple::riscv64:
263     return CreateWithABI<OrcRiscv64>(EPC);
264 
265   case Triple::x86_64:
266     if (TT.getOS() == Triple::OSType::Win32)
267       return CreateWithABI<OrcX86_64_Win32>(EPC);
268     else
269       return CreateWithABI<OrcX86_64_SysV>(EPC);
270   }
271 }
272 
273 Error EPCIndirectionUtils::cleanup() {
274 
275   auto &MemMgr = EPC.getMemMgr();
276   auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
277 
278   if (TP)
279     Err = joinErrors(std::move(Err),
280                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
281 
282   if (ResolverBlock)
283     Err =
284         joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
285 
286   return Err;
287 }
288 
289 Expected<ExecutorAddr>
290 EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,
291                                         ExecutorAddr ReentryCtxAddr) {
292   using namespace jitlink;
293 
294   assert(ABI && "ABI can not be null");
295   auto ResolverSize = ABI->getResolverCodeSize();
296 
297   auto Alloc =
298       SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
299                                  {{MemProt::Read | MemProt::Exec,
300                                    {ResolverSize, Align(EPC.getPageSize())}}});
301 
302   if (!Alloc)
303     return Alloc.takeError();
304 
305   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
306   ResolverBlockAddr = SegInfo.Addr;
307   ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
308                          ReentryFnAddr, ReentryCtxAddr);
309 
310   auto FA = Alloc->finalize();
311   if (!FA)
312     return FA.takeError();
313 
314   ResolverBlock = std::move(*FA);
315   return ResolverBlockAddr;
316 }
317 
318 std::unique_ptr<IndirectStubsManager>
319 EPCIndirectionUtils::createIndirectStubsManager() {
320   return std::make_unique<EPCIndirectStubsManager>(*this);
321 }
322 
323 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
324   if (!TP)
325     TP = std::make_unique<EPCTrampolinePool>(*this);
326   return *TP;
327 }
328 
329 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
330     ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {
331   assert(!LCTM &&
332          "createLazyCallThroughManager can not have been called before");
333   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
334                                                   &getTrampolinePool());
335   return *LCTM;
336 }
337 
338 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
339                                          std::unique_ptr<ABISupport> ABI)
340     : EPC(EPC), ABI(std::move(ABI)) {
341   assert(this->ABI && "ABI can not be null");
342 
343   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
344          "Stubs larger than one page are not supported");
345 }
346 
347 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
348 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
349   using namespace jitlink;
350 
351   std::lock_guard<std::mutex> Lock(EPCUIMutex);
352 
353   // If there aren't enough stubs available then allocate some more.
354   if (NumStubs > AvailableIndirectStubs.size()) {
355     auto NumStubsToAllocate = NumStubs;
356     auto PageSize = EPC.getPageSize();
357     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
358     NumStubsToAllocate = StubBytes / ABI->getStubSize();
359     auto PtrBytes =
360         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
361 
362     auto StubProt = MemProt::Read | MemProt::Exec;
363     auto PtrProt = MemProt::Read | MemProt::Write;
364 
365     auto Alloc = SimpleSegmentAlloc::Create(
366         EPC.getMemMgr(), nullptr,
367         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
368          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
369 
370     if (!Alloc)
371       return Alloc.takeError();
372 
373     auto StubSeg = Alloc->getSegInfo(StubProt);
374     auto PtrSeg = Alloc->getSegInfo(PtrProt);
375 
376     ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,
377                                  PtrSeg.Addr, NumStubsToAllocate);
378 
379     auto FA = Alloc->finalize();
380     if (!FA)
381       return FA.takeError();
382 
383     IndirectStubAllocs.push_back(std::move(*FA));
384 
385     auto StubExecutorAddr = StubSeg.Addr;
386     auto PtrExecutorAddr = PtrSeg.Addr;
387     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
388       AvailableIndirectStubs.push_back(
389           IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));
390       StubExecutorAddr += ABI->getStubSize();
391       PtrExecutorAddr += ABI->getPointerSize();
392     }
393   }
394 
395   assert(NumStubs <= AvailableIndirectStubs.size() &&
396          "Sufficient stubs should have been allocated above");
397 
398   IndirectStubInfoVector Result;
399   while (NumStubs--) {
400     Result.push_back(AvailableIndirectStubs.back());
401     AvailableIndirectStubs.pop_back();
402   }
403 
404   return std::move(Result);
405 }
406 
407 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
408                                 JITTargetAddress TrampolineAddr) {
409   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
410   std::promise<ExecutorAddr> LandingAddrP;
411   auto LandingAddrF = LandingAddrP.get_future();
412   LCTM.resolveTrampolineLandingAddress(
413       ExecutorAddr(TrampolineAddr),
414       [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });
415   return LandingAddrF.get().getValue();
416 }
417 
418 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
419   auto &LCTM = EPCIU.getLazyCallThroughManager();
420   return EPCIU
421       .writeResolverBlock(ExecutorAddr::fromPtr(&reentry),
422                           ExecutorAddr::fromPtr(&LCTM))
423       .takeError();
424 }
425 
426 } // end namespace orc
427 } // end namespace llvm
428