1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10 
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13 
14 #include <future>
15 
16 using namespace llvm;
17 using namespace llvm::orc;
18 
19 namespace llvm {
20 namespace orc {
21 
22 class EPCIndirectionUtilsAccess {
23 public:
24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26 
27   static Expected<IndirectStubInfoVector>
28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29     return EPCIU.getIndirectStubs(NumStubs);
30   };
31 };
32 
33 } // end namespace orc
34 } // end namespace llvm
35 
36 namespace {
37 
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41   Error deallocatePool();
42 
43 protected:
44   Error grow() override;
45 
46   using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;
47 
48   EPCIndirectionUtils &EPCIU;
49   unsigned TrampolineSize = 0;
50   unsigned TrampolinesPerPage = 0;
51   std::vector<FinalizedAlloc> TrampolineBlocks;
52 };
53 
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55                                 private EPCIndirectionUtilsAccess {
56 public:
57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58 
59   Error deallocateStubs();
60 
61   Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62                    JITSymbolFlags StubFlags) override;
63 
64   Error createStubs(const StubInitsMap &StubInits) override;
65 
66   JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67 
68   JITEvaluatedSymbol findPointer(StringRef Name) override;
69 
70   Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
71 
72 private:
73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74 
75   std::mutex ISMMutex;
76   EPCIndirectionUtils &EPCIU;
77   StringMap<StubInfo> StubInfos;
78 };
79 
80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81     : EPCIU(EPCIU) {
82   auto &EPC = EPCIU.getExecutorProcessControl();
83   auto &ABI = EPCIU.getABISupport();
84 
85   TrampolineSize = ABI.getTrampolineSize();
86   TrampolinesPerPage =
87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89 
90 Error EPCTrampolinePool::deallocatePool() {
91   std::promise<MSVCPError> DeallocResultP;
92   auto DeallocResultF = DeallocResultP.get_future();
93 
94   EPCIU.getExecutorProcessControl().getMemMgr().deallocate(
95       std::move(TrampolineBlocks),
96       [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97 
98   return DeallocResultF.get();
99 }
100 
101 Error EPCTrampolinePool::grow() {
102   using namespace jitlink;
103 
104   assert(AvailableTrampolines.empty() &&
105          "Grow called with trampolines still available");
106 
107   auto ResolverAddress = EPCIU.getResolverBlockAddress();
108   assert(ResolverAddress && "Resolver address can not be null");
109 
110   auto &EPC = EPCIU.getExecutorProcessControl();
111   auto PageSize = EPC.getPageSize();
112   auto Alloc = SimpleSegmentAlloc::Create(
113       EPC.getMemMgr(), nullptr,
114       {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115   if (!Alloc)
116     return Alloc.takeError();
117 
118   unsigned NumTrampolines = TrampolinesPerPage;
119 
120   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121   EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(),
122                                          SegInfo.Addr.getValue(),
123                                          ResolverAddress, NumTrampolines);
124   for (unsigned I = 0; I < NumTrampolines; ++I)
125     AvailableTrampolines.push_back(SegInfo.Addr.getValue() +
126                                    (I * TrampolineSize));
127 
128   auto FA = Alloc->finalize();
129   if (!FA)
130     return FA.takeError();
131 
132   TrampolineBlocks.push_back(std::move(*FA));
133 
134   return Error::success();
135 }
136 
137 Error EPCIndirectStubsManager::createStub(StringRef StubName,
138                                           JITTargetAddress StubAddr,
139                                           JITSymbolFlags StubFlags) {
140   StubInitsMap SIM;
141   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142   return createStubs(SIM);
143 }
144 
145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
147   if (!AvailableStubInfos)
148     return AvailableStubInfos.takeError();
149 
150   {
151     std::lock_guard<std::mutex> Lock(ISMMutex);
152     unsigned ASIdx = 0;
153     for (auto &SI : StubInits) {
154       auto &A = (*AvailableStubInfos)[ASIdx++];
155       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156     }
157   }
158 
159   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
160   switch (EPCIU.getABISupport().getPointerSize()) {
161   case 4: {
162     unsigned ASIdx = 0;
163     std::vector<tpctypes::UInt32Write> PtrUpdates;
164     for (auto &SI : StubInits)
165       PtrUpdates.push_back(
166           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
167            static_cast<uint32_t>(SI.second.first)});
168     return MemAccess.writeUInt32s(PtrUpdates);
169   }
170   case 8: {
171     unsigned ASIdx = 0;
172     std::vector<tpctypes::UInt64Write> PtrUpdates;
173     for (auto &SI : StubInits)
174       PtrUpdates.push_back(
175           {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
176            static_cast<uint64_t>(SI.second.first)});
177     return MemAccess.writeUInt64s(PtrUpdates);
178   }
179   default:
180     return make_error<StringError>("Unsupported pointer size",
181                                    inconvertibleErrorCode());
182   }
183 }
184 
185 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
186                                                      bool ExportedStubsOnly) {
187   std::lock_guard<std::mutex> Lock(ISMMutex);
188   auto I = StubInfos.find(Name);
189   if (I == StubInfos.end())
190     return nullptr;
191   return {I->second.first.StubAddress, I->second.second};
192 }
193 
194 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
195   std::lock_guard<std::mutex> Lock(ISMMutex);
196   auto I = StubInfos.find(Name);
197   if (I == StubInfos.end())
198     return nullptr;
199   return {I->second.first.PointerAddress, I->second.second};
200 }
201 
202 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
203                                              JITTargetAddress NewAddr) {
204 
205   JITTargetAddress PtrAddr = 0;
206   {
207     std::lock_guard<std::mutex> Lock(ISMMutex);
208     auto I = StubInfos.find(Name);
209     if (I == StubInfos.end())
210       return make_error<StringError>("Unknown stub name",
211                                      inconvertibleErrorCode());
212     PtrAddr = I->second.first.PointerAddress;
213   }
214 
215   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
216   switch (EPCIU.getABISupport().getPointerSize()) {
217   case 4: {
218     tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
219     return MemAccess.writeUInt32s(PUpdate);
220   }
221   case 8: {
222     tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
223     return MemAccess.writeUInt64s(PUpdate);
224   }
225   default:
226     return make_error<StringError>("Unsupported pointer size",
227                                    inconvertibleErrorCode());
228   }
229 }
230 
231 } // end anonymous namespace.
232 
233 namespace llvm {
234 namespace orc {
235 
236 EPCIndirectionUtils::ABISupport::~ABISupport() = default;
237 
238 Expected<std::unique_ptr<EPCIndirectionUtils>>
239 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
240   const auto &TT = EPC.getTargetTriple();
241   switch (TT.getArch()) {
242   default:
243     return make_error<StringError>(
244         std::string("No EPCIndirectionUtils available for ") + TT.str(),
245         inconvertibleErrorCode());
246   case Triple::aarch64:
247   case Triple::aarch64_32:
248     return CreateWithABI<OrcAArch64>(EPC);
249 
250   case Triple::x86:
251     return CreateWithABI<OrcI386>(EPC);
252 
253   case Triple::mips:
254     return CreateWithABI<OrcMips32Be>(EPC);
255 
256   case Triple::mipsel:
257     return CreateWithABI<OrcMips32Le>(EPC);
258 
259   case Triple::mips64:
260   case Triple::mips64el:
261     return CreateWithABI<OrcMips64>(EPC);
262 
263   case Triple::riscv64:
264     return CreateWithABI<OrcRiscv64>(EPC);
265 
266   case Triple::x86_64:
267     if (TT.getOS() == Triple::OSType::Win32)
268       return CreateWithABI<OrcX86_64_Win32>(EPC);
269     else
270       return CreateWithABI<OrcX86_64_SysV>(EPC);
271   }
272 }
273 
274 Error EPCIndirectionUtils::cleanup() {
275 
276   auto &MemMgr = EPC.getMemMgr();
277   auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
278 
279   if (TP)
280     Err = joinErrors(std::move(Err),
281                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
282 
283   if (ResolverBlock)
284     Err =
285         joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
286 
287   return Err;
288 }
289 
290 Expected<JITTargetAddress>
291 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
292                                         JITTargetAddress ReentryCtxAddr) {
293   using namespace jitlink;
294 
295   assert(ABI && "ABI can not be null");
296   auto ResolverSize = ABI->getResolverCodeSize();
297 
298   auto Alloc =
299       SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
300                                  {{MemProt::Read | MemProt::Exec,
301                                    {ResolverSize, Align(EPC.getPageSize())}}});
302 
303   if (!Alloc)
304     return Alloc.takeError();
305 
306   auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
307   ResolverBlockAddr = SegInfo.Addr.getValue();
308   ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
309                          ReentryFnAddr, ReentryCtxAddr);
310 
311   auto FA = Alloc->finalize();
312   if (!FA)
313     return FA.takeError();
314 
315   ResolverBlock = std::move(*FA);
316   return ResolverBlockAddr;
317 }
318 
319 std::unique_ptr<IndirectStubsManager>
320 EPCIndirectionUtils::createIndirectStubsManager() {
321   return std::make_unique<EPCIndirectStubsManager>(*this);
322 }
323 
324 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
325   if (!TP)
326     TP = std::make_unique<EPCTrampolinePool>(*this);
327   return *TP;
328 }
329 
330 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
331     ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
332   assert(!LCTM &&
333          "createLazyCallThroughManager can not have been called before");
334   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
335                                                   &getTrampolinePool());
336   return *LCTM;
337 }
338 
339 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
340                                          std::unique_ptr<ABISupport> ABI)
341     : EPC(EPC), ABI(std::move(ABI)) {
342   assert(this->ABI && "ABI can not be null");
343 
344   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
345          "Stubs larger than one page are not supported");
346 }
347 
348 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
349 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
350   using namespace jitlink;
351 
352   std::lock_guard<std::mutex> Lock(EPCUIMutex);
353 
354   // If there aren't enough stubs available then allocate some more.
355   if (NumStubs > AvailableIndirectStubs.size()) {
356     auto NumStubsToAllocate = NumStubs;
357     auto PageSize = EPC.getPageSize();
358     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
359     NumStubsToAllocate = StubBytes / ABI->getStubSize();
360     auto PtrBytes =
361         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
362 
363     auto StubProt = MemProt::Read | MemProt::Exec;
364     auto PtrProt = MemProt::Read | MemProt::Write;
365 
366     auto Alloc = SimpleSegmentAlloc::Create(
367         EPC.getMemMgr(), nullptr,
368         {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
369          {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
370 
371     if (!Alloc)
372       return Alloc.takeError();
373 
374     auto StubSeg = Alloc->getSegInfo(StubProt);
375     auto PtrSeg = Alloc->getSegInfo(PtrProt);
376 
377     ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(),
378                                  StubSeg.Addr.getValue(),
379                                  PtrSeg.Addr.getValue(), NumStubsToAllocate);
380 
381     auto FA = Alloc->finalize();
382     if (!FA)
383       return FA.takeError();
384 
385     IndirectStubAllocs.push_back(std::move(*FA));
386 
387     auto StubExecutorAddr = StubSeg.Addr;
388     auto PtrExecutorAddr = PtrSeg.Addr;
389     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
390       AvailableIndirectStubs.push_back(IndirectStubInfo(
391           StubExecutorAddr.getValue(), PtrExecutorAddr.getValue()));
392       StubExecutorAddr += ABI->getStubSize();
393       PtrExecutorAddr += ABI->getPointerSize();
394     }
395   }
396 
397   assert(NumStubs <= AvailableIndirectStubs.size() &&
398          "Sufficient stubs should have been allocated above");
399 
400   IndirectStubInfoVector Result;
401   while (NumStubs--) {
402     Result.push_back(AvailableIndirectStubs.back());
403     AvailableIndirectStubs.pop_back();
404   }
405 
406   return std::move(Result);
407 }
408 
409 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
410                                 JITTargetAddress TrampolineAddr) {
411   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
412   std::promise<JITTargetAddress> LandingAddrP;
413   auto LandingAddrF = LandingAddrP.get_future();
414   LCTM.resolveTrampolineLandingAddress(
415       TrampolineAddr,
416       [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
417   return LandingAddrF.get();
418 }
419 
420 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
421   auto &LCTM = EPCIU.getLazyCallThroughManager();
422   return EPCIU
423       .writeResolverBlock(pointerToJITTargetAddress(&reentry),
424                           pointerToJITTargetAddress(&LCTM))
425       .takeError();
426 }
427 
428 } // end namespace orc
429 } // end namespace llvm
430