1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10 
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13 
14 #include <future>
15 
16 using namespace llvm;
17 using namespace llvm::orc;
18 
19 namespace llvm {
20 namespace orc {
21 
22 class EPCIndirectionUtilsAccess {
23 public:
24   using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25   using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26 
27   static Expected<IndirectStubInfoVector>
getIndirectStubs(EPCIndirectionUtils & EPCIU,unsigned NumStubs)28   getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29     return EPCIU.getIndirectStubs(NumStubs);
30   };
31 };
32 
33 } // end namespace orc
34 } // end namespace llvm
35 
36 namespace {
37 
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40   EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41   Error deallocatePool();
42 
43 protected:
44   Error grow() override;
45 
46   using Allocation = jitlink::JITLinkMemoryManager::Allocation;
47 
48   EPCIndirectionUtils &EPCIU;
49   unsigned TrampolineSize = 0;
50   unsigned TrampolinesPerPage = 0;
51   std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
52 };
53 
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55                                 private EPCIndirectionUtilsAccess {
56 public:
EPCIndirectStubsManager(EPCIndirectionUtils & EPCIU)57   EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58 
59   Error deallocateStubs();
60 
61   Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62                    JITSymbolFlags StubFlags) override;
63 
64   Error createStubs(const StubInitsMap &StubInits) override;
65 
66   JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67 
68   JITEvaluatedSymbol findPointer(StringRef Name) override;
69 
70   Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
71 
72 private:
73   using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74 
75   std::mutex ISMMutex;
76   EPCIndirectionUtils &EPCIU;
77   StringMap<StubInfo> StubInfos;
78 };
79 
EPCTrampolinePool(EPCIndirectionUtils & EPCIU)80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81     : EPCIU(EPCIU) {
82   auto &EPC = EPCIU.getExecutorProcessControl();
83   auto &ABI = EPCIU.getABISupport();
84 
85   TrampolineSize = ABI.getTrampolineSize();
86   TrampolinesPerPage =
87       (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89 
deallocatePool()90 Error EPCTrampolinePool::deallocatePool() {
91   Error Err = Error::success();
92   for (auto &Alloc : TrampolineBlocks)
93     Err = joinErrors(std::move(Err), Alloc->deallocate());
94   return Err;
95 }
96 
grow()97 Error EPCTrampolinePool::grow() {
98   assert(AvailableTrampolines.empty() &&
99          "Grow called with trampolines still available");
100 
101   auto ResolverAddress = EPCIU.getResolverBlockAddress();
102   assert(ResolverAddress && "Resolver address can not be null");
103 
104   auto &EPC = EPCIU.getExecutorProcessControl();
105   constexpr auto TrampolinePagePermissions =
106       static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
107                                                 sys::Memory::MF_EXEC);
108   auto PageSize = EPC.getPageSize();
109   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
110   Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
111                                         0};
112   auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
113 
114   if (!Alloc)
115     return Alloc.takeError();
116 
117   unsigned NumTrampolines = TrampolinesPerPage;
118 
119   auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions);
120   auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
121 
122   EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress,
123                                          ResolverAddress, NumTrampolines);
124 
125   auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
126   for (unsigned I = 0; I < NumTrampolines; ++I)
127     AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
128 
129   if (auto Err = (*Alloc)->finalize())
130     return Err;
131 
132   TrampolineBlocks.push_back(std::move(*Alloc));
133 
134   return Error::success();
135 }
136 
createStub(StringRef StubName,JITTargetAddress StubAddr,JITSymbolFlags StubFlags)137 Error EPCIndirectStubsManager::createStub(StringRef StubName,
138                                           JITTargetAddress StubAddr,
139                                           JITSymbolFlags StubFlags) {
140   StubInitsMap SIM;
141   SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142   return createStubs(SIM);
143 }
144 
createStubs(const StubInitsMap & StubInits)145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146   auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
147   if (!AvailableStubInfos)
148     return AvailableStubInfos.takeError();
149 
150   {
151     std::lock_guard<std::mutex> Lock(ISMMutex);
152     unsigned ASIdx = 0;
153     for (auto &SI : StubInits) {
154       auto &A = (*AvailableStubInfos)[ASIdx++];
155       StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156     }
157   }
158 
159   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
160   switch (EPCIU.getABISupport().getPointerSize()) {
161   case 4: {
162     unsigned ASIdx = 0;
163     std::vector<tpctypes::UInt32Write> PtrUpdates;
164     for (auto &SI : StubInits)
165       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
166                             static_cast<uint32_t>(SI.second.first)});
167     return MemAccess.writeUInt32s(PtrUpdates);
168   }
169   case 8: {
170     unsigned ASIdx = 0;
171     std::vector<tpctypes::UInt64Write> PtrUpdates;
172     for (auto &SI : StubInits)
173       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
174                             static_cast<uint64_t>(SI.second.first)});
175     return MemAccess.writeUInt64s(PtrUpdates);
176   }
177   default:
178     return make_error<StringError>("Unsupported pointer size",
179                                    inconvertibleErrorCode());
180   }
181 }
182 
findStub(StringRef Name,bool ExportedStubsOnly)183 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
184                                                      bool ExportedStubsOnly) {
185   std::lock_guard<std::mutex> Lock(ISMMutex);
186   auto I = StubInfos.find(Name);
187   if (I == StubInfos.end())
188     return nullptr;
189   return {I->second.first.StubAddress, I->second.second};
190 }
191 
findPointer(StringRef Name)192 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
193   std::lock_guard<std::mutex> Lock(ISMMutex);
194   auto I = StubInfos.find(Name);
195   if (I == StubInfos.end())
196     return nullptr;
197   return {I->second.first.PointerAddress, I->second.second};
198 }
199 
updatePointer(StringRef Name,JITTargetAddress NewAddr)200 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
201                                              JITTargetAddress NewAddr) {
202 
203   JITTargetAddress PtrAddr = 0;
204   {
205     std::lock_guard<std::mutex> Lock(ISMMutex);
206     auto I = StubInfos.find(Name);
207     if (I == StubInfos.end())
208       return make_error<StringError>("Unknown stub name",
209                                      inconvertibleErrorCode());
210     PtrAddr = I->second.first.PointerAddress;
211   }
212 
213   auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
214   switch (EPCIU.getABISupport().getPointerSize()) {
215   case 4: {
216     tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr);
217     return MemAccess.writeUInt32s(PUpdate);
218   }
219   case 8: {
220     tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr);
221     return MemAccess.writeUInt64s(PUpdate);
222   }
223   default:
224     return make_error<StringError>("Unsupported pointer size",
225                                    inconvertibleErrorCode());
226   }
227 }
228 
229 } // end anonymous namespace.
230 
231 namespace llvm {
232 namespace orc {
233 
~ABISupport()234 EPCIndirectionUtils::ABISupport::~ABISupport() {}
235 
236 Expected<std::unique_ptr<EPCIndirectionUtils>>
Create(ExecutorProcessControl & EPC)237 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
238   const auto &TT = EPC.getTargetTriple();
239   switch (TT.getArch()) {
240   default:
241     return make_error<StringError>(
242         std::string("No EPCIndirectionUtils available for ") + TT.str(),
243         inconvertibleErrorCode());
244   case Triple::aarch64:
245   case Triple::aarch64_32:
246     return CreateWithABI<OrcAArch64>(EPC);
247 
248   case Triple::x86:
249     return CreateWithABI<OrcI386>(EPC);
250 
251   case Triple::mips:
252     return CreateWithABI<OrcMips32Be>(EPC);
253 
254   case Triple::mipsel:
255     return CreateWithABI<OrcMips32Le>(EPC);
256 
257   case Triple::mips64:
258   case Triple::mips64el:
259     return CreateWithABI<OrcMips64>(EPC);
260 
261   case Triple::x86_64:
262     if (TT.getOS() == Triple::OSType::Win32)
263       return CreateWithABI<OrcX86_64_Win32>(EPC);
264     else
265       return CreateWithABI<OrcX86_64_SysV>(EPC);
266   }
267 }
268 
cleanup()269 Error EPCIndirectionUtils::cleanup() {
270   Error Err = Error::success();
271 
272   for (auto &A : IndirectStubAllocs)
273     Err = joinErrors(std::move(Err), A->deallocate());
274 
275   if (TP)
276     Err = joinErrors(std::move(Err),
277                      static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
278 
279   if (ResolverBlock)
280     Err = joinErrors(std::move(Err), ResolverBlock->deallocate());
281 
282   return Err;
283 }
284 
285 Expected<JITTargetAddress>
writeResolverBlock(JITTargetAddress ReentryFnAddr,JITTargetAddress ReentryCtxAddr)286 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
287                                         JITTargetAddress ReentryCtxAddr) {
288   assert(ABI && "ABI can not be null");
289   constexpr auto ResolverBlockPermissions =
290       static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
291                                                 sys::Memory::MF_EXEC);
292   auto ResolverSize = ABI->getResolverCodeSize();
293 
294   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
295   Request[ResolverBlockPermissions] = {EPC.getPageSize(),
296                                        static_cast<size_t>(ResolverSize), 0};
297   auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
298   if (!Alloc)
299     return Alloc.takeError();
300 
301   auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
302   ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
303   ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
304                          ReentryCtxAddr);
305 
306   if (auto Err = (*Alloc)->finalize())
307     return std::move(Err);
308 
309   ResolverBlock = std::move(*Alloc);
310   return ResolverBlockAddr;
311 }
312 
313 std::unique_ptr<IndirectStubsManager>
createIndirectStubsManager()314 EPCIndirectionUtils::createIndirectStubsManager() {
315   return std::make_unique<EPCIndirectStubsManager>(*this);
316 }
317 
getTrampolinePool()318 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
319   if (!TP)
320     TP = std::make_unique<EPCTrampolinePool>(*this);
321   return *TP;
322 }
323 
createLazyCallThroughManager(ExecutionSession & ES,JITTargetAddress ErrorHandlerAddr)324 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
325     ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
326   assert(!LCTM &&
327          "createLazyCallThroughManager can not have been called before");
328   LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
329                                                   &getTrampolinePool());
330   return *LCTM;
331 }
332 
EPCIndirectionUtils(ExecutorProcessControl & EPC,std::unique_ptr<ABISupport> ABI)333 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
334                                          std::unique_ptr<ABISupport> ABI)
335     : EPC(EPC), ABI(std::move(ABI)) {
336   assert(this->ABI && "ABI can not be null");
337 
338   assert(EPC.getPageSize() > getABISupport().getStubSize() &&
339          "Stubs larger than one page are not supported");
340 }
341 
342 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
getIndirectStubs(unsigned NumStubs)343 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
344 
345   std::lock_guard<std::mutex> Lock(EPCUIMutex);
346 
347   // If there aren't enough stubs available then allocate some more.
348   if (NumStubs > AvailableIndirectStubs.size()) {
349     auto NumStubsToAllocate = NumStubs;
350     auto PageSize = EPC.getPageSize();
351     auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
352     NumStubsToAllocate = StubBytes / ABI->getStubSize();
353     auto PointerBytes =
354         alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
355 
356     constexpr auto StubPagePermissions =
357         static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
358                                                   sys::Memory::MF_EXEC);
359     constexpr auto PointerPagePermissions =
360         static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
361                                                   sys::Memory::MF_WRITE);
362 
363     jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
364     Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
365                                     0};
366     Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
367     auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
368     if (!Alloc)
369       return Alloc.takeError();
370 
371     auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions);
372     auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions);
373 
374     ABI->writeIndirectStubsBlock(
375         (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr,
376         PointerTargetAddr, NumStubsToAllocate);
377 
378     if (auto Err = (*Alloc)->finalize())
379       return std::move(Err);
380 
381     for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
382       AvailableIndirectStubs.push_back(
383           IndirectStubInfo(StubTargetAddr, PointerTargetAddr));
384       StubTargetAddr += ABI->getStubSize();
385       PointerTargetAddr += ABI->getPointerSize();
386     }
387 
388     IndirectStubAllocs.push_back(std::move(*Alloc));
389   }
390 
391   assert(NumStubs <= AvailableIndirectStubs.size() &&
392          "Sufficient stubs should have been allocated above");
393 
394   IndirectStubInfoVector Result;
395   while (NumStubs--) {
396     Result.push_back(AvailableIndirectStubs.back());
397     AvailableIndirectStubs.pop_back();
398   }
399 
400   return std::move(Result);
401 }
402 
reentry(JITTargetAddress LCTMAddr,JITTargetAddress TrampolineAddr)403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
404                                 JITTargetAddress TrampolineAddr) {
405   auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
406   std::promise<JITTargetAddress> LandingAddrP;
407   auto LandingAddrF = LandingAddrP.get_future();
408   LCTM.resolveTrampolineLandingAddress(
409       TrampolineAddr,
410       [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
411   return LandingAddrF.get();
412 }
413 
setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils & EPCIU)414 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
415   auto &LCTM = EPCIU.getLazyCallThroughManager();
416   return EPCIU
417       .writeResolverBlock(pointerToJITTargetAddress(&reentry),
418                           pointerToJITTargetAddress(&LCTM))
419       .takeError();
420 }
421 
422 } // end namespace orc
423 } // end namespace llvm
424