1 //===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"
10
11 #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"
12 #include "llvm/Support/MathExtras.h"
13
14 #include <future>
15
16 using namespace llvm;
17 using namespace llvm::orc;
18
19 namespace llvm {
20 namespace orc {
21
22 class EPCIndirectionUtilsAccess {
23 public:
24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26
27 static Expected<IndirectStubInfoVector>
getIndirectStubs(EPCIndirectionUtils & EPCIU,unsigned NumStubs)28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29 return EPCIU.getIndirectStubs(NumStubs);
30 };
31 };
32
33 } // end namespace orc
34 } // end namespace llvm
35
36 namespace {
37
38 class EPCTrampolinePool : public TrampolinePool {
39 public:
40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41 Error deallocatePool();
42
43 protected:
44 Error grow() override;
45
46 using Allocation = jitlink::JITLinkMemoryManager::Allocation;
47
48 EPCIndirectionUtils &EPCIU;
49 unsigned TrampolineSize = 0;
50 unsigned TrampolinesPerPage = 0;
51 std::vector<std::unique_ptr<Allocation>> TrampolineBlocks;
52 };
53
54 class EPCIndirectStubsManager : public IndirectStubsManager,
55 private EPCIndirectionUtilsAccess {
56 public:
EPCIndirectStubsManager(EPCIndirectionUtils & EPCIU)57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58
59 Error deallocateStubs();
60
61 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62 JITSymbolFlags StubFlags) override;
63
64 Error createStubs(const StubInitsMap &StubInits) override;
65
66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67
68 JITEvaluatedSymbol findPointer(StringRef Name) override;
69
70 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override;
71
72 private:
73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74
75 std::mutex ISMMutex;
76 EPCIndirectionUtils &EPCIU;
77 StringMap<StubInfo> StubInfos;
78 };
79
EPCTrampolinePool(EPCIndirectionUtils & EPCIU)80 EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81 : EPCIU(EPCIU) {
82 auto &EPC = EPCIU.getExecutorProcessControl();
83 auto &ABI = EPCIU.getABISupport();
84
85 TrampolineSize = ABI.getTrampolineSize();
86 TrampolinesPerPage =
87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88 }
89
deallocatePool()90 Error EPCTrampolinePool::deallocatePool() {
91 Error Err = Error::success();
92 for (auto &Alloc : TrampolineBlocks)
93 Err = joinErrors(std::move(Err), Alloc->deallocate());
94 return Err;
95 }
96
grow()97 Error EPCTrampolinePool::grow() {
98 assert(AvailableTrampolines.empty() &&
99 "Grow called with trampolines still available");
100
101 auto ResolverAddress = EPCIU.getResolverBlockAddress();
102 assert(ResolverAddress && "Resolver address can not be null");
103
104 auto &EPC = EPCIU.getExecutorProcessControl();
105 constexpr auto TrampolinePagePermissions =
106 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
107 sys::Memory::MF_EXEC);
108 auto PageSize = EPC.getPageSize();
109 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
110 Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
111 0};
112 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
113
114 if (!Alloc)
115 return Alloc.takeError();
116
117 unsigned NumTrampolines = TrampolinesPerPage;
118
119 auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions);
120 auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
121
122 EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress,
123 ResolverAddress, NumTrampolines);
124
125 auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions);
126 for (unsigned I = 0; I < NumTrampolines; ++I)
127 AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize));
128
129 if (auto Err = (*Alloc)->finalize())
130 return Err;
131
132 TrampolineBlocks.push_back(std::move(*Alloc));
133
134 return Error::success();
135 }
136
createStub(StringRef StubName,JITTargetAddress StubAddr,JITSymbolFlags StubFlags)137 Error EPCIndirectStubsManager::createStub(StringRef StubName,
138 JITTargetAddress StubAddr,
139 JITSymbolFlags StubFlags) {
140 StubInitsMap SIM;
141 SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142 return createStubs(SIM);
143 }
144
createStubs(const StubInitsMap & StubInits)145 Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
147 if (!AvailableStubInfos)
148 return AvailableStubInfos.takeError();
149
150 {
151 std::lock_guard<std::mutex> Lock(ISMMutex);
152 unsigned ASIdx = 0;
153 for (auto &SI : StubInits) {
154 auto &A = (*AvailableStubInfos)[ASIdx++];
155 StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156 }
157 }
158
159 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
160 switch (EPCIU.getABISupport().getPointerSize()) {
161 case 4: {
162 unsigned ASIdx = 0;
163 std::vector<tpctypes::UInt32Write> PtrUpdates;
164 for (auto &SI : StubInits)
165 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
166 static_cast<uint32_t>(SI.second.first)});
167 return MemAccess.writeUInt32s(PtrUpdates);
168 }
169 case 8: {
170 unsigned ASIdx = 0;
171 std::vector<tpctypes::UInt64Write> PtrUpdates;
172 for (auto &SI : StubInits)
173 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
174 static_cast<uint64_t>(SI.second.first)});
175 return MemAccess.writeUInt64s(PtrUpdates);
176 }
177 default:
178 return make_error<StringError>("Unsupported pointer size",
179 inconvertibleErrorCode());
180 }
181 }
182
findStub(StringRef Name,bool ExportedStubsOnly)183 JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
184 bool ExportedStubsOnly) {
185 std::lock_guard<std::mutex> Lock(ISMMutex);
186 auto I = StubInfos.find(Name);
187 if (I == StubInfos.end())
188 return nullptr;
189 return {I->second.first.StubAddress, I->second.second};
190 }
191
findPointer(StringRef Name)192 JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
193 std::lock_guard<std::mutex> Lock(ISMMutex);
194 auto I = StubInfos.find(Name);
195 if (I == StubInfos.end())
196 return nullptr;
197 return {I->second.first.PointerAddress, I->second.second};
198 }
199
updatePointer(StringRef Name,JITTargetAddress NewAddr)200 Error EPCIndirectStubsManager::updatePointer(StringRef Name,
201 JITTargetAddress NewAddr) {
202
203 JITTargetAddress PtrAddr = 0;
204 {
205 std::lock_guard<std::mutex> Lock(ISMMutex);
206 auto I = StubInfos.find(Name);
207 if (I == StubInfos.end())
208 return make_error<StringError>("Unknown stub name",
209 inconvertibleErrorCode());
210 PtrAddr = I->second.first.PointerAddress;
211 }
212
213 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
214 switch (EPCIU.getABISupport().getPointerSize()) {
215 case 4: {
216 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr);
217 return MemAccess.writeUInt32s(PUpdate);
218 }
219 case 8: {
220 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr);
221 return MemAccess.writeUInt64s(PUpdate);
222 }
223 default:
224 return make_error<StringError>("Unsupported pointer size",
225 inconvertibleErrorCode());
226 }
227 }
228
229 } // end anonymous namespace.
230
231 namespace llvm {
232 namespace orc {
233
~ABISupport()234 EPCIndirectionUtils::ABISupport::~ABISupport() {}
235
236 Expected<std::unique_ptr<EPCIndirectionUtils>>
Create(ExecutorProcessControl & EPC)237 EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {
238 const auto &TT = EPC.getTargetTriple();
239 switch (TT.getArch()) {
240 default:
241 return make_error<StringError>(
242 std::string("No EPCIndirectionUtils available for ") + TT.str(),
243 inconvertibleErrorCode());
244 case Triple::aarch64:
245 case Triple::aarch64_32:
246 return CreateWithABI<OrcAArch64>(EPC);
247
248 case Triple::x86:
249 return CreateWithABI<OrcI386>(EPC);
250
251 case Triple::mips:
252 return CreateWithABI<OrcMips32Be>(EPC);
253
254 case Triple::mipsel:
255 return CreateWithABI<OrcMips32Le>(EPC);
256
257 case Triple::mips64:
258 case Triple::mips64el:
259 return CreateWithABI<OrcMips64>(EPC);
260
261 case Triple::x86_64:
262 if (TT.getOS() == Triple::OSType::Win32)
263 return CreateWithABI<OrcX86_64_Win32>(EPC);
264 else
265 return CreateWithABI<OrcX86_64_SysV>(EPC);
266 }
267 }
268
cleanup()269 Error EPCIndirectionUtils::cleanup() {
270 Error Err = Error::success();
271
272 for (auto &A : IndirectStubAllocs)
273 Err = joinErrors(std::move(Err), A->deallocate());
274
275 if (TP)
276 Err = joinErrors(std::move(Err),
277 static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
278
279 if (ResolverBlock)
280 Err = joinErrors(std::move(Err), ResolverBlock->deallocate());
281
282 return Err;
283 }
284
285 Expected<JITTargetAddress>
writeResolverBlock(JITTargetAddress ReentryFnAddr,JITTargetAddress ReentryCtxAddr)286 EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
287 JITTargetAddress ReentryCtxAddr) {
288 assert(ABI && "ABI can not be null");
289 constexpr auto ResolverBlockPermissions =
290 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
291 sys::Memory::MF_EXEC);
292 auto ResolverSize = ABI->getResolverCodeSize();
293
294 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
295 Request[ResolverBlockPermissions] = {EPC.getPageSize(),
296 static_cast<size_t>(ResolverSize), 0};
297 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
298 if (!Alloc)
299 return Alloc.takeError();
300
301 auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions);
302 ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions);
303 ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr,
304 ReentryCtxAddr);
305
306 if (auto Err = (*Alloc)->finalize())
307 return std::move(Err);
308
309 ResolverBlock = std::move(*Alloc);
310 return ResolverBlockAddr;
311 }
312
313 std::unique_ptr<IndirectStubsManager>
createIndirectStubsManager()314 EPCIndirectionUtils::createIndirectStubsManager() {
315 return std::make_unique<EPCIndirectStubsManager>(*this);
316 }
317
getTrampolinePool()318 TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {
319 if (!TP)
320 TP = std::make_unique<EPCTrampolinePool>(*this);
321 return *TP;
322 }
323
createLazyCallThroughManager(ExecutionSession & ES,JITTargetAddress ErrorHandlerAddr)324 LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(
325 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
326 assert(!LCTM &&
327 "createLazyCallThroughManager can not have been called before");
328 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
329 &getTrampolinePool());
330 return *LCTM;
331 }
332
EPCIndirectionUtils(ExecutorProcessControl & EPC,std::unique_ptr<ABISupport> ABI)333 EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
334 std::unique_ptr<ABISupport> ABI)
335 : EPC(EPC), ABI(std::move(ABI)) {
336 assert(this->ABI && "ABI can not be null");
337
338 assert(EPC.getPageSize() > getABISupport().getStubSize() &&
339 "Stubs larger than one page are not supported");
340 }
341
342 Expected<EPCIndirectionUtils::IndirectStubInfoVector>
getIndirectStubs(unsigned NumStubs)343 EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
344
345 std::lock_guard<std::mutex> Lock(EPCUIMutex);
346
347 // If there aren't enough stubs available then allocate some more.
348 if (NumStubs > AvailableIndirectStubs.size()) {
349 auto NumStubsToAllocate = NumStubs;
350 auto PageSize = EPC.getPageSize();
351 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
352 NumStubsToAllocate = StubBytes / ABI->getStubSize();
353 auto PointerBytes =
354 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
355
356 constexpr auto StubPagePermissions =
357 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
358 sys::Memory::MF_EXEC);
359 constexpr auto PointerPagePermissions =
360 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
361 sys::Memory::MF_WRITE);
362
363 jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
364 Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
365 0};
366 Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
367 auto Alloc = EPC.getMemMgr().allocate(nullptr, Request);
368 if (!Alloc)
369 return Alloc.takeError();
370
371 auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions);
372 auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions);
373
374 ABI->writeIndirectStubsBlock(
375 (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr,
376 PointerTargetAddr, NumStubsToAllocate);
377
378 if (auto Err = (*Alloc)->finalize())
379 return std::move(Err);
380
381 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
382 AvailableIndirectStubs.push_back(
383 IndirectStubInfo(StubTargetAddr, PointerTargetAddr));
384 StubTargetAddr += ABI->getStubSize();
385 PointerTargetAddr += ABI->getPointerSize();
386 }
387
388 IndirectStubAllocs.push_back(std::move(*Alloc));
389 }
390
391 assert(NumStubs <= AvailableIndirectStubs.size() &&
392 "Sufficient stubs should have been allocated above");
393
394 IndirectStubInfoVector Result;
395 while (NumStubs--) {
396 Result.push_back(AvailableIndirectStubs.back());
397 AvailableIndirectStubs.pop_back();
398 }
399
400 return std::move(Result);
401 }
402
reentry(JITTargetAddress LCTMAddr,JITTargetAddress TrampolineAddr)403 static JITTargetAddress reentry(JITTargetAddress LCTMAddr,
404 JITTargetAddress TrampolineAddr) {
405 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
406 std::promise<JITTargetAddress> LandingAddrP;
407 auto LandingAddrF = LandingAddrP.get_future();
408 LCTM.resolveTrampolineLandingAddress(
409 TrampolineAddr,
410 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
411 return LandingAddrF.get();
412 }
413
setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils & EPCIU)414 Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {
415 auto &LCTM = EPCIU.getLazyCallThroughManager();
416 return EPCIU
417 .writeResolverBlock(pointerToJITTargetAddress(&reentry),
418 pointerToJITTargetAddress(&LCTM))
419 .takeError();
420 }
421
422 } // end namespace orc
423 } // end namespace llvm
424