1 //===- OrcRemoteTargetClient.h - Orc Remote-target Client -------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines the OrcRemoteTargetClient class and helpers. This class 10 // can be used to communicate over an RawByteChannel with an 11 // OrcRemoteTargetServer instance to support remote-JITing. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETCLIENT_H 16 #define LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETCLIENT_H 17 18 #include "llvm/ADT/Optional.h" 19 #include "llvm/ADT/STLExtras.h" 20 #include "llvm/ADT/StringMap.h" 21 #include "llvm/ADT/StringRef.h" 22 #include "llvm/ExecutionEngine/JITSymbol.h" 23 #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" 24 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" 25 #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" 26 #include "llvm/ExecutionEngine/RuntimeDyld.h" 27 #include "llvm/Support/Debug.h" 28 #include "llvm/Support/Error.h" 29 #include "llvm/Support/ErrorHandling.h" 30 #include "llvm/Support/Format.h" 31 #include "llvm/Support/MathExtras.h" 32 #include "llvm/Support/Memory.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <algorithm> 35 #include <cassert> 36 #include <cstdint> 37 #include <memory> 38 #include <string> 39 #include <tuple> 40 #include <utility> 41 #include <vector> 42 43 #define DEBUG_TYPE "orc-remote" 44 45 namespace llvm { 46 namespace orc { 47 namespace remote { 48 49 /// This class provides utilities (including memory manager, indirect stubs 50 /// manager, and compile callback manager types) that support remote JITing 51 /// in ORC. 52 /// 53 /// Each of the utility classes talks to a JIT server (an instance of the 54 /// OrcRemoteTargetServer class) via an RPC system (see RPCUtils.h) to carry out 55 /// its actions. 56 class OrcRemoteTargetClient 57 : public shared::SingleThreadedRPCEndpoint<shared::RawByteChannel> { 58 public: 59 /// Remote-mapped RuntimeDyld-compatible memory manager. 60 class RemoteRTDyldMemoryManager : public RuntimeDyld::MemoryManager { 61 friend class OrcRemoteTargetClient; 62 63 public: ~RemoteRTDyldMemoryManager()64 ~RemoteRTDyldMemoryManager() { 65 Client.destroyRemoteAllocator(Id); 66 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); 67 } 68 69 RemoteRTDyldMemoryManager(const RemoteRTDyldMemoryManager &) = delete; 70 RemoteRTDyldMemoryManager & 71 operator=(const RemoteRTDyldMemoryManager &) = delete; 72 RemoteRTDyldMemoryManager(RemoteRTDyldMemoryManager &&) = default; 73 RemoteRTDyldMemoryManager &operator=(RemoteRTDyldMemoryManager &&) = delete; 74 allocateCodeSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName)75 uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, 76 unsigned SectionID, 77 StringRef SectionName) override { 78 Unmapped.back().CodeAllocs.emplace_back(Size, Alignment); 79 uint8_t *Alloc = reinterpret_cast<uint8_t *>( 80 Unmapped.back().CodeAllocs.back().getLocalAddress()); 81 LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated code for " 82 << SectionName << ": " << Alloc << " (" << Size 83 << " bytes, alignment " << Alignment << ")\n"); 84 return Alloc; 85 } 86 allocateDataSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName,bool IsReadOnly)87 uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, 88 unsigned SectionID, StringRef SectionName, 89 bool IsReadOnly) override { 90 if (IsReadOnly) { 91 Unmapped.back().RODataAllocs.emplace_back(Size, Alignment); 92 uint8_t *Alloc = reinterpret_cast<uint8_t *>( 93 Unmapped.back().RODataAllocs.back().getLocalAddress()); 94 LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated ro-data for " 95 << SectionName << ": " << Alloc << " (" << Size 96 << " bytes, alignment " << Alignment << ")\n"); 97 return Alloc; 98 } // else... 99 100 Unmapped.back().RWDataAllocs.emplace_back(Size, Alignment); 101 uint8_t *Alloc = reinterpret_cast<uint8_t *>( 102 Unmapped.back().RWDataAllocs.back().getLocalAddress()); 103 LLVM_DEBUG(dbgs() << "Allocator " << Id << " allocated rw-data for " 104 << SectionName << ": " << Alloc << " (" << Size 105 << " bytes, alignment " << Alignment << ")\n"); 106 return Alloc; 107 } 108 reserveAllocationSpace(uintptr_t CodeSize,uint32_t CodeAlign,uintptr_t RODataSize,uint32_t RODataAlign,uintptr_t RWDataSize,uint32_t RWDataAlign)109 void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, 110 uintptr_t RODataSize, uint32_t RODataAlign, 111 uintptr_t RWDataSize, 112 uint32_t RWDataAlign) override { 113 Unmapped.push_back(ObjectAllocs()); 114 115 LLVM_DEBUG(dbgs() << "Allocator " << Id << " reserved:\n"); 116 117 if (CodeSize != 0) { 118 Unmapped.back().RemoteCodeAddr = 119 Client.reserveMem(Id, CodeSize, CodeAlign); 120 121 LLVM_DEBUG( 122 dbgs() << " code: " 123 << format("0x%016" PRIx64, Unmapped.back().RemoteCodeAddr) 124 << " (" << CodeSize << " bytes, alignment " << CodeAlign 125 << ")\n"); 126 } 127 128 if (RODataSize != 0) { 129 Unmapped.back().RemoteRODataAddr = 130 Client.reserveMem(Id, RODataSize, RODataAlign); 131 132 LLVM_DEBUG( 133 dbgs() << " ro-data: " 134 << format("0x%016" PRIx64, Unmapped.back().RemoteRODataAddr) 135 << " (" << RODataSize << " bytes, alignment " << RODataAlign 136 << ")\n"); 137 } 138 139 if (RWDataSize != 0) { 140 Unmapped.back().RemoteRWDataAddr = 141 Client.reserveMem(Id, RWDataSize, RWDataAlign); 142 143 LLVM_DEBUG( 144 dbgs() << " rw-data: " 145 << format("0x%016" PRIx64, Unmapped.back().RemoteRWDataAddr) 146 << " (" << RWDataSize << " bytes, alignment " << RWDataAlign 147 << ")\n"); 148 } 149 } 150 needsToReserveAllocationSpace()151 bool needsToReserveAllocationSpace() override { return true; } 152 registerEHFrames(uint8_t * Addr,uint64_t LoadAddr,size_t Size)153 void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, 154 size_t Size) override { 155 UnfinalizedEHFrames.push_back({LoadAddr, Size}); 156 } 157 deregisterEHFrames()158 void deregisterEHFrames() override { 159 for (auto &Frame : RegisteredEHFrames) { 160 // FIXME: Add error poll. 161 Client.deregisterEHFrames(Frame.Addr, Frame.Size); 162 } 163 } 164 notifyObjectLoaded(RuntimeDyld & Dyld,const object::ObjectFile & Obj)165 void notifyObjectLoaded(RuntimeDyld &Dyld, 166 const object::ObjectFile &Obj) override { 167 LLVM_DEBUG(dbgs() << "Allocator " << Id << " applied mappings:\n"); 168 for (auto &ObjAllocs : Unmapped) { 169 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, 170 ObjAllocs.RemoteCodeAddr); 171 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs, 172 ObjAllocs.RemoteRODataAddr); 173 mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs, 174 ObjAllocs.RemoteRWDataAddr); 175 Unfinalized.push_back(std::move(ObjAllocs)); 176 } 177 Unmapped.clear(); 178 } 179 180 bool finalizeMemory(std::string *ErrMsg = nullptr) override { 181 LLVM_DEBUG(dbgs() << "Allocator " << Id << " finalizing:\n"); 182 183 for (auto &ObjAllocs : Unfinalized) { 184 if (copyAndProtect(ObjAllocs.CodeAllocs, ObjAllocs.RemoteCodeAddr, 185 sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 186 return true; 187 188 if (copyAndProtect(ObjAllocs.RODataAllocs, ObjAllocs.RemoteRODataAddr, 189 sys::Memory::MF_READ)) 190 return true; 191 192 if (copyAndProtect(ObjAllocs.RWDataAllocs, ObjAllocs.RemoteRWDataAddr, 193 sys::Memory::MF_READ | sys::Memory::MF_WRITE)) 194 return true; 195 } 196 Unfinalized.clear(); 197 198 for (auto &EHFrame : UnfinalizedEHFrames) { 199 if (auto Err = Client.registerEHFrames(EHFrame.Addr, EHFrame.Size)) { 200 // FIXME: Replace this once finalizeMemory can return an Error. 201 handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { 202 if (ErrMsg) { 203 raw_string_ostream ErrOut(*ErrMsg); 204 EIB.log(ErrOut); 205 } 206 }); 207 return false; 208 } 209 } 210 RegisteredEHFrames = std::move(UnfinalizedEHFrames); 211 UnfinalizedEHFrames = {}; 212 213 return false; 214 } 215 216 private: 217 class Alloc { 218 public: Alloc(uint64_t Size,unsigned Align)219 Alloc(uint64_t Size, unsigned Align) 220 : Size(Size), Align(Align), Contents(new char[Size + Align - 1]) {} 221 222 Alloc(const Alloc &) = delete; 223 Alloc &operator=(const Alloc &) = delete; 224 Alloc(Alloc &&) = default; 225 Alloc &operator=(Alloc &&) = default; 226 getSize()227 uint64_t getSize() const { return Size; } 228 getAlign()229 unsigned getAlign() const { return Align; } 230 getLocalAddress()231 char *getLocalAddress() const { 232 uintptr_t LocalAddr = reinterpret_cast<uintptr_t>(Contents.get()); 233 LocalAddr = alignTo(LocalAddr, Align); 234 return reinterpret_cast<char *>(LocalAddr); 235 } 236 setRemoteAddress(JITTargetAddress RemoteAddr)237 void setRemoteAddress(JITTargetAddress RemoteAddr) { 238 this->RemoteAddr = RemoteAddr; 239 } 240 getRemoteAddress()241 JITTargetAddress getRemoteAddress() const { return RemoteAddr; } 242 243 private: 244 uint64_t Size; 245 unsigned Align; 246 std::unique_ptr<char[]> Contents; 247 JITTargetAddress RemoteAddr = 0; 248 }; 249 250 struct ObjectAllocs { 251 ObjectAllocs() = default; 252 ObjectAllocs(const ObjectAllocs &) = delete; 253 ObjectAllocs &operator=(const ObjectAllocs &) = delete; 254 ObjectAllocs(ObjectAllocs &&) = default; 255 ObjectAllocs &operator=(ObjectAllocs &&) = default; 256 257 JITTargetAddress RemoteCodeAddr = 0; 258 JITTargetAddress RemoteRODataAddr = 0; 259 JITTargetAddress RemoteRWDataAddr = 0; 260 std::vector<Alloc> CodeAllocs, RODataAllocs, RWDataAllocs; 261 }; 262 RemoteRTDyldMemoryManager(OrcRemoteTargetClient & Client,ResourceIdMgr::ResourceId Id)263 RemoteRTDyldMemoryManager(OrcRemoteTargetClient &Client, 264 ResourceIdMgr::ResourceId Id) 265 : Client(Client), Id(Id) { 266 LLVM_DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); 267 } 268 269 // Maps all allocations in Allocs to aligned blocks mapAllocsToRemoteAddrs(RuntimeDyld & Dyld,std::vector<Alloc> & Allocs,JITTargetAddress NextAddr)270 void mapAllocsToRemoteAddrs(RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, 271 JITTargetAddress NextAddr) { 272 for (auto &Alloc : Allocs) { 273 NextAddr = alignTo(NextAddr, Alloc.getAlign()); 274 Dyld.mapSectionAddress(Alloc.getLocalAddress(), NextAddr); 275 LLVM_DEBUG( 276 dbgs() << " " << static_cast<void *>(Alloc.getLocalAddress()) 277 << " -> " << format("0x%016" PRIx64, NextAddr) << "\n"); 278 Alloc.setRemoteAddress(NextAddr); 279 280 // Only advance NextAddr if it was non-null to begin with, 281 // otherwise leave it as null. 282 if (NextAddr) 283 NextAddr += Alloc.getSize(); 284 } 285 } 286 287 // Copies data for each alloc in the list, then set permissions on the 288 // segment. copyAndProtect(const std::vector<Alloc> & Allocs,JITTargetAddress RemoteSegmentAddr,unsigned Permissions)289 bool copyAndProtect(const std::vector<Alloc> &Allocs, 290 JITTargetAddress RemoteSegmentAddr, 291 unsigned Permissions) { 292 if (RemoteSegmentAddr) { 293 assert(!Allocs.empty() && "No sections in allocated segment"); 294 295 for (auto &Alloc : Allocs) { 296 LLVM_DEBUG(dbgs() << " copying section: " 297 << static_cast<void *>(Alloc.getLocalAddress()) 298 << " -> " 299 << format("0x%016" PRIx64, Alloc.getRemoteAddress()) 300 << " (" << Alloc.getSize() << " bytes)\n";); 301 302 if (Client.writeMem(Alloc.getRemoteAddress(), Alloc.getLocalAddress(), 303 Alloc.getSize())) 304 return true; 305 } 306 307 LLVM_DEBUG(dbgs() << " setting " 308 << (Permissions & sys::Memory::MF_READ ? 'R' : '-') 309 << (Permissions & sys::Memory::MF_WRITE ? 'W' : '-') 310 << (Permissions & sys::Memory::MF_EXEC ? 'X' : '-') 311 << " permissions on block: " 312 << format("0x%016" PRIx64, RemoteSegmentAddr) 313 << "\n"); 314 if (Client.setProtections(Id, RemoteSegmentAddr, Permissions)) 315 return true; 316 } 317 return false; 318 } 319 320 OrcRemoteTargetClient &Client; 321 ResourceIdMgr::ResourceId Id; 322 std::vector<ObjectAllocs> Unmapped; 323 std::vector<ObjectAllocs> Unfinalized; 324 325 struct EHFrame { 326 JITTargetAddress Addr; 327 uint64_t Size; 328 }; 329 std::vector<EHFrame> UnfinalizedEHFrames; 330 std::vector<EHFrame> RegisteredEHFrames; 331 }; 332 333 class RPCMMAlloc : public jitlink::JITLinkMemoryManager::Allocation { 334 using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>; 335 using FinalizeContinuation = 336 jitlink::JITLinkMemoryManager::Allocation::FinalizeContinuation; 337 using ProtectionFlags = sys::Memory::ProtectionFlags; 338 using SegmentsRequestMap = 339 DenseMap<unsigned, jitlink::JITLinkMemoryManager::SegmentRequest>; 340 RPCMMAlloc(OrcRemoteTargetClient & Client,ResourceIdMgr::ResourceId Id)341 RPCMMAlloc(OrcRemoteTargetClient &Client, ResourceIdMgr::ResourceId Id) 342 : Client(Client), Id(Id) {} 343 344 public: 345 static Expected<std::unique_ptr<RPCMMAlloc>> Create(OrcRemoteTargetClient & Client,ResourceIdMgr::ResourceId Id,const SegmentsRequestMap & Request)346 Create(OrcRemoteTargetClient &Client, ResourceIdMgr::ResourceId Id, 347 const SegmentsRequestMap &Request) { 348 auto *MM = new RPCMMAlloc(Client, Id); 349 350 if (Error Err = MM->allocateHostBlocks(Request)) 351 return std::move(Err); 352 353 if (Error Err = MM->allocateTargetBlocks()) 354 return std::move(Err); 355 356 return std::unique_ptr<RPCMMAlloc>(MM); 357 } 358 getWorkingMemory(ProtectionFlags Seg)359 MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override { 360 assert(HostSegBlocks.count(Seg) && "No allocation for segment"); 361 return {static_cast<char *>(HostSegBlocks[Seg].base()), 362 HostSegBlocks[Seg].allocatedSize()}; 363 } 364 getTargetMemory(ProtectionFlags Seg)365 JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { 366 assert(TargetSegBlocks.count(Seg) && "No allocation for segment"); 367 return pointerToJITTargetAddress(TargetSegBlocks[Seg].base()); 368 } 369 finalizeAsync(FinalizeContinuation OnFinalize)370 void finalizeAsync(FinalizeContinuation OnFinalize) override { 371 // Host allocations (working memory) remain ReadWrite. 372 OnFinalize(copyAndProtect()); 373 } 374 deallocate()375 Error deallocate() override { 376 // TODO: Cannot release target allocation. RPCAPI has no function 377 // symmetric to reserveMem(). Add RPC call like freeMem()? 378 return errorCodeToError(sys::Memory::releaseMappedMemory(HostAllocation)); 379 } 380 381 private: 382 OrcRemoteTargetClient &Client; 383 ResourceIdMgr::ResourceId Id; 384 AllocationMap HostSegBlocks; 385 AllocationMap TargetSegBlocks; 386 JITTargetAddress TargetSegmentAddr; 387 sys::MemoryBlock HostAllocation; 388 allocateHostBlocks(const SegmentsRequestMap & Request)389 Error allocateHostBlocks(const SegmentsRequestMap &Request) { 390 unsigned TargetPageSize = Client.getPageSize(); 391 392 if (!isPowerOf2_64(static_cast<uint64_t>(TargetPageSize))) 393 return make_error<StringError>("Host page size is not a power of 2", 394 inconvertibleErrorCode()); 395 396 auto TotalSize = calcTotalAllocSize(Request, TargetPageSize); 397 if (!TotalSize) 398 return TotalSize.takeError(); 399 400 // Allocate one slab to cover all the segments. 401 const sys::Memory::ProtectionFlags ReadWrite = 402 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | 403 sys::Memory::MF_WRITE); 404 std::error_code EC; 405 HostAllocation = 406 sys::Memory::allocateMappedMemory(*TotalSize, nullptr, ReadWrite, EC); 407 if (EC) 408 return errorCodeToError(EC); 409 410 char *SlabAddr = static_cast<char *>(HostAllocation.base()); 411 #ifndef NDEBUG 412 char *SlabAddrEnd = SlabAddr + HostAllocation.allocatedSize(); 413 #endif 414 415 // Allocate segment memory from the slab. 416 for (auto &KV : Request) { 417 const auto &Seg = KV.second; 418 419 uint64_t SegmentSize = Seg.getContentSize() + Seg.getZeroFillSize(); 420 uint64_t AlignedSegmentSize = alignTo(SegmentSize, TargetPageSize); 421 422 // Zero out zero-fill memory. 423 char *ZeroFillBegin = SlabAddr + Seg.getContentSize(); 424 memset(ZeroFillBegin, 0, Seg.getZeroFillSize()); 425 426 // Record the block for this segment. 427 HostSegBlocks[KV.first] = 428 sys::MemoryBlock(SlabAddr, AlignedSegmentSize); 429 430 SlabAddr += AlignedSegmentSize; 431 assert(SlabAddr <= SlabAddrEnd && "Out of range"); 432 } 433 434 return Error::success(); 435 } 436 allocateTargetBlocks()437 Error allocateTargetBlocks() { 438 // Reserve memory for all blocks on the target. We need as much space on 439 // the target as we allocated on the host. 440 TargetSegmentAddr = Client.reserveMem(Id, HostAllocation.allocatedSize(), 441 Client.getPageSize()); 442 if (!TargetSegmentAddr) 443 return make_error<StringError>("Failed to reserve memory on the target", 444 inconvertibleErrorCode()); 445 446 // Map memory blocks into the allocation, that match the host allocation. 447 JITTargetAddress TargetAllocAddr = TargetSegmentAddr; 448 for (const auto &KV : HostSegBlocks) { 449 size_t TargetAllocSize = KV.second.allocatedSize(); 450 451 TargetSegBlocks[KV.first] = 452 sys::MemoryBlock(jitTargetAddressToPointer<void *>(TargetAllocAddr), 453 TargetAllocSize); 454 455 TargetAllocAddr += TargetAllocSize; 456 assert(TargetAllocAddr - TargetSegmentAddr <= 457 HostAllocation.allocatedSize() && 458 "Out of range on target"); 459 } 460 461 return Error::success(); 462 } 463 copyAndProtect()464 Error copyAndProtect() { 465 unsigned Permissions = 0u; 466 467 // Copy segments one by one. 468 for (auto &KV : TargetSegBlocks) { 469 Permissions |= KV.first; 470 471 const sys::MemoryBlock &TargetBlock = KV.second; 472 const sys::MemoryBlock &HostBlock = HostSegBlocks.lookup(KV.first); 473 474 size_t TargetAllocSize = TargetBlock.allocatedSize(); 475 auto TargetAllocAddr = pointerToJITTargetAddress(TargetBlock.base()); 476 auto *HostAllocBegin = static_cast<const char *>(HostBlock.base()); 477 478 bool CopyErr = 479 Client.writeMem(TargetAllocAddr, HostAllocBegin, TargetAllocSize); 480 if (CopyErr) 481 return createStringError(inconvertibleErrorCode(), 482 "Failed to copy %d segment to the target", 483 KV.first); 484 } 485 486 // Set permission flags for all segments at once. 487 bool ProtectErr = 488 Client.setProtections(Id, TargetSegmentAddr, Permissions); 489 if (ProtectErr) 490 return createStringError(inconvertibleErrorCode(), 491 "Failed to apply permissions for %d segment " 492 "on the target", 493 Permissions); 494 return Error::success(); 495 } 496 497 static Expected<size_t> calcTotalAllocSize(const SegmentsRequestMap & Request,unsigned TargetPageSize)498 calcTotalAllocSize(const SegmentsRequestMap &Request, 499 unsigned TargetPageSize) { 500 size_t TotalSize = 0; 501 for (const auto &KV : Request) { 502 const auto &Seg = KV.second; 503 504 if (Seg.getAlignment() > TargetPageSize) 505 return make_error<StringError>("Cannot request alignment higher than " 506 "page alignment on target", 507 inconvertibleErrorCode()); 508 509 TotalSize = alignTo(TotalSize, TargetPageSize); 510 TotalSize += Seg.getContentSize(); 511 TotalSize += Seg.getZeroFillSize(); 512 } 513 514 return TotalSize; 515 } 516 }; 517 518 class RemoteJITLinkMemoryManager : public jitlink::JITLinkMemoryManager { 519 public: RemoteJITLinkMemoryManager(OrcRemoteTargetClient & Client,ResourceIdMgr::ResourceId Id)520 RemoteJITLinkMemoryManager(OrcRemoteTargetClient &Client, 521 ResourceIdMgr::ResourceId Id) 522 : Client(Client), Id(Id) {} 523 524 RemoteJITLinkMemoryManager(const RemoteJITLinkMemoryManager &) = delete; 525 RemoteJITLinkMemoryManager(RemoteJITLinkMemoryManager &&) = default; 526 527 RemoteJITLinkMemoryManager & 528 operator=(const RemoteJITLinkMemoryManager &) = delete; 529 RemoteJITLinkMemoryManager & 530 operator=(RemoteJITLinkMemoryManager &&) = delete; 531 ~RemoteJITLinkMemoryManager()532 ~RemoteJITLinkMemoryManager() { 533 Client.destroyRemoteAllocator(Id); 534 LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << Id << "\n"); 535 } 536 537 Expected<std::unique_ptr<Allocation>> allocate(const jitlink::JITLinkDylib * JD,const SegmentsRequestMap & Request)538 allocate(const jitlink::JITLinkDylib *JD, 539 const SegmentsRequestMap &Request) override { 540 return RPCMMAlloc::Create(Client, Id, Request); 541 } 542 543 private: 544 OrcRemoteTargetClient &Client; 545 ResourceIdMgr::ResourceId Id; 546 }; 547 548 /// Remote indirect stubs manager. 549 class RemoteIndirectStubsManager : public IndirectStubsManager { 550 public: RemoteIndirectStubsManager(OrcRemoteTargetClient & Client,ResourceIdMgr::ResourceId Id)551 RemoteIndirectStubsManager(OrcRemoteTargetClient &Client, 552 ResourceIdMgr::ResourceId Id) 553 : Client(Client), Id(Id) {} 554 ~RemoteIndirectStubsManager()555 ~RemoteIndirectStubsManager() override { 556 Client.destroyIndirectStubsManager(Id); 557 } 558 createStub(StringRef StubName,JITTargetAddress StubAddr,JITSymbolFlags StubFlags)559 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 560 JITSymbolFlags StubFlags) override { 561 if (auto Err = reserveStubs(1)) 562 return Err; 563 564 return createStubInternal(StubName, StubAddr, StubFlags); 565 } 566 createStubs(const StubInitsMap & StubInits)567 Error createStubs(const StubInitsMap &StubInits) override { 568 if (auto Err = reserveStubs(StubInits.size())) 569 return Err; 570 571 for (auto &Entry : StubInits) 572 if (auto Err = createStubInternal(Entry.first(), Entry.second.first, 573 Entry.second.second)) 574 return Err; 575 576 return Error::success(); 577 } 578 findStub(StringRef Name,bool ExportedStubsOnly)579 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { 580 auto I = StubIndexes.find(Name); 581 if (I == StubIndexes.end()) 582 return nullptr; 583 auto Key = I->second.first; 584 auto Flags = I->second.second; 585 auto StubSymbol = JITEvaluatedSymbol(getStubAddr(Key), Flags); 586 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) 587 return nullptr; 588 return StubSymbol; 589 } 590 findPointer(StringRef Name)591 JITEvaluatedSymbol findPointer(StringRef Name) override { 592 auto I = StubIndexes.find(Name); 593 if (I == StubIndexes.end()) 594 return nullptr; 595 auto Key = I->second.first; 596 auto Flags = I->second.second; 597 return JITEvaluatedSymbol(getPtrAddr(Key), Flags); 598 } 599 updatePointer(StringRef Name,JITTargetAddress NewAddr)600 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { 601 auto I = StubIndexes.find(Name); 602 assert(I != StubIndexes.end() && "No stub pointer for symbol"); 603 auto Key = I->second.first; 604 return Client.writePointer(getPtrAddr(Key), NewAddr); 605 } 606 607 private: 608 struct RemoteIndirectStubsInfo { 609 JITTargetAddress StubBase; 610 JITTargetAddress PtrBase; 611 unsigned NumStubs; 612 }; 613 614 using StubKey = std::pair<uint16_t, uint16_t>; 615 reserveStubs(unsigned NumStubs)616 Error reserveStubs(unsigned NumStubs) { 617 if (NumStubs <= FreeStubs.size()) 618 return Error::success(); 619 620 unsigned NewStubsRequired = NumStubs - FreeStubs.size(); 621 JITTargetAddress StubBase; 622 JITTargetAddress PtrBase; 623 unsigned NumStubsEmitted; 624 625 if (auto StubInfoOrErr = Client.emitIndirectStubs(Id, NewStubsRequired)) 626 std::tie(StubBase, PtrBase, NumStubsEmitted) = *StubInfoOrErr; 627 else 628 return StubInfoOrErr.takeError(); 629 630 unsigned NewBlockId = RemoteIndirectStubsInfos.size(); 631 RemoteIndirectStubsInfos.push_back({StubBase, PtrBase, NumStubsEmitted}); 632 633 for (unsigned I = 0; I < NumStubsEmitted; ++I) 634 FreeStubs.push_back(std::make_pair(NewBlockId, I)); 635 636 return Error::success(); 637 } 638 createStubInternal(StringRef StubName,JITTargetAddress InitAddr,JITSymbolFlags StubFlags)639 Error createStubInternal(StringRef StubName, JITTargetAddress InitAddr, 640 JITSymbolFlags StubFlags) { 641 auto Key = FreeStubs.back(); 642 FreeStubs.pop_back(); 643 StubIndexes[StubName] = std::make_pair(Key, StubFlags); 644 return Client.writePointer(getPtrAddr(Key), InitAddr); 645 } 646 getStubAddr(StubKey K)647 JITTargetAddress getStubAddr(StubKey K) { 648 assert(RemoteIndirectStubsInfos[K.first].StubBase != 0 && 649 "Missing stub address"); 650 return RemoteIndirectStubsInfos[K.first].StubBase + 651 K.second * Client.getIndirectStubSize(); 652 } 653 getPtrAddr(StubKey K)654 JITTargetAddress getPtrAddr(StubKey K) { 655 assert(RemoteIndirectStubsInfos[K.first].PtrBase != 0 && 656 "Missing pointer address"); 657 return RemoteIndirectStubsInfos[K.first].PtrBase + 658 K.second * Client.getPointerSize(); 659 } 660 661 OrcRemoteTargetClient &Client; 662 ResourceIdMgr::ResourceId Id; 663 std::vector<RemoteIndirectStubsInfo> RemoteIndirectStubsInfos; 664 std::vector<StubKey> FreeStubs; 665 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; 666 }; 667 668 class RemoteTrampolinePool : public TrampolinePool { 669 public: RemoteTrampolinePool(OrcRemoteTargetClient & Client)670 RemoteTrampolinePool(OrcRemoteTargetClient &Client) : Client(Client) {} 671 672 private: grow()673 Error grow() override { 674 JITTargetAddress BlockAddr = 0; 675 uint32_t NumTrampolines = 0; 676 if (auto TrampolineInfoOrErr = Client.emitTrampolineBlock()) 677 std::tie(BlockAddr, NumTrampolines) = *TrampolineInfoOrErr; 678 else 679 return TrampolineInfoOrErr.takeError(); 680 681 uint32_t TrampolineSize = Client.getTrampolineSize(); 682 for (unsigned I = 0; I < NumTrampolines; ++I) 683 AvailableTrampolines.push_back(BlockAddr + (I * TrampolineSize)); 684 685 return Error::success(); 686 } 687 688 OrcRemoteTargetClient &Client; 689 }; 690 691 /// Remote compile callback manager. 692 class RemoteCompileCallbackManager : public JITCompileCallbackManager { 693 public: RemoteCompileCallbackManager(OrcRemoteTargetClient & Client,ExecutionSession & ES,JITTargetAddress ErrorHandlerAddress)694 RemoteCompileCallbackManager(OrcRemoteTargetClient &Client, 695 ExecutionSession &ES, 696 JITTargetAddress ErrorHandlerAddress) 697 : JITCompileCallbackManager( 698 std::make_unique<RemoteTrampolinePool>(Client), ES, 699 ErrorHandlerAddress) {} 700 }; 701 702 /// Create an OrcRemoteTargetClient. 703 /// Channel is the ChannelT instance to communicate on. It is assumed that 704 /// the channel is ready to be read from and written to. 705 static Expected<std::unique_ptr<OrcRemoteTargetClient>> Create(shared::RawByteChannel & Channel,ExecutionSession & ES)706 Create(shared::RawByteChannel &Channel, ExecutionSession &ES) { 707 Error Err = Error::success(); 708 auto Client = std::unique_ptr<OrcRemoteTargetClient>( 709 new OrcRemoteTargetClient(Channel, ES, Err)); 710 if (Err) 711 return std::move(Err); 712 return std::move(Client); 713 } 714 715 /// Call the int(void) function at the given address in the target and return 716 /// its result. callIntVoid(JITTargetAddress Addr)717 Expected<int> callIntVoid(JITTargetAddress Addr) { 718 LLVM_DEBUG(dbgs() << "Calling int(*)(void) " 719 << format("0x%016" PRIx64, Addr) << "\n"); 720 return callB<exec::CallIntVoid>(Addr); 721 } 722 723 /// Call the int(int) function at the given address in the target and return 724 /// its result. callIntInt(JITTargetAddress Addr,int Arg)725 Expected<int> callIntInt(JITTargetAddress Addr, int Arg) { 726 LLVM_DEBUG(dbgs() << "Calling int(*)(int) " << format("0x%016" PRIx64, Addr) 727 << "\n"); 728 return callB<exec::CallIntInt>(Addr, Arg); 729 } 730 731 /// Call the int(int, char*[]) function at the given address in the target and 732 /// return its result. callMain(JITTargetAddress Addr,const std::vector<std::string> & Args)733 Expected<int> callMain(JITTargetAddress Addr, 734 const std::vector<std::string> &Args) { 735 LLVM_DEBUG(dbgs() << "Calling int(*)(int, char*[]) " 736 << format("0x%016" PRIx64, Addr) << "\n"); 737 return callB<exec::CallMain>(Addr, Args); 738 } 739 740 /// Call the void() function at the given address in the target and wait for 741 /// it to finish. callVoidVoid(JITTargetAddress Addr)742 Error callVoidVoid(JITTargetAddress Addr) { 743 LLVM_DEBUG(dbgs() << "Calling void(*)(void) " 744 << format("0x%016" PRIx64, Addr) << "\n"); 745 return callB<exec::CallVoidVoid>(Addr); 746 } 747 748 /// Create an RCMemoryManager which will allocate its memory on the remote 749 /// target. 750 Expected<std::unique_ptr<RemoteRTDyldMemoryManager>> createRemoteMemoryManager()751 createRemoteMemoryManager() { 752 auto Id = AllocatorIds.getNext(); 753 if (auto Err = callB<mem::CreateRemoteAllocator>(Id)) 754 return std::move(Err); 755 return std::unique_ptr<RemoteRTDyldMemoryManager>( 756 new RemoteRTDyldMemoryManager(*this, Id)); 757 } 758 759 /// Create a JITLink-compatible memory manager which will allocate working 760 /// memory on the host and target memory on the remote target. 761 Expected<std::unique_ptr<RemoteJITLinkMemoryManager>> createRemoteJITLinkMemoryManager()762 createRemoteJITLinkMemoryManager() { 763 auto Id = AllocatorIds.getNext(); 764 if (auto Err = callB<mem::CreateRemoteAllocator>(Id)) 765 return std::move(Err); 766 LLVM_DEBUG(dbgs() << "Created remote allocator " << Id << "\n"); 767 return std::unique_ptr<RemoteJITLinkMemoryManager>( 768 new RemoteJITLinkMemoryManager(*this, Id)); 769 } 770 771 /// Create an RCIndirectStubsManager that will allocate stubs on the remote 772 /// target. 773 Expected<std::unique_ptr<RemoteIndirectStubsManager>> createIndirectStubsManager()774 createIndirectStubsManager() { 775 auto Id = IndirectStubOwnerIds.getNext(); 776 if (auto Err = callB<stubs::CreateIndirectStubsOwner>(Id)) 777 return std::move(Err); 778 return std::make_unique<RemoteIndirectStubsManager>(*this, Id); 779 } 780 781 Expected<RemoteCompileCallbackManager &> enableCompileCallbacks(JITTargetAddress ErrorHandlerAddress)782 enableCompileCallbacks(JITTargetAddress ErrorHandlerAddress) { 783 assert(!CallbackManager && "CallbackManager already obtained"); 784 785 // Emit the resolver block on the JIT server. 786 if (auto Err = callB<stubs::EmitResolverBlock>()) 787 return std::move(Err); 788 789 // Create the callback manager. 790 CallbackManager.emplace(*this, ES, ErrorHandlerAddress); 791 RemoteCompileCallbackManager &Mgr = *CallbackManager; 792 return Mgr; 793 } 794 795 /// Search for symbols in the remote process. Note: This should be used by 796 /// symbol resolvers *after* they've searched the local symbol table in the 797 /// JIT stack. getSymbolAddress(StringRef Name)798 Expected<JITTargetAddress> getSymbolAddress(StringRef Name) { 799 return callB<utils::GetSymbolAddress>(Name); 800 } 801 802 /// Get the triple for the remote target. getTargetTriple()803 const std::string &getTargetTriple() const { return RemoteTargetTriple; } 804 terminateSession()805 Error terminateSession() { return callB<utils::TerminateSession>(); } 806 807 private: OrcRemoteTargetClient(shared::RawByteChannel & Channel,ExecutionSession & ES,Error & Err)808 OrcRemoteTargetClient(shared::RawByteChannel &Channel, ExecutionSession &ES, 809 Error &Err) 810 : shared::SingleThreadedRPCEndpoint<shared::RawByteChannel>(Channel, 811 true), 812 ES(ES) { 813 ErrorAsOutParameter EAO(&Err); 814 815 addHandler<utils::RequestCompile>( 816 [this](JITTargetAddress Addr) -> JITTargetAddress { 817 if (CallbackManager) 818 return CallbackManager->executeCompileCallback(Addr); 819 return 0; 820 }); 821 822 if (auto RIOrErr = callB<utils::GetRemoteInfo>()) { 823 std::tie(RemoteTargetTriple, RemotePointerSize, RemotePageSize, 824 RemoteTrampolineSize, RemoteIndirectStubSize) = *RIOrErr; 825 Err = Error::success(); 826 } else 827 Err = RIOrErr.takeError(); 828 } 829 deregisterEHFrames(JITTargetAddress Addr,uint32_t Size)830 void deregisterEHFrames(JITTargetAddress Addr, uint32_t Size) { 831 if (auto Err = callB<eh::RegisterEHFrames>(Addr, Size)) 832 ES.reportError(std::move(Err)); 833 } 834 destroyRemoteAllocator(ResourceIdMgr::ResourceId Id)835 void destroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { 836 if (auto Err = callB<mem::DestroyRemoteAllocator>(Id)) { 837 // FIXME: This will be triggered by a removeModuleSet call: Propagate 838 // error return up through that. 839 llvm_unreachable("Failed to destroy remote allocator."); 840 AllocatorIds.release(Id); 841 } 842 } 843 destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id)844 void destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { 845 IndirectStubOwnerIds.release(Id); 846 if (auto Err = callB<stubs::DestroyIndirectStubsOwner>(Id)) 847 ES.reportError(std::move(Err)); 848 } 849 850 Expected<std::tuple<JITTargetAddress, JITTargetAddress, uint32_t>> emitIndirectStubs(ResourceIdMgr::ResourceId Id,uint32_t NumStubsRequired)851 emitIndirectStubs(ResourceIdMgr::ResourceId Id, uint32_t NumStubsRequired) { 852 return callB<stubs::EmitIndirectStubs>(Id, NumStubsRequired); 853 } 854 emitTrampolineBlock()855 Expected<std::tuple<JITTargetAddress, uint32_t>> emitTrampolineBlock() { 856 return callB<stubs::EmitTrampolineBlock>(); 857 } 858 getIndirectStubSize()859 uint32_t getIndirectStubSize() const { return RemoteIndirectStubSize; } getPageSize()860 uint32_t getPageSize() const { return RemotePageSize; } getPointerSize()861 uint32_t getPointerSize() const { return RemotePointerSize; } 862 getTrampolineSize()863 uint32_t getTrampolineSize() const { return RemoteTrampolineSize; } 864 readMem(char * Dst,JITTargetAddress Src,uint64_t Size)865 Expected<std::vector<uint8_t>> readMem(char *Dst, JITTargetAddress Src, 866 uint64_t Size) { 867 return callB<mem::ReadMem>(Src, Size); 868 } 869 registerEHFrames(JITTargetAddress & RAddr,uint32_t Size)870 Error registerEHFrames(JITTargetAddress &RAddr, uint32_t Size) { 871 // FIXME: Duplicate error and report it via ReportError too? 872 return callB<eh::RegisterEHFrames>(RAddr, Size); 873 } 874 reserveMem(ResourceIdMgr::ResourceId Id,uint64_t Size,uint32_t Align)875 JITTargetAddress reserveMem(ResourceIdMgr::ResourceId Id, uint64_t Size, 876 uint32_t Align) { 877 if (auto AddrOrErr = callB<mem::ReserveMem>(Id, Size, Align)) 878 return *AddrOrErr; 879 else { 880 ES.reportError(AddrOrErr.takeError()); 881 return 0; 882 } 883 } 884 setProtections(ResourceIdMgr::ResourceId Id,JITTargetAddress RemoteSegAddr,unsigned ProtFlags)885 bool setProtections(ResourceIdMgr::ResourceId Id, 886 JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { 887 if (auto Err = callB<mem::SetProtections>(Id, RemoteSegAddr, ProtFlags)) { 888 ES.reportError(std::move(Err)); 889 return true; 890 } else 891 return false; 892 } 893 writeMem(JITTargetAddress Addr,const char * Src,uint64_t Size)894 bool writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { 895 if (auto Err = callB<mem::WriteMem>(DirectBufferWriter(Src, Addr, Size))) { 896 ES.reportError(std::move(Err)); 897 return true; 898 } else 899 return false; 900 } 901 writePointer(JITTargetAddress Addr,JITTargetAddress PtrVal)902 Error writePointer(JITTargetAddress Addr, JITTargetAddress PtrVal) { 903 return callB<mem::WritePtr>(Addr, PtrVal); 904 } 905 doNothing()906 static Error doNothing() { return Error::success(); } 907 908 ExecutionSession &ES; 909 std::function<void(Error)> ReportError; 910 std::string RemoteTargetTriple; 911 uint32_t RemotePointerSize = 0; 912 uint32_t RemotePageSize = 0; 913 uint32_t RemoteTrampolineSize = 0; 914 uint32_t RemoteIndirectStubSize = 0; 915 ResourceIdMgr AllocatorIds, IndirectStubOwnerIds; 916 Optional<RemoteCompileCallbackManager> CallbackManager; 917 }; 918 919 } // end namespace remote 920 } // end namespace orc 921 } // end namespace llvm 922 923 #undef DEBUG_TYPE 924 925 #endif // LLVM_EXECUTIONENGINE_ORC_ORCREMOTETARGETCLIENT_H 926