1 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // Contains utilities for adding indirections and breaking up modules. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 14 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 15 16 #include "llvm/ADT/StringMap.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/ExecutionEngine/JITSymbol.h" 19 #include "llvm/ExecutionEngine/Orc/Core.h" 20 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" 21 #include "llvm/Support/Error.h" 22 #include "llvm/Support/Memory.h" 23 #include "llvm/Support/Process.h" 24 #include "llvm/Transforms/Utils/ValueMapper.h" 25 #include <algorithm> 26 #include <cassert> 27 #include <cstdint> 28 #include <functional> 29 #include <future> 30 #include <map> 31 #include <memory> 32 #include <system_error> 33 #include <utility> 34 #include <vector> 35 36 namespace llvm { 37 38 class Constant; 39 class Function; 40 class FunctionType; 41 class GlobalAlias; 42 class GlobalVariable; 43 class Module; 44 class PointerType; 45 class Triple; 46 class Twine; 47 class Value; 48 49 namespace orc { 50 51 /// Base class for pools of compiler re-entry trampolines. 52 /// These trampolines are callable addresses that save all register state 53 /// before calling a supplied function to return the trampoline landing 54 /// address, then restore all state before jumping to that address. They 55 /// are used by various ORC APIs to support lazy compilation 56 class TrampolinePool { 57 public: 58 using NotifyLandingResolvedFunction = 59 unique_function<void(JITTargetAddress) const>; 60 61 using ResolveLandingFunction = unique_function<void( 62 JITTargetAddress TrampolineAddr, 63 NotifyLandingResolvedFunction OnLandingResolved) const>; 64 65 virtual ~TrampolinePool() {} 66 67 /// Get an available trampoline address. 68 /// Returns an error if no trampoline can be created. 69 virtual Expected<JITTargetAddress> getTrampoline() = 0; 70 71 private: 72 virtual void anchor(); 73 }; 74 75 /// A trampoline pool for trampolines within the current process. 76 template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool { 77 public: 78 /// Creates a LocalTrampolinePool with the given RunCallback function. 79 /// Returns an error if this function is unable to correctly allocate, write 80 /// and protect the resolver code block. 81 static Expected<std::unique_ptr<LocalTrampolinePool>> 82 Create(ResolveLandingFunction ResolveLanding) { 83 Error Err = Error::success(); 84 85 auto LTP = std::unique_ptr<LocalTrampolinePool>( 86 new LocalTrampolinePool(std::move(ResolveLanding), Err)); 87 88 if (Err) 89 return std::move(Err); 90 return std::move(LTP); 91 } 92 93 /// Get a free trampoline. Returns an error if one can not be provided (e.g. 94 /// because the pool is empty and can not be grown). 95 Expected<JITTargetAddress> getTrampoline() override { 96 std::lock_guard<std::mutex> Lock(LTPMutex); 97 if (AvailableTrampolines.empty()) { 98 if (auto Err = grow()) 99 return std::move(Err); 100 } 101 assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool"); 102 auto TrampolineAddr = AvailableTrampolines.back(); 103 AvailableTrampolines.pop_back(); 104 return TrampolineAddr; 105 } 106 107 /// Returns the given trampoline to the pool for re-use. 108 void releaseTrampoline(JITTargetAddress TrampolineAddr) { 109 std::lock_guard<std::mutex> Lock(LTPMutex); 110 AvailableTrampolines.push_back(TrampolineAddr); 111 } 112 113 private: 114 static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) { 115 LocalTrampolinePool<ORCABI> *TrampolinePool = 116 static_cast<LocalTrampolinePool *>(TrampolinePoolPtr); 117 118 std::promise<JITTargetAddress> LandingAddressP; 119 auto LandingAddressF = LandingAddressP.get_future(); 120 121 TrampolinePool->ResolveLanding(pointerToJITTargetAddress(TrampolineId), 122 [&](JITTargetAddress LandingAddress) { 123 LandingAddressP.set_value(LandingAddress); 124 }); 125 return LandingAddressF.get(); 126 } 127 128 LocalTrampolinePool(ResolveLandingFunction ResolveLanding, Error &Err) 129 : ResolveLanding(std::move(ResolveLanding)) { 130 131 ErrorAsOutParameter _(&Err); 132 133 /// Try to set up the resolver block. 134 std::error_code EC; 135 ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 136 ORCABI::ResolverCodeSize, nullptr, 137 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 138 if (EC) { 139 Err = errorCodeToError(EC); 140 return; 141 } 142 143 ORCABI::writeResolverCode(static_cast<char *>(ResolverBlock.base()), 144 pointerToJITTargetAddress(ResolverBlock.base()), 145 pointerToJITTargetAddress(&reenter), 146 pointerToJITTargetAddress(this)); 147 148 EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(), 149 sys::Memory::MF_READ | 150 sys::Memory::MF_EXEC); 151 if (EC) { 152 Err = errorCodeToError(EC); 153 return; 154 } 155 } 156 157 Error grow() { 158 assert(this->AvailableTrampolines.empty() && "Growing prematurely?"); 159 160 std::error_code EC; 161 auto TrampolineBlock = 162 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 163 sys::Process::getPageSizeEstimate(), nullptr, 164 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 165 if (EC) 166 return errorCodeToError(EC); 167 168 unsigned NumTrampolines = 169 (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / 170 ORCABI::TrampolineSize; 171 172 char *TrampolineMem = static_cast<char *>(TrampolineBlock.base()); 173 ORCABI::writeTrampolines( 174 TrampolineMem, pointerToJITTargetAddress(TrampolineMem), 175 pointerToJITTargetAddress(ResolverBlock.base()), NumTrampolines); 176 177 for (unsigned I = 0; I < NumTrampolines; ++I) 178 this->AvailableTrampolines.push_back(pointerToJITTargetAddress( 179 TrampolineMem + (I * ORCABI::TrampolineSize))); 180 181 if (auto EC = sys::Memory::protectMappedMemory( 182 TrampolineBlock.getMemoryBlock(), 183 sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 184 return errorCodeToError(EC); 185 186 TrampolineBlocks.push_back(std::move(TrampolineBlock)); 187 return Error::success(); 188 } 189 190 ResolveLandingFunction ResolveLanding; 191 192 std::mutex LTPMutex; 193 sys::OwningMemoryBlock ResolverBlock; 194 std::vector<sys::OwningMemoryBlock> TrampolineBlocks; 195 std::vector<JITTargetAddress> AvailableTrampolines; 196 }; 197 198 /// Target-independent base class for compile callback management. 199 class JITCompileCallbackManager { 200 public: 201 using CompileFunction = std::function<JITTargetAddress()>; 202 203 virtual ~JITCompileCallbackManager() = default; 204 205 /// Reserve a compile callback. 206 Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile); 207 208 /// Execute the callback for the given trampoline id. Called by the JIT 209 /// to compile functions on demand. 210 JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr); 211 212 protected: 213 /// Construct a JITCompileCallbackManager. 214 JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP, 215 ExecutionSession &ES, 216 JITTargetAddress ErrorHandlerAddress) 217 : TP(std::move(TP)), ES(ES), 218 CallbacksJD(ES.createBareJITDylib("<Callbacks>")), 219 ErrorHandlerAddress(ErrorHandlerAddress) {} 220 221 void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) { 222 this->TP = std::move(TP); 223 } 224 225 private: 226 std::mutex CCMgrMutex; 227 std::unique_ptr<TrampolinePool> TP; 228 ExecutionSession &ES; 229 JITDylib &CallbacksJD; 230 JITTargetAddress ErrorHandlerAddress; 231 std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol; 232 size_t NextCallbackId = 0; 233 }; 234 235 /// Manage compile callbacks for in-process JITs. 236 template <typename ORCABI> 237 class LocalJITCompileCallbackManager : public JITCompileCallbackManager { 238 public: 239 /// Create a new LocalJITCompileCallbackManager. 240 static Expected<std::unique_ptr<LocalJITCompileCallbackManager>> 241 Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { 242 Error Err = Error::success(); 243 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>( 244 new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err)); 245 if (Err) 246 return std::move(Err); 247 return std::move(CCMgr); 248 } 249 250 private: 251 /// Construct a InProcessJITCompileCallbackManager. 252 /// @param ErrorHandlerAddress The address of an error handler in the target 253 /// process to be used if a compile callback fails. 254 LocalJITCompileCallbackManager(ExecutionSession &ES, 255 JITTargetAddress ErrorHandlerAddress, 256 Error &Err) 257 : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) { 258 using NotifyLandingResolvedFunction = 259 TrampolinePool::NotifyLandingResolvedFunction; 260 261 ErrorAsOutParameter _(&Err); 262 auto TP = LocalTrampolinePool<ORCABI>::Create( 263 [this](JITTargetAddress TrampolineAddr, 264 NotifyLandingResolvedFunction NotifyLandingResolved) { 265 NotifyLandingResolved(executeCompileCallback(TrampolineAddr)); 266 }); 267 268 if (!TP) { 269 Err = TP.takeError(); 270 return; 271 } 272 273 setTrampolinePool(std::move(*TP)); 274 } 275 }; 276 277 /// Base class for managing collections of named indirect stubs. 278 class IndirectStubsManager { 279 public: 280 /// Map type for initializing the manager. See init. 281 using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>; 282 283 virtual ~IndirectStubsManager() = default; 284 285 /// Create a single stub with the given name, target address and flags. 286 virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, 287 JITSymbolFlags StubFlags) = 0; 288 289 /// Create StubInits.size() stubs with the given names, target 290 /// addresses, and flags. 291 virtual Error createStubs(const StubInitsMap &StubInits) = 0; 292 293 /// Find the stub with the given name. If ExportedStubsOnly is true, 294 /// this will only return a result if the stub's flags indicate that it 295 /// is exported. 296 virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0; 297 298 /// Find the implementation-pointer for the stub. 299 virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0; 300 301 /// Change the value of the implementation pointer for the stub. 302 virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0; 303 304 private: 305 virtual void anchor(); 306 }; 307 308 template <typename ORCABI> class LocalIndirectStubsInfo { 309 public: 310 LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem) 311 : NumStubs(NumStubs), StubsMem(std::move(StubsMem)) {} 312 313 static Expected<LocalIndirectStubsInfo> create(unsigned MinStubs, 314 unsigned PageSize) { 315 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs, PageSize); 316 317 assert((ISAS.StubBytes % PageSize == 0) && 318 "StubBytes is not a page size multiple"); 319 uint64_t PointerAlloc = alignTo(ISAS.PointerBytes, PageSize); 320 321 // Allocate memory for stubs and pointers in one call. 322 std::error_code EC; 323 auto StubsAndPtrsMem = 324 sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( 325 ISAS.StubBytes + PointerAlloc, nullptr, 326 sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); 327 if (EC) 328 return errorCodeToError(EC); 329 330 sys::MemoryBlock StubsBlock(StubsAndPtrsMem.base(), ISAS.StubBytes); 331 auto StubsBlockMem = static_cast<char *>(StubsAndPtrsMem.base()); 332 auto PtrBlockAddress = 333 pointerToJITTargetAddress(StubsBlockMem) + ISAS.StubBytes; 334 335 ORCABI::writeIndirectStubsBlock(StubsBlockMem, 336 pointerToJITTargetAddress(StubsBlockMem), 337 PtrBlockAddress, ISAS.NumStubs); 338 339 if (auto EC = sys::Memory::protectMappedMemory( 340 StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) 341 return errorCodeToError(EC); 342 343 return LocalIndirectStubsInfo(ISAS.NumStubs, std::move(StubsAndPtrsMem)); 344 } 345 346 unsigned getNumStubs() const { return NumStubs; } 347 348 void *getStub(unsigned Idx) const { 349 return static_cast<char *>(StubsMem.base()) + Idx * ORCABI::StubSize; 350 } 351 352 void **getPtr(unsigned Idx) const { 353 char *PtrsBase = 354 static_cast<char *>(StubsMem.base()) + NumStubs * ORCABI::StubSize; 355 return reinterpret_cast<void **>(PtrsBase) + Idx; 356 } 357 358 private: 359 unsigned NumStubs = 0; 360 sys::OwningMemoryBlock StubsMem; 361 }; 362 363 /// IndirectStubsManager implementation for the host architecture, e.g. 364 /// OrcX86_64. (See OrcArchitectureSupport.h). 365 template <typename TargetT> 366 class LocalIndirectStubsManager : public IndirectStubsManager { 367 public: 368 Error createStub(StringRef StubName, JITTargetAddress StubAddr, 369 JITSymbolFlags StubFlags) override { 370 std::lock_guard<std::mutex> Lock(StubsMutex); 371 if (auto Err = reserveStubs(1)) 372 return Err; 373 374 createStubInternal(StubName, StubAddr, StubFlags); 375 376 return Error::success(); 377 } 378 379 Error createStubs(const StubInitsMap &StubInits) override { 380 std::lock_guard<std::mutex> Lock(StubsMutex); 381 if (auto Err = reserveStubs(StubInits.size())) 382 return Err; 383 384 for (auto &Entry : StubInits) 385 createStubInternal(Entry.first(), Entry.second.first, 386 Entry.second.second); 387 388 return Error::success(); 389 } 390 391 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override { 392 std::lock_guard<std::mutex> Lock(StubsMutex); 393 auto I = StubIndexes.find(Name); 394 if (I == StubIndexes.end()) 395 return nullptr; 396 auto Key = I->second.first; 397 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second); 398 assert(StubAddr && "Missing stub address"); 399 auto StubTargetAddr = 400 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr)); 401 auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second); 402 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported()) 403 return nullptr; 404 return StubSymbol; 405 } 406 407 JITEvaluatedSymbol findPointer(StringRef Name) override { 408 std::lock_guard<std::mutex> Lock(StubsMutex); 409 auto I = StubIndexes.find(Name); 410 if (I == StubIndexes.end()) 411 return nullptr; 412 auto Key = I->second.first; 413 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second); 414 assert(PtrAddr && "Missing pointer address"); 415 auto PtrTargetAddr = 416 static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr)); 417 return JITEvaluatedSymbol(PtrTargetAddr, I->second.second); 418 } 419 420 Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override { 421 using AtomicIntPtr = std::atomic<uintptr_t>; 422 423 std::lock_guard<std::mutex> Lock(StubsMutex); 424 auto I = StubIndexes.find(Name); 425 assert(I != StubIndexes.end() && "No stub pointer for symbol"); 426 auto Key = I->second.first; 427 AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>( 428 IndirectStubsInfos[Key.first].getPtr(Key.second)); 429 *AtomicStubPtr = static_cast<uintptr_t>(NewAddr); 430 return Error::success(); 431 } 432 433 private: 434 Error reserveStubs(unsigned NumStubs) { 435 if (NumStubs <= FreeStubs.size()) 436 return Error::success(); 437 438 unsigned NewStubsRequired = NumStubs - FreeStubs.size(); 439 unsigned NewBlockId = IndirectStubsInfos.size(); 440 auto ISI = 441 LocalIndirectStubsInfo<TargetT>::create(NewStubsRequired, PageSize); 442 if (!ISI) 443 return ISI.takeError(); 444 for (unsigned I = 0; I < ISI->getNumStubs(); ++I) 445 FreeStubs.push_back(std::make_pair(NewBlockId, I)); 446 IndirectStubsInfos.push_back(std::move(*ISI)); 447 return Error::success(); 448 } 449 450 void createStubInternal(StringRef StubName, JITTargetAddress InitAddr, 451 JITSymbolFlags StubFlags) { 452 auto Key = FreeStubs.back(); 453 FreeStubs.pop_back(); 454 *IndirectStubsInfos[Key.first].getPtr(Key.second) = 455 jitTargetAddressToPointer<void *>(InitAddr); 456 StubIndexes[StubName] = std::make_pair(Key, StubFlags); 457 } 458 459 unsigned PageSize = sys::Process::getPageSizeEstimate(); 460 std::mutex StubsMutex; 461 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos; 462 using StubKey = std::pair<uint16_t, uint16_t>; 463 std::vector<StubKey> FreeStubs; 464 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes; 465 }; 466 467 /// Create a local compile callback manager. 468 /// 469 /// The given target triple will determine the ABI, and the given 470 /// ErrorHandlerAddress will be used by the resulting compile callback 471 /// manager if a compile callback fails. 472 Expected<std::unique_ptr<JITCompileCallbackManager>> 473 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, 474 JITTargetAddress ErrorHandlerAddress); 475 476 /// Create a local indriect stubs manager builder. 477 /// 478 /// The given target triple will determine the ABI. 479 std::function<std::unique_ptr<IndirectStubsManager>()> 480 createLocalIndirectStubsManagerBuilder(const Triple &T); 481 482 /// Build a function pointer of FunctionType with the given constant 483 /// address. 484 /// 485 /// Usage example: Turn a trampoline address into a function pointer constant 486 /// for use in a stub. 487 Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr); 488 489 /// Create a function pointer with the given type, name, and initializer 490 /// in the given Module. 491 GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name, 492 Constant *Initializer); 493 494 /// Turn a function declaration into a stub function that makes an 495 /// indirect call using the given function pointer. 496 void makeStub(Function &F, Value &ImplPointer); 497 498 /// Promotes private symbols to global hidden, and renames to prevent clashes 499 /// with other promoted symbols. The same SymbolPromoter instance should be 500 /// used for all symbols to be added to a single JITDylib. 501 class SymbolLinkagePromoter { 502 public: 503 /// Promote symbols in the given module. Returns the set of global values 504 /// that have been renamed/promoted. 505 std::vector<GlobalValue *> operator()(Module &M); 506 507 private: 508 unsigned NextId = 0; 509 }; 510 511 /// Clone a function declaration into a new module. 512 /// 513 /// This function can be used as the first step towards creating a callback 514 /// stub (see makeStub), or moving a function body (see moveFunctionBody). 515 /// 516 /// If the VMap argument is non-null, a mapping will be added between F and 517 /// the new declaration, and between each of F's arguments and the new 518 /// declaration's arguments. This map can then be passed in to moveFunction to 519 /// move the function body if required. Note: When moving functions between 520 /// modules with these utilities, all decls should be cloned (and added to a 521 /// single VMap) before any bodies are moved. This will ensure that references 522 /// between functions all refer to the versions in the new module. 523 Function *cloneFunctionDecl(Module &Dst, const Function &F, 524 ValueToValueMapTy *VMap = nullptr); 525 526 /// Move the body of function 'F' to a cloned function declaration in a 527 /// different module (See related cloneFunctionDecl). 528 /// 529 /// If the target function declaration is not supplied via the NewF parameter 530 /// then it will be looked up via the VMap. 531 /// 532 /// This will delete the body of function 'F' from its original parent module, 533 /// but leave its declaration. 534 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, 535 ValueMaterializer *Materializer = nullptr, 536 Function *NewF = nullptr); 537 538 /// Clone a global variable declaration into a new module. 539 GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, 540 ValueToValueMapTy *VMap = nullptr); 541 542 /// Move global variable GV from its parent module to cloned global 543 /// declaration in a different module. 544 /// 545 /// If the target global declaration is not supplied via the NewGV parameter 546 /// then it will be looked up via the VMap. 547 /// 548 /// This will delete the initializer of GV from its original parent module, 549 /// but leave its declaration. 550 void moveGlobalVariableInitializer(GlobalVariable &OrigGV, 551 ValueToValueMapTy &VMap, 552 ValueMaterializer *Materializer = nullptr, 553 GlobalVariable *NewGV = nullptr); 554 555 /// Clone a global alias declaration into a new module. 556 GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, 557 ValueToValueMapTy &VMap); 558 559 /// Clone module flags metadata into the destination module. 560 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, 561 ValueToValueMapTy &VMap); 562 563 } // end namespace orc 564 565 } // end namespace llvm 566 567 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H 568