1 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
10 #include "llvm/ADT/STLExtras.h"
11 #include "llvm/ADT/Triple.h"
12 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
13 #include "llvm/ExecutionEngine/Orc/OrcABISupport.h"
14 #include "llvm/IR/IRBuilder.h"
15 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
16 #include "llvm/MC/MCInstrAnalysis.h"
17 #include "llvm/Support/Format.h"
18 #include "llvm/Transforms/Utils/Cloning.h"
19 #include <sstream>
20 
21 #define DEBUG_TYPE "orc"
22 
23 using namespace llvm;
24 using namespace llvm::orc;
25 
26 namespace {
27 
28 class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {
29 public:
30   using CompileFunction = JITCompileCallbackManager::CompileFunction;
31 
32   CompileCallbackMaterializationUnit(SymbolStringPtr Name,
33                                      CompileFunction Compile)
34       : MaterializationUnit(Interface(
35             SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),
36         Name(std::move(Name)), Compile(std::move(Compile)) {}
37 
38   StringRef getName() const override { return "<Compile Callbacks>"; }
39 
40 private:
41   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42     SymbolMap Result;
43     Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
44     // No dependencies, so these calls cannot fail.
45     cantFail(R->notifyResolved(Result));
46     cantFail(R->notifyEmitted());
47   }
48 
49   void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
50     llvm_unreachable("Discard should never occur on a LMU?");
51   }
52 
53   SymbolStringPtr Name;
54   CompileFunction Compile;
55 };
56 
57 } // namespace
58 
59 namespace llvm {
60 namespace orc {
61 
62 TrampolinePool::~TrampolinePool() = default;
63 void IndirectStubsManager::anchor() {}
64 
65 Expected<JITTargetAddress>
66 JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
67   if (auto TrampolineAddr = TP->getTrampoline()) {
68     auto CallbackName =
69         ES.intern(std::string("cc") + std::to_string(++NextCallbackId));
70 
71     std::lock_guard<std::mutex> Lock(CCMgrMutex);
72     AddrToSymbol[*TrampolineAddr] = CallbackName;
73     cantFail(
74         CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(
75             std::move(CallbackName), std::move(Compile))));
76     return *TrampolineAddr;
77   } else
78     return TrampolineAddr.takeError();
79 }
80 
81 JITTargetAddress JITCompileCallbackManager::executeCompileCallback(
82     JITTargetAddress TrampolineAddr) {
83   SymbolStringPtr Name;
84 
85   {
86     std::unique_lock<std::mutex> Lock(CCMgrMutex);
87     auto I = AddrToSymbol.find(TrampolineAddr);
88 
89     // If this address is not associated with a compile callback then report an
90     // error to the execution session and return ErrorHandlerAddress to the
91     // callee.
92     if (I == AddrToSymbol.end()) {
93       Lock.unlock();
94       std::string ErrMsg;
95       {
96         raw_string_ostream ErrMsgStream(ErrMsg);
97         ErrMsgStream << "No compile callback for trampoline at "
98                      << format("0x%016" PRIx64, TrampolineAddr);
99       }
100       ES.reportError(
101           make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()));
102       return ErrorHandlerAddress;
103     } else
104       Name = I->second;
105   }
106 
107   if (auto Sym =
108           ES.lookup(makeJITDylibSearchOrder(
109                         &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),
110                     Name))
111     return Sym->getAddress();
112   else {
113     llvm::dbgs() << "Didn't find callback.\n";
114     // If anything goes wrong materializing Sym then report it to the session
115     // and return the ErrorHandlerAddress;
116     ES.reportError(Sym.takeError());
117     return ErrorHandlerAddress;
118   }
119 }
120 
121 Expected<std::unique_ptr<JITCompileCallbackManager>>
122 createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
123                                   JITTargetAddress ErrorHandlerAddress) {
124   switch (T.getArch()) {
125   default:
126     return make_error<StringError>(
127         std::string("No callback manager available for ") + T.str(),
128         inconvertibleErrorCode());
129   case Triple::aarch64:
130   case Triple::aarch64_32: {
131     typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
132     return CCMgrT::Create(ES, ErrorHandlerAddress);
133     }
134 
135     case Triple::x86: {
136       typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;
137       return CCMgrT::Create(ES, ErrorHandlerAddress);
138     }
139 
140     case Triple::mips: {
141       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;
142       return CCMgrT::Create(ES, ErrorHandlerAddress);
143     }
144     case Triple::mipsel: {
145       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;
146       return CCMgrT::Create(ES, ErrorHandlerAddress);
147     }
148 
149     case Triple::mips64:
150     case Triple::mips64el: {
151       typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;
152       return CCMgrT::Create(ES, ErrorHandlerAddress);
153     }
154 
155     case Triple::riscv64: {
156       typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;
157       return CCMgrT::Create(ES, ErrorHandlerAddress);
158     }
159 
160     case Triple::x86_64: {
161       if (T.getOS() == Triple::OSType::Win32) {
162         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;
163         return CCMgrT::Create(ES, ErrorHandlerAddress);
164       } else {
165         typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;
166         return CCMgrT::Create(ES, ErrorHandlerAddress);
167       }
168     }
169 
170   }
171 }
172 
173 std::function<std::unique_ptr<IndirectStubsManager>()>
174 createLocalIndirectStubsManagerBuilder(const Triple &T) {
175   switch (T.getArch()) {
176     default:
177       return [](){
178         return std::make_unique<
179                        orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
180       };
181 
182     case Triple::aarch64:
183     case Triple::aarch64_32:
184       return [](){
185         return std::make_unique<
186                        orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
187       };
188 
189     case Triple::x86:
190       return [](){
191         return std::make_unique<
192                        orc::LocalIndirectStubsManager<orc::OrcI386>>();
193       };
194 
195     case Triple::mips:
196       return [](){
197           return std::make_unique<
198                       orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
199       };
200 
201     case Triple::mipsel:
202       return [](){
203           return std::make_unique<
204                       orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
205       };
206 
207     case Triple::mips64:
208     case Triple::mips64el:
209       return [](){
210           return std::make_unique<
211                       orc::LocalIndirectStubsManager<orc::OrcMips64>>();
212       };
213 
214     case Triple::riscv64:
215       return []() {
216         return std::make_unique<
217             orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();
218       };
219 
220     case Triple::x86_64:
221       if (T.getOS() == Triple::OSType::Win32) {
222         return [](){
223           return std::make_unique<
224                      orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
225         };
226       } else {
227         return [](){
228           return std::make_unique<
229                      orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
230         };
231       }
232 
233   }
234 }
235 
236 Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) {
237   Constant *AddrIntVal =
238     ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr);
239   Constant *AddrPtrVal =
240     ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal,
241                           PointerType::get(&FT, 0));
242   return AddrPtrVal;
243 }
244 
245 GlobalVariable* createImplPointer(PointerType &PT, Module &M,
246                                   const Twine &Name, Constant *Initializer) {
247   auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,
248                                Initializer, Name, nullptr,
249                                GlobalValue::NotThreadLocal, 0, true);
250   IP->setVisibility(GlobalValue::HiddenVisibility);
251   return IP;
252 }
253 
254 void makeStub(Function &F, Value &ImplPointer) {
255   assert(F.isDeclaration() && "Can't turn a definition into a stub.");
256   assert(F.getParent() && "Function isn't in a module.");
257   Module &M = *F.getParent();
258   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
259   IRBuilder<> Builder(EntryBlock);
260   LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
261   std::vector<Value*> CallArgs;
262   for (auto &A : F.args())
263     CallArgs.push_back(&A);
264   CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
265   Call->setTailCall();
266   Call->setAttributes(F.getAttributes());
267   if (F.getReturnType()->isVoidTy())
268     Builder.CreateRetVoid();
269   else
270     Builder.CreateRet(Call);
271 }
272 
273 std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {
274   std::vector<GlobalValue *> PromotedGlobals;
275 
276   for (auto &GV : M.global_values()) {
277     bool Promoted = true;
278 
279     // Rename if necessary.
280     if (!GV.hasName())
281       GV.setName("__orc_anon." + Twine(NextId++));
282     else if (GV.getName().startswith("\01L"))
283       GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));
284     else if (GV.hasLocalLinkage())
285       GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));
286     else
287       Promoted = false;
288 
289     if (GV.hasLocalLinkage()) {
290       GV.setLinkage(GlobalValue::ExternalLinkage);
291       GV.setVisibility(GlobalValue::HiddenVisibility);
292       Promoted = true;
293     }
294     GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);
295 
296     if (Promoted)
297       PromotedGlobals.push_back(&GV);
298   }
299 
300   return PromotedGlobals;
301 }
302 
303 Function* cloneFunctionDecl(Module &Dst, const Function &F,
304                             ValueToValueMapTy *VMap) {
305   Function *NewF =
306     Function::Create(cast<FunctionType>(F.getValueType()),
307                      F.getLinkage(), F.getName(), &Dst);
308   NewF->copyAttributesFrom(&F);
309 
310   if (VMap) {
311     (*VMap)[&F] = NewF;
312     auto NewArgI = NewF->arg_begin();
313     for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;
314          ++ArgI, ++NewArgI)
315       (*VMap)[&*ArgI] = &*NewArgI;
316   }
317 
318   return NewF;
319 }
320 
321 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
322                       ValueMaterializer *Materializer,
323                       Function *NewF) {
324   assert(!OrigF.isDeclaration() && "Nothing to move");
325   if (!NewF)
326     NewF = cast<Function>(VMap[&OrigF]);
327   else
328     assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap.");
329   assert(NewF && "Function mapping missing from VMap.");
330   assert(NewF->getParent() != OrigF.getParent() &&
331          "moveFunctionBody should only be used to move bodies between "
332          "modules.");
333 
334   SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.
335   CloneFunctionInto(NewF, &OrigF, VMap,
336                     CloneFunctionChangeType::DifferentModule, Returns, "",
337                     nullptr, nullptr, Materializer);
338   OrigF.deleteBody();
339 }
340 
341 GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
342                                         ValueToValueMapTy *VMap) {
343   GlobalVariable *NewGV = new GlobalVariable(
344       Dst, GV.getValueType(), GV.isConstant(),
345       GV.getLinkage(), nullptr, GV.getName(), nullptr,
346       GV.getThreadLocalMode(), GV.getType()->getAddressSpace());
347   NewGV->copyAttributesFrom(&GV);
348   if (VMap)
349     (*VMap)[&GV] = NewGV;
350   return NewGV;
351 }
352 
353 void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
354                                    ValueToValueMapTy &VMap,
355                                    ValueMaterializer *Materializer,
356                                    GlobalVariable *NewGV) {
357   assert(OrigGV.hasInitializer() && "Nothing to move");
358   if (!NewGV)
359     NewGV = cast<GlobalVariable>(VMap[&OrigGV]);
360   else
361     assert(VMap[&OrigGV] == NewGV &&
362            "Incorrect global variable mapping in VMap.");
363   assert(NewGV->getParent() != OrigGV.getParent() &&
364          "moveGlobalVariableInitializer should only be used to move "
365          "initializers between modules");
366 
367   NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None,
368                                  nullptr, Materializer));
369 }
370 
371 GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
372                                   ValueToValueMapTy &VMap) {
373   assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");
374   auto *NewA = GlobalAlias::create(OrigA.getValueType(),
375                                    OrigA.getType()->getPointerAddressSpace(),
376                                    OrigA.getLinkage(), OrigA.getName(), &Dst);
377   NewA->copyAttributesFrom(&OrigA);
378   VMap[&OrigA] = NewA;
379   return NewA;
380 }
381 
382 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
383                               ValueToValueMapTy &VMap) {
384   auto *MFs = Src.getModuleFlagsMetadata();
385   if (!MFs)
386     return;
387   for (auto *MF : MFs->operands())
388     Dst.addModuleFlag(MapMetadata(MF, VMap));
389 }
390 
391 Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,
392                                                    jitlink::LinkGraph &G,
393                                                    MCDisassembler &Disassembler,
394                                                    MCInstrAnalysis &MIA) {
395   // AArch64 appears to already come with the necessary relocations. Among other
396   // architectures, only x86_64 is currently implemented here.
397   if (G.getTargetTriple().getArch() != Triple::x86_64)
398     return Error::success();
399 
400   raw_null_ostream CommentStream;
401   auto &STI = Disassembler.getSubtargetInfo();
402 
403   // Determine the function bounds
404   auto &B = Sym.getBlock();
405   assert(!B.isZeroFill() && "expected content block");
406   auto SymAddress = Sym.getAddress();
407   auto SymStartInBlock =
408       (const uint8_t *)B.getContent().data() + Sym.getOffset();
409   auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();
410   auto Content = makeArrayRef(SymStartInBlock, SymSize);
411 
412   LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");
413 
414   SmallDenseSet<uintptr_t, 8> ExistingRelocations;
415   for (auto &E : B.edges()) {
416     if (E.isRelocation())
417       ExistingRelocations.insert(E.getOffset());
418   }
419 
420   size_t I = 0;
421   while (I < Content.size()) {
422     MCInst Instr;
423     uint64_t InstrSize = 0;
424     uint64_t InstrStart = SymAddress.getValue() + I;
425     auto DecodeStatus = Disassembler.getInstruction(
426         Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);
427     if (DecodeStatus != MCDisassembler::Success) {
428       LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "
429                         << InstrStart);
430       return make_error<StringError>(
431           formatv("failed to disassemble at address {0:x16}", InstrStart),
432           inconvertibleErrorCode());
433     }
434     // Advance to the next instruction.
435     I += InstrSize;
436 
437     // Check for a PC-relative address equal to the symbol itself.
438     auto PCRelAddr =
439         MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);
440     if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())
441       continue;
442 
443     auto RelocOffInInstr =
444         MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);
445     if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {
446       LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "
447                         << InstrStart);
448       continue;
449     }
450 
451     auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -
452                            SymAddress + Sym.getOffset();
453     if (ExistingRelocations.contains(RelocOffInBlock))
454       continue;
455 
456     LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);
457     B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);
458   }
459   return Error::success();
460 }
461 
462 } // End namespace orc.
463 } // End namespace llvm.
464