1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/aarch64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
14 #include "EHFrameSupportImpl.h"
15 #include "ELFLinkGraphBuilder.h"
16 #include "JITLinkGeneric.h"
17 #include "llvm/BinaryFormat/ELF.h"
18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
19 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
20 #include "llvm/Object/ELFObjectFile.h"
21 #include "llvm/Support/Endian.h"
22 
23 #define DEBUG_TYPE "jitlink"
24 
25 using namespace llvm;
26 using namespace llvm::jitlink;
27 
28 namespace {
29 
30 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
31   friend class JITLinker<ELFJITLinker_aarch64>;
32 
33 public:
34   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
35                        std::unique_ptr<LinkGraph> G,
36                        PassConfiguration PassConfig)
37       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
38 
39 private:
40   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
41     return aarch64::applyFixup(G, B, E);
42   }
43 };
44 
45 template <typename ELFT>
46 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
47 private:
48   enum ELFAArch64RelocationKind : Edge::Kind {
49     ELFCall26 = Edge::FirstRelocation,
50     ELFAdrPage21,
51     ELFAddAbs12,
52     ELFLdSt8Abs12,
53     ELFLdSt16Abs12,
54     ELFLdSt32Abs12,
55     ELFLdSt64Abs12,
56     ELFLdSt128Abs12,
57     ELFMovwAbsG0,
58     ELFMovwAbsG1,
59     ELFMovwAbsG2,
60     ELFMovwAbsG3,
61     ELFAbs64,
62     ELFPrel32,
63     ELFPrel64,
64     ELFAdrGOTPage21,
65     ELFLd64GOTLo12,
66   };
67 
68   static Expected<ELFAArch64RelocationKind>
69   getRelocationKind(const uint32_t Type) {
70     using namespace aarch64;
71     switch (Type) {
72     case ELF::R_AARCH64_CALL26:
73     case ELF::R_AARCH64_JUMP26:
74       return ELFCall26;
75     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
76       return ELFAdrPage21;
77     case ELF::R_AARCH64_ADD_ABS_LO12_NC:
78       return ELFAddAbs12;
79     case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
80       return ELFLdSt8Abs12;
81     case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
82       return ELFLdSt16Abs12;
83     case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
84       return ELFLdSt32Abs12;
85     case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
86       return ELFLdSt64Abs12;
87     case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
88       return ELFLdSt128Abs12;
89     case ELF::R_AARCH64_MOVW_UABS_G0_NC:
90       return ELFMovwAbsG0;
91     case ELF::R_AARCH64_MOVW_UABS_G1_NC:
92       return ELFMovwAbsG1;
93     case ELF::R_AARCH64_MOVW_UABS_G2_NC:
94       return ELFMovwAbsG2;
95     case ELF::R_AARCH64_MOVW_UABS_G3:
96       return ELFMovwAbsG3;
97     case ELF::R_AARCH64_ABS64:
98       return ELFAbs64;
99     case ELF::R_AARCH64_PREL32:
100       return ELFPrel32;
101     case ELF::R_AARCH64_PREL64:
102       return ELFPrel64;
103     case ELF::R_AARCH64_ADR_GOT_PAGE:
104       return ELFAdrGOTPage21;
105     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
106       return ELFLd64GOTLo12;
107     }
108 
109     return make_error<JITLinkError>(
110         "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) +
111         object::getELFRelocationTypeName(ELF::EM_AARCH64, Type));
112   }
113 
114   Error addRelocations() override {
115     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
116 
117     using Base = ELFLinkGraphBuilder<ELFT>;
118     using Self = ELFLinkGraphBuilder_aarch64<ELFT>;
119     for (const auto &RelSect : Base::Sections)
120       if (Error Err = Base::forEachRelocation(RelSect, this,
121                                               &Self::addSingleRelocation))
122         return Err;
123 
124     return Error::success();
125   }
126 
127   Error addSingleRelocation(const typename ELFT::Rela &Rel,
128                             const typename ELFT::Shdr &FixupSect,
129                             Block &BlockToFix) {
130     using support::ulittle32_t;
131     using Base = ELFLinkGraphBuilder<ELFT>;
132 
133     uint32_t SymbolIndex = Rel.getSymbol(false);
134     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
135     if (!ObjSymbol)
136       return ObjSymbol.takeError();
137 
138     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
139     if (!GraphSymbol)
140       return make_error<StringError>(
141           formatv("Could not find symbol at given index, did you add it to "
142                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
143                   SymbolIndex, (*ObjSymbol)->st_shndx,
144                   Base::GraphSymbols.size()),
145           inconvertibleErrorCode());
146 
147     uint32_t Type = Rel.getType(false);
148     Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type);
149     if (!RelocKind)
150       return RelocKind.takeError();
151 
152     int64_t Addend = Rel.r_addend;
153     orc::ExecutorAddr FixupAddress =
154         orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
155     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
156 
157     // Get a pointer to the fixup content.
158     const void *FixupContent = BlockToFix.getContent().data() +
159                                (FixupAddress - BlockToFix.getAddress());
160 
161     Edge::Kind Kind = Edge::Invalid;
162 
163     switch (*RelocKind) {
164     case ELFCall26: {
165       Kind = aarch64::Branch26;
166       break;
167     }
168     case ELFAdrPage21: {
169       Kind = aarch64::Page21;
170       break;
171     }
172     case ELFAddAbs12: {
173       Kind = aarch64::PageOffset12;
174       break;
175     }
176     case ELFLdSt8Abs12: {
177       uint32_t Instr = *(const ulittle32_t *)FixupContent;
178       if (!aarch64::isLoadStoreImm12(Instr) ||
179           aarch64::getPageOffset12Shift(Instr) != 0)
180         return make_error<JITLinkError>(
181             "R_AARCH64_LDST8_ABS_LO12_NC target is not a "
182             "LDRB/STRB (imm12) instruction");
183 
184       Kind = aarch64::PageOffset12;
185       break;
186     }
187     case ELFLdSt16Abs12: {
188       uint32_t Instr = *(const ulittle32_t *)FixupContent;
189       if (!aarch64::isLoadStoreImm12(Instr) ||
190           aarch64::getPageOffset12Shift(Instr) != 1)
191         return make_error<JITLinkError>(
192             "R_AARCH64_LDST16_ABS_LO12_NC target is not a "
193             "LDRH/STRH (imm12) instruction");
194 
195       Kind = aarch64::PageOffset12;
196       break;
197     }
198     case ELFLdSt32Abs12: {
199       uint32_t Instr = *(const ulittle32_t *)FixupContent;
200       if (!aarch64::isLoadStoreImm12(Instr) ||
201           aarch64::getPageOffset12Shift(Instr) != 2)
202         return make_error<JITLinkError>(
203             "R_AARCH64_LDST32_ABS_LO12_NC target is not a "
204             "LDR/STR (imm12, 32 bit) instruction");
205 
206       Kind = aarch64::PageOffset12;
207       break;
208     }
209     case ELFLdSt64Abs12: {
210       uint32_t Instr = *(const ulittle32_t *)FixupContent;
211       if (!aarch64::isLoadStoreImm12(Instr) ||
212           aarch64::getPageOffset12Shift(Instr) != 3)
213         return make_error<JITLinkError>(
214             "R_AARCH64_LDST64_ABS_LO12_NC target is not a "
215             "LDR/STR (imm12, 64 bit) instruction");
216 
217       Kind = aarch64::PageOffset12;
218       break;
219     }
220     case ELFLdSt128Abs12: {
221       uint32_t Instr = *(const ulittle32_t *)FixupContent;
222       if (!aarch64::isLoadStoreImm12(Instr) ||
223           aarch64::getPageOffset12Shift(Instr) != 4)
224         return make_error<JITLinkError>(
225             "R_AARCH64_LDST128_ABS_LO12_NC target is not a "
226             "LDR/STR (imm12, 128 bit) instruction");
227 
228       Kind = aarch64::PageOffset12;
229       break;
230     }
231     case ELFMovwAbsG0: {
232       uint32_t Instr = *(const ulittle32_t *)FixupContent;
233       if (!aarch64::isMoveWideImm16(Instr) ||
234           aarch64::getMoveWide16Shift(Instr) != 0)
235         return make_error<JITLinkError>(
236             "R_AARCH64_MOVW_UABS_G0_NC target is not a "
237             "MOVK/MOVZ (imm16, LSL #0) instruction");
238 
239       Kind = aarch64::MoveWide16;
240       break;
241     }
242     case ELFMovwAbsG1: {
243       uint32_t Instr = *(const ulittle32_t *)FixupContent;
244       if (!aarch64::isMoveWideImm16(Instr) ||
245           aarch64::getMoveWide16Shift(Instr) != 16)
246         return make_error<JITLinkError>(
247             "R_AARCH64_MOVW_UABS_G1_NC target is not a "
248             "MOVK/MOVZ (imm16, LSL #16) instruction");
249 
250       Kind = aarch64::MoveWide16;
251       break;
252     }
253     case ELFMovwAbsG2: {
254       uint32_t Instr = *(const ulittle32_t *)FixupContent;
255       if (!aarch64::isMoveWideImm16(Instr) ||
256           aarch64::getMoveWide16Shift(Instr) != 32)
257         return make_error<JITLinkError>(
258             "R_AARCH64_MOVW_UABS_G2_NC target is not a "
259             "MOVK/MOVZ (imm16, LSL #32) instruction");
260 
261       Kind = aarch64::MoveWide16;
262       break;
263     }
264     case ELFMovwAbsG3: {
265       uint32_t Instr = *(const ulittle32_t *)FixupContent;
266       if (!aarch64::isMoveWideImm16(Instr) ||
267           aarch64::getMoveWide16Shift(Instr) != 48)
268         return make_error<JITLinkError>(
269             "R_AARCH64_MOVW_UABS_G3 target is not a "
270             "MOVK/MOVZ (imm16, LSL #48) instruction");
271 
272       Kind = aarch64::MoveWide16;
273       break;
274     }
275     case ELFAbs64: {
276       Kind = aarch64::Pointer64;
277       break;
278     }
279     case ELFPrel32: {
280       Kind = aarch64::Delta32;
281       break;
282     }
283     case ELFPrel64: {
284       Kind = aarch64::Delta64;
285       break;
286     }
287     case ELFAdrGOTPage21: {
288       Kind = aarch64::GOTPage21;
289       break;
290     }
291     case ELFLd64GOTLo12: {
292       Kind = aarch64::GOTPageOffset12;
293       break;
294     }
295     };
296 
297     Edge GE(Kind, Offset, *GraphSymbol, Addend);
298     LLVM_DEBUG({
299       dbgs() << "    ";
300       printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind));
301       dbgs() << "\n";
302     });
303 
304     BlockToFix.addEdge(std::move(GE));
305     return Error::success();
306   }
307 
308   /// Return the string name of the given ELF aarch64 edge kind.
309   const char *getELFAArch64RelocationKindName(Edge::Kind R) {
310     switch (R) {
311     case ELFCall26:
312       return "ELFCall26";
313     case ELFAdrPage21:
314       return "ELFAdrPage21";
315     case ELFAddAbs12:
316       return "ELFAddAbs12";
317     case ELFLdSt8Abs12:
318       return "ELFLdSt8Abs12";
319     case ELFLdSt16Abs12:
320       return "ELFLdSt16Abs12";
321     case ELFLdSt32Abs12:
322       return "ELFLdSt32Abs12";
323     case ELFLdSt64Abs12:
324       return "ELFLdSt64Abs12";
325     case ELFLdSt128Abs12:
326       return "ELFLdSt128Abs12";
327     case ELFMovwAbsG0:
328       return "ELFMovwAbsG0";
329     case ELFMovwAbsG1:
330       return "ELFMovwAbsG1";
331     case ELFMovwAbsG2:
332       return "ELFMovwAbsG2";
333     case ELFMovwAbsG3:
334       return "ELFMovwAbsG3";
335     case ELFAbs64:
336       return "ELFAbs64";
337     case ELFPrel32:
338       return "ELFPrel32";
339     case ELFPrel64:
340       return "ELFPrel64";
341     case ELFAdrGOTPage21:
342       return "ELFAdrGOTPage21";
343     case ELFLd64GOTLo12:
344       return "ELFLd64GOTLo12";
345     default:
346       return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
347     }
348   }
349 
350 public:
351   ELFLinkGraphBuilder_aarch64(StringRef FileName,
352                               const object::ELFFile<ELFT> &Obj, const Triple T)
353       : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName,
354                                   aarch64::getEdgeKindName) {}
355 };
356 
357 Error buildTables_ELF_aarch64(LinkGraph &G) {
358   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
359 
360   aarch64::GOTTableManager GOT;
361   aarch64::PLTTableManager PLT(GOT);
362   visitExistingEdges(G, GOT, PLT);
363   return Error::success();
364 }
365 
366 } // namespace
367 
368 namespace llvm {
369 namespace jitlink {
370 
371 Expected<std::unique_ptr<LinkGraph>>
372 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
373   LLVM_DEBUG({
374     dbgs() << "Building jitlink graph for new input "
375            << ObjectBuffer.getBufferIdentifier() << "...\n";
376   });
377 
378   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
379   if (!ELFObj)
380     return ELFObj.takeError();
381 
382   assert((*ELFObj)->getArch() == Triple::aarch64 &&
383          "Only AArch64 (little endian) is supported for now");
384 
385   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
386   return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(),
387                                                       ELFObjFile.getELFFile(),
388                                                       (*ELFObj)->makeTriple())
389       .buildGraph();
390 }
391 
392 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
393                       std::unique_ptr<JITLinkContext> Ctx) {
394   PassConfiguration Config;
395   const Triple &TT = G->getTargetTriple();
396   if (Ctx->shouldAddDefaultTargetPasses(TT)) {
397     // Add eh-frame passses.
398     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
399     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
400         ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64,
401         aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32));
402 
403     // Add a mark-live pass.
404     if (auto MarkLive = Ctx->getMarkLivePass(TT))
405       Config.PrePrunePasses.push_back(std::move(MarkLive));
406     else
407       Config.PrePrunePasses.push_back(markAllSymbolsLive);
408 
409     // Add an in-place GOT/Stubs build pass.
410     Config.PostPrunePasses.push_back(buildTables_ELF_aarch64);
411   }
412 
413   if (auto Err = Ctx->modifyPassConfig(*G, Config))
414     return Ctx->notifyFailed(std::move(Err));
415 
416   ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config));
417 }
418 
419 } // namespace jitlink
420 } // namespace llvm
421