1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/aarch64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
14 #include "EHFrameSupportImpl.h"
15 #include "ELFLinkGraphBuilder.h"
16 #include "JITLinkGeneric.h"
17 #include "llvm/BinaryFormat/ELF.h"
18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
19 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
20 #include "llvm/Object/ELFObjectFile.h"
21 #include "llvm/Support/Endian.h"
22 
23 #define DEBUG_TYPE "jitlink"
24 
25 using namespace llvm;
26 using namespace llvm::jitlink;
27 
28 namespace {
29 
30 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> {
31   friend class JITLinker<ELFJITLinker_aarch64>;
32 
33 public:
34   ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx,
35                        std::unique_ptr<LinkGraph> G,
36                        PassConfiguration PassConfig)
37       : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}
38 
39 private:
40   Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
41     return aarch64::applyFixup(G, B, E);
42   }
43 };
44 
45 template <typename ELFT>
46 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
47 private:
48   enum ELFAArch64RelocationKind : Edge::Kind {
49     ELFCall26 = Edge::FirstRelocation,
50     ELFAdrPage21,
51     ELFAddAbs12,
52     ELFLdSt8Abs12,
53     ELFLdSt16Abs12,
54     ELFLdSt32Abs12,
55     ELFLdSt64Abs12,
56     ELFLdSt128Abs12,
57     ELFMovwAbsG0,
58     ELFMovwAbsG1,
59     ELFMovwAbsG2,
60     ELFMovwAbsG3,
61     ELFAbs64,
62     ELFPrel32,
63     ELFPrel64,
64     ELFAdrGOTPage21,
65     ELFLd64GOTLo12,
66     ELFTLSDescAdrPage21,
67     ELFTLSDescAddLo12,
68     ELFTLSDescLd64Lo12,
69     ELFTLSDescCall,
70   };
71 
72   static Expected<ELFAArch64RelocationKind>
73   getRelocationKind(const uint32_t Type) {
74     using namespace aarch64;
75     switch (Type) {
76     case ELF::R_AARCH64_CALL26:
77     case ELF::R_AARCH64_JUMP26:
78       return ELFCall26;
79     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
80       return ELFAdrPage21;
81     case ELF::R_AARCH64_ADD_ABS_LO12_NC:
82       return ELFAddAbs12;
83     case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
84       return ELFLdSt8Abs12;
85     case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
86       return ELFLdSt16Abs12;
87     case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
88       return ELFLdSt32Abs12;
89     case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
90       return ELFLdSt64Abs12;
91     case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
92       return ELFLdSt128Abs12;
93     case ELF::R_AARCH64_MOVW_UABS_G0_NC:
94       return ELFMovwAbsG0;
95     case ELF::R_AARCH64_MOVW_UABS_G1_NC:
96       return ELFMovwAbsG1;
97     case ELF::R_AARCH64_MOVW_UABS_G2_NC:
98       return ELFMovwAbsG2;
99     case ELF::R_AARCH64_MOVW_UABS_G3:
100       return ELFMovwAbsG3;
101     case ELF::R_AARCH64_ABS64:
102       return ELFAbs64;
103     case ELF::R_AARCH64_PREL32:
104       return ELFPrel32;
105     case ELF::R_AARCH64_PREL64:
106       return ELFPrel64;
107     case ELF::R_AARCH64_ADR_GOT_PAGE:
108       return ELFAdrGOTPage21;
109     case ELF::R_AARCH64_LD64_GOT_LO12_NC:
110       return ELFLd64GOTLo12;
111     case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:
112       return ELFTLSDescAdrPage21;
113     case ELF::R_AARCH64_TLSDESC_ADD_LO12:
114       return ELFTLSDescAddLo12;
115     case ELF::R_AARCH64_TLSDESC_LD64_LO12:
116       return ELFTLSDescLd64Lo12;
117     case ELF::R_AARCH64_TLSDESC_CALL:
118       return ELFTLSDescCall;
119     }
120 
121     return make_error<JITLinkError>(
122         "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) +
123         object::getELFRelocationTypeName(ELF::EM_AARCH64, Type));
124   }
125 
126   Error addRelocations() override {
127     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
128 
129     using Base = ELFLinkGraphBuilder<ELFT>;
130     using Self = ELFLinkGraphBuilder_aarch64<ELFT>;
131     for (const auto &RelSect : Base::Sections)
132       if (Error Err = Base::forEachRelaRelocation(RelSect, this,
133                                                   &Self::addSingleRelocation))
134         return Err;
135 
136     return Error::success();
137   }
138 
139   Error addSingleRelocation(const typename ELFT::Rela &Rel,
140                             const typename ELFT::Shdr &FixupSect,
141                             Block &BlockToFix) {
142     using support::ulittle32_t;
143     using Base = ELFLinkGraphBuilder<ELFT>;
144 
145     uint32_t SymbolIndex = Rel.getSymbol(false);
146     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
147     if (!ObjSymbol)
148       return ObjSymbol.takeError();
149 
150     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
151     if (!GraphSymbol)
152       return make_error<StringError>(
153           formatv("Could not find symbol at given index, did you add it to "
154                   "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
155                   SymbolIndex, (*ObjSymbol)->st_shndx,
156                   Base::GraphSymbols.size()),
157           inconvertibleErrorCode());
158 
159     uint32_t Type = Rel.getType(false);
160     Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type);
161     if (!RelocKind)
162       return RelocKind.takeError();
163 
164     int64_t Addend = Rel.r_addend;
165     orc::ExecutorAddr FixupAddress =
166         orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
167     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
168 
169     // Get a pointer to the fixup content.
170     const void *FixupContent = BlockToFix.getContent().data() +
171                                (FixupAddress - BlockToFix.getAddress());
172 
173     Edge::Kind Kind = Edge::Invalid;
174 
175     switch (*RelocKind) {
176     case ELFCall26: {
177       Kind = aarch64::Branch26PCRel;
178       break;
179     }
180     case ELFAdrPage21: {
181       Kind = aarch64::Page21;
182       break;
183     }
184     case ELFAddAbs12: {
185       Kind = aarch64::PageOffset12;
186       break;
187     }
188     case ELFLdSt8Abs12: {
189       uint32_t Instr = *(const ulittle32_t *)FixupContent;
190       if (!aarch64::isLoadStoreImm12(Instr) ||
191           aarch64::getPageOffset12Shift(Instr) != 0)
192         return make_error<JITLinkError>(
193             "R_AARCH64_LDST8_ABS_LO12_NC target is not a "
194             "LDRB/STRB (imm12) instruction");
195 
196       Kind = aarch64::PageOffset12;
197       break;
198     }
199     case ELFLdSt16Abs12: {
200       uint32_t Instr = *(const ulittle32_t *)FixupContent;
201       if (!aarch64::isLoadStoreImm12(Instr) ||
202           aarch64::getPageOffset12Shift(Instr) != 1)
203         return make_error<JITLinkError>(
204             "R_AARCH64_LDST16_ABS_LO12_NC target is not a "
205             "LDRH/STRH (imm12) instruction");
206 
207       Kind = aarch64::PageOffset12;
208       break;
209     }
210     case ELFLdSt32Abs12: {
211       uint32_t Instr = *(const ulittle32_t *)FixupContent;
212       if (!aarch64::isLoadStoreImm12(Instr) ||
213           aarch64::getPageOffset12Shift(Instr) != 2)
214         return make_error<JITLinkError>(
215             "R_AARCH64_LDST32_ABS_LO12_NC target is not a "
216             "LDR/STR (imm12, 32 bit) instruction");
217 
218       Kind = aarch64::PageOffset12;
219       break;
220     }
221     case ELFLdSt64Abs12: {
222       uint32_t Instr = *(const ulittle32_t *)FixupContent;
223       if (!aarch64::isLoadStoreImm12(Instr) ||
224           aarch64::getPageOffset12Shift(Instr) != 3)
225         return make_error<JITLinkError>(
226             "R_AARCH64_LDST64_ABS_LO12_NC target is not a "
227             "LDR/STR (imm12, 64 bit) instruction");
228 
229       Kind = aarch64::PageOffset12;
230       break;
231     }
232     case ELFLdSt128Abs12: {
233       uint32_t Instr = *(const ulittle32_t *)FixupContent;
234       if (!aarch64::isLoadStoreImm12(Instr) ||
235           aarch64::getPageOffset12Shift(Instr) != 4)
236         return make_error<JITLinkError>(
237             "R_AARCH64_LDST128_ABS_LO12_NC target is not a "
238             "LDR/STR (imm12, 128 bit) instruction");
239 
240       Kind = aarch64::PageOffset12;
241       break;
242     }
243     case ELFMovwAbsG0: {
244       uint32_t Instr = *(const ulittle32_t *)FixupContent;
245       if (!aarch64::isMoveWideImm16(Instr) ||
246           aarch64::getMoveWide16Shift(Instr) != 0)
247         return make_error<JITLinkError>(
248             "R_AARCH64_MOVW_UABS_G0_NC target is not a "
249             "MOVK/MOVZ (imm16, LSL #0) instruction");
250 
251       Kind = aarch64::MoveWide16;
252       break;
253     }
254     case ELFMovwAbsG1: {
255       uint32_t Instr = *(const ulittle32_t *)FixupContent;
256       if (!aarch64::isMoveWideImm16(Instr) ||
257           aarch64::getMoveWide16Shift(Instr) != 16)
258         return make_error<JITLinkError>(
259             "R_AARCH64_MOVW_UABS_G1_NC target is not a "
260             "MOVK/MOVZ (imm16, LSL #16) instruction");
261 
262       Kind = aarch64::MoveWide16;
263       break;
264     }
265     case ELFMovwAbsG2: {
266       uint32_t Instr = *(const ulittle32_t *)FixupContent;
267       if (!aarch64::isMoveWideImm16(Instr) ||
268           aarch64::getMoveWide16Shift(Instr) != 32)
269         return make_error<JITLinkError>(
270             "R_AARCH64_MOVW_UABS_G2_NC target is not a "
271             "MOVK/MOVZ (imm16, LSL #32) instruction");
272 
273       Kind = aarch64::MoveWide16;
274       break;
275     }
276     case ELFMovwAbsG3: {
277       uint32_t Instr = *(const ulittle32_t *)FixupContent;
278       if (!aarch64::isMoveWideImm16(Instr) ||
279           aarch64::getMoveWide16Shift(Instr) != 48)
280         return make_error<JITLinkError>(
281             "R_AARCH64_MOVW_UABS_G3 target is not a "
282             "MOVK/MOVZ (imm16, LSL #48) instruction");
283 
284       Kind = aarch64::MoveWide16;
285       break;
286     }
287     case ELFAbs64: {
288       Kind = aarch64::Pointer64;
289       break;
290     }
291     case ELFPrel32: {
292       Kind = aarch64::Delta32;
293       break;
294     }
295     case ELFPrel64: {
296       Kind = aarch64::Delta64;
297       break;
298     }
299     case ELFAdrGOTPage21: {
300       Kind = aarch64::RequestGOTAndTransformToPage21;
301       break;
302     }
303     case ELFLd64GOTLo12: {
304       Kind = aarch64::RequestGOTAndTransformToPageOffset12;
305       break;
306     }
307     case ELFTLSDescAdrPage21: {
308       Kind = aarch64::RequestTLSDescEntryAndTransformToPage21;
309       break;
310     }
311     case ELFTLSDescAddLo12:
312     case ELFTLSDescLd64Lo12: {
313       Kind = aarch64::RequestTLSDescEntryAndTransformToPageOffset12;
314       break;
315     }
316     case ELFTLSDescCall: {
317       return Error::success();
318     }
319     };
320 
321     Edge GE(Kind, Offset, *GraphSymbol, Addend);
322     LLVM_DEBUG({
323       dbgs() << "    ";
324       printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind));
325       dbgs() << "\n";
326     });
327 
328     BlockToFix.addEdge(std::move(GE));
329 
330     return Error::success();
331   }
332 
333   /// Return the string name of the given ELF aarch64 edge kind.
334   const char *getELFAArch64RelocationKindName(Edge::Kind R) {
335     switch (R) {
336     case ELFCall26:
337       return "ELFCall26";
338     case ELFAdrPage21:
339       return "ELFAdrPage21";
340     case ELFAddAbs12:
341       return "ELFAddAbs12";
342     case ELFLdSt8Abs12:
343       return "ELFLdSt8Abs12";
344     case ELFLdSt16Abs12:
345       return "ELFLdSt16Abs12";
346     case ELFLdSt32Abs12:
347       return "ELFLdSt32Abs12";
348     case ELFLdSt64Abs12:
349       return "ELFLdSt64Abs12";
350     case ELFLdSt128Abs12:
351       return "ELFLdSt128Abs12";
352     case ELFMovwAbsG0:
353       return "ELFMovwAbsG0";
354     case ELFMovwAbsG1:
355       return "ELFMovwAbsG1";
356     case ELFMovwAbsG2:
357       return "ELFMovwAbsG2";
358     case ELFMovwAbsG3:
359       return "ELFMovwAbsG3";
360     case ELFAbs64:
361       return "ELFAbs64";
362     case ELFPrel32:
363       return "ELFPrel32";
364     case ELFPrel64:
365       return "ELFPrel64";
366     case ELFAdrGOTPage21:
367       return "ELFAdrGOTPage21";
368     case ELFLd64GOTLo12:
369       return "ELFLd64GOTLo12";
370     case ELFTLSDescAdrPage21:
371       return "ELFTLSDescAdrPage21";
372     case ELFTLSDescAddLo12:
373       return "ELFTLSDescAddLo12";
374     case ELFTLSDescLd64Lo12:
375       return "ELFTLSDescLd64Lo12";
376     case ELFTLSDescCall:
377       return "ELFTLSDescCall";
378     default:
379       return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
380     }
381   }
382 
383 public:
384   ELFLinkGraphBuilder_aarch64(StringRef FileName,
385                               const object::ELFFile<ELFT> &Obj, const Triple T)
386       : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName,
387                                   aarch64::getEdgeKindName) {}
388 };
389 
390 // TLS Info Builder.
391 class TLSInfoTableManager_ELF_aarch64
392     : public TableManager<TLSInfoTableManager_ELF_aarch64> {
393 public:
394   static StringRef getSectionName() { return "$__TLSINFO"; }
395 
396   static const uint8_t TLSInfoEntryContent[16];
397 
398   bool visitEdge(LinkGraph &G, Block *B, Edge &E) { return false; }
399 
400   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
401     // the TLS Info entry's key value will be written by the fixTLVSectionByName
402     // pass, so create mutable content.
403     auto &TLSInfoEntry = G.createMutableContentBlock(
404         getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
405         orc::ExecutorAddr(), 8, 0);
406     TLSInfoEntry.addEdge(aarch64::Pointer64, 8, Target, 0);
407     return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
408   }
409 
410 private:
411   Section &getTLSInfoSection(LinkGraph &G) {
412     if (!TLSInfoTable)
413       TLSInfoTable = &G.createSection(getSectionName(), orc::MemProt::Read);
414     return *TLSInfoTable;
415   }
416 
417   ArrayRef<char> getTLSInfoEntryContent() const {
418     return {reinterpret_cast<const char *>(TLSInfoEntryContent),
419             sizeof(TLSInfoEntryContent)};
420   }
421 
422   Section *TLSInfoTable = nullptr;
423 };
424 
425 const uint8_t TLSInfoTableManager_ELF_aarch64::TLSInfoEntryContent[16] = {
426     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
427     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00  /*data address*/
428 };
429 
430 // TLS Descriptor Builder.
431 class TLSDescTableManager_ELF_aarch64
432     : public TableManager<TLSDescTableManager_ELF_aarch64> {
433 public:
434   TLSDescTableManager_ELF_aarch64(
435       TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager)
436       : TLSInfoTableManager(TLSInfoTableManager) {}
437 
438   static StringRef getSectionName() { return "$__TLSDESC"; }
439 
440   static const uint8_t TLSDescEntryContent[16];
441 
442   bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
443     Edge::Kind KindToSet = Edge::Invalid;
444     switch (E.getKind()) {
445     case aarch64::RequestTLSDescEntryAndTransformToPage21: {
446       KindToSet = aarch64::Page21;
447       break;
448     }
449     case aarch64::RequestTLSDescEntryAndTransformToPageOffset12: {
450       KindToSet = aarch64::PageOffset12;
451       break;
452     }
453     default:
454       return false;
455     }
456     assert(KindToSet != Edge::Invalid &&
457            "Fell through switch, but no new kind to set");
458     DEBUG_WITH_TYPE("jitlink", {
459       dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
460              << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
461              << formatv("{0:x}", E.getOffset()) << ")\n";
462     });
463     E.setKind(KindToSet);
464     E.setTarget(getEntryForTarget(G, E.getTarget()));
465     return true;
466   }
467 
468   Symbol &createEntry(LinkGraph &G, Symbol &Target) {
469     auto &EntryBlock =
470         G.createContentBlock(getTLSDescSection(G), getTLSDescBlockContent(),
471                              orc::ExecutorAddr(), 8, 0);
472     EntryBlock.addEdge(aarch64::Pointer64, 0, getTLSDescResolver(G), 0);
473     EntryBlock.addEdge(aarch64::Pointer64, 8,
474                        TLSInfoTableManager.getEntryForTarget(G, Target), 0);
475     return G.addAnonymousSymbol(EntryBlock, 0, 8, false, false);
476   }
477 
478 private:
479   Section &getTLSDescSection(LinkGraph &G) {
480     if (!GOTSection)
481       GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);
482     return *GOTSection;
483   }
484 
485   Symbol &getTLSDescResolver(LinkGraph &G) {
486     if (!TLSDescResolver)
487       TLSDescResolver = &G.addExternalSymbol("__tlsdesc_resolver", 8, false);
488     return *TLSDescResolver;
489   }
490 
491   ArrayRef<char> getTLSDescBlockContent() {
492     return {reinterpret_cast<const char *>(TLSDescEntryContent),
493             sizeof(TLSDescEntryContent)};
494   }
495 
496   Section *GOTSection = nullptr;
497   Symbol *TLSDescResolver = nullptr;
498   TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager;
499 };
500 
501 const uint8_t TLSDescTableManager_ELF_aarch64::TLSDescEntryContent[16] = {
502     0x00, 0x00, 0x00, 0x00,
503     0x00, 0x00, 0x00, 0x00, /*resolver function pointer*/
504     0x00, 0x00, 0x00, 0x00,
505     0x00, 0x00, 0x00, 0x00 /*pointer to tls info*/
506 };
507 
508 Error buildTables_ELF_aarch64(LinkGraph &G) {
509   LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
510 
511   aarch64::GOTTableManager GOT;
512   aarch64::PLTTableManager PLT(GOT);
513   TLSInfoTableManager_ELF_aarch64 TLSInfo;
514   TLSDescTableManager_ELF_aarch64 TLSDesc(TLSInfo);
515   visitExistingEdges(G, GOT, PLT, TLSDesc, TLSInfo);
516   return Error::success();
517 }
518 
519 } // namespace
520 
521 namespace llvm {
522 namespace jitlink {
523 
524 Expected<std::unique_ptr<LinkGraph>>
525 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) {
526   LLVM_DEBUG({
527     dbgs() << "Building jitlink graph for new input "
528            << ObjectBuffer.getBufferIdentifier() << "...\n";
529   });
530 
531   auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
532   if (!ELFObj)
533     return ELFObj.takeError();
534 
535   assert((*ELFObj)->getArch() == Triple::aarch64 &&
536          "Only AArch64 (little endian) is supported for now");
537 
538   auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
539   return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(),
540                                                       ELFObjFile.getELFFile(),
541                                                       (*ELFObj)->makeTriple())
542       .buildGraph();
543 }
544 
545 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G,
546                       std::unique_ptr<JITLinkContext> Ctx) {
547   PassConfiguration Config;
548   const Triple &TT = G->getTargetTriple();
549   if (Ctx->shouldAddDefaultTargetPasses(TT)) {
550     // Add eh-frame passses.
551     Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
552     Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
553         ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64,
554         aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32));
555     Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
556 
557     // Add a mark-live pass.
558     if (auto MarkLive = Ctx->getMarkLivePass(TT))
559       Config.PrePrunePasses.push_back(std::move(MarkLive));
560     else
561       Config.PrePrunePasses.push_back(markAllSymbolsLive);
562 
563     // Add an in-place GOT/TLS/Stubs build pass.
564     Config.PostPrunePasses.push_back(buildTables_ELF_aarch64);
565   }
566 
567   if (auto Err = Ctx->modifyPassConfig(*G, Config))
568     return Ctx->notifyFailed(std::move(Err));
569 
570   ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config));
571 }
572 
573 } // namespace jitlink
574 } // namespace llvm
575