1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/aarch64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" 14 #include "EHFrameSupportImpl.h" 15 #include "ELFLinkGraphBuilder.h" 16 #include "JITLinkGeneric.h" 17 #include "llvm/BinaryFormat/ELF.h" 18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" 19 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 20 #include "llvm/Object/ELFObjectFile.h" 21 #include "llvm/Support/Endian.h" 22 23 #define DEBUG_TYPE "jitlink" 24 25 using namespace llvm; 26 using namespace llvm::jitlink; 27 28 namespace { 29 30 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> { 31 friend class JITLinker<ELFJITLinker_aarch64>; 32 33 public: 34 ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx, 35 std::unique_ptr<LinkGraph> G, 36 PassConfiguration PassConfig) 37 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} 38 39 private: 40 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 41 return aarch64::applyFixup(G, B, E); 42 } 43 }; 44 45 template <typename ELFT> 46 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> { 47 private: 48 enum ELFAArch64RelocationKind : Edge::Kind { 49 ELFCall26 = Edge::FirstRelocation, 50 ELFAdrPage21, 51 ELFAddAbs12, 52 ELFLdSt8Abs12, 53 ELFLdSt16Abs12, 54 ELFLdSt32Abs12, 55 ELFLdSt64Abs12, 56 ELFLdSt128Abs12, 57 ELFMovwAbsG0, 58 ELFMovwAbsG1, 59 ELFMovwAbsG2, 60 ELFMovwAbsG3, 61 ELFAbs64, 62 ELFPrel32, 63 ELFPrel64, 64 ELFAdrGOTPage21, 65 ELFLd64GOTLo12, 66 }; 67 68 static Expected<ELFAArch64RelocationKind> 69 getRelocationKind(const uint32_t Type) { 70 using namespace aarch64; 71 switch (Type) { 72 case ELF::R_AARCH64_CALL26: 73 case ELF::R_AARCH64_JUMP26: 74 return ELFCall26; 75 case ELF::R_AARCH64_ADR_PREL_PG_HI21: 76 return ELFAdrPage21; 77 case ELF::R_AARCH64_ADD_ABS_LO12_NC: 78 return ELFAddAbs12; 79 case ELF::R_AARCH64_LDST8_ABS_LO12_NC: 80 return ELFLdSt8Abs12; 81 case ELF::R_AARCH64_LDST16_ABS_LO12_NC: 82 return ELFLdSt16Abs12; 83 case ELF::R_AARCH64_LDST32_ABS_LO12_NC: 84 return ELFLdSt32Abs12; 85 case ELF::R_AARCH64_LDST64_ABS_LO12_NC: 86 return ELFLdSt64Abs12; 87 case ELF::R_AARCH64_LDST128_ABS_LO12_NC: 88 return ELFLdSt128Abs12; 89 case ELF::R_AARCH64_MOVW_UABS_G0_NC: 90 return ELFMovwAbsG0; 91 case ELF::R_AARCH64_MOVW_UABS_G1_NC: 92 return ELFMovwAbsG1; 93 case ELF::R_AARCH64_MOVW_UABS_G2_NC: 94 return ELFMovwAbsG2; 95 case ELF::R_AARCH64_MOVW_UABS_G3: 96 return ELFMovwAbsG3; 97 case ELF::R_AARCH64_ABS64: 98 return ELFAbs64; 99 case ELF::R_AARCH64_PREL32: 100 return ELFPrel32; 101 case ELF::R_AARCH64_PREL64: 102 return ELFPrel64; 103 case ELF::R_AARCH64_ADR_GOT_PAGE: 104 return ELFAdrGOTPage21; 105 case ELF::R_AARCH64_LD64_GOT_LO12_NC: 106 return ELFLd64GOTLo12; 107 } 108 109 return make_error<JITLinkError>( 110 "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) + 111 object::getELFRelocationTypeName(ELF::EM_AARCH64, Type)); 112 } 113 114 Error addRelocations() override { 115 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 116 117 using Base = ELFLinkGraphBuilder<ELFT>; 118 using Self = ELFLinkGraphBuilder_aarch64<ELFT>; 119 for (const auto &RelSect : Base::Sections) 120 if (Error Err = Base::forEachRelocation(RelSect, this, 121 &Self::addSingleRelocation)) 122 return Err; 123 124 return Error::success(); 125 } 126 127 Error addSingleRelocation(const typename ELFT::Rela &Rel, 128 const typename ELFT::Shdr &FixupSect, 129 Block &BlockToFix) { 130 using support::ulittle32_t; 131 using Base = ELFLinkGraphBuilder<ELFT>; 132 133 uint32_t SymbolIndex = Rel.getSymbol(false); 134 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 135 if (!ObjSymbol) 136 return ObjSymbol.takeError(); 137 138 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 139 if (!GraphSymbol) 140 return make_error<StringError>( 141 formatv("Could not find symbol at given index, did you add it to " 142 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 143 SymbolIndex, (*ObjSymbol)->st_shndx, 144 Base::GraphSymbols.size()), 145 inconvertibleErrorCode()); 146 147 uint32_t Type = Rel.getType(false); 148 Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type); 149 if (!RelocKind) 150 return RelocKind.takeError(); 151 152 int64_t Addend = Rel.r_addend; 153 orc::ExecutorAddr FixupAddress = 154 orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; 155 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 156 157 // Get a pointer to the fixup content. 158 const void *FixupContent = BlockToFix.getContent().data() + 159 (FixupAddress - BlockToFix.getAddress()); 160 161 Edge::Kind Kind = Edge::Invalid; 162 163 switch (*RelocKind) { 164 case ELFCall26: { 165 Kind = aarch64::Branch26; 166 break; 167 } 168 case ELFAdrPage21: { 169 Kind = aarch64::Page21; 170 break; 171 } 172 case ELFAddAbs12: { 173 Kind = aarch64::PageOffset12; 174 break; 175 } 176 case ELFLdSt8Abs12: { 177 uint32_t Instr = *(const ulittle32_t *)FixupContent; 178 if (!aarch64::isLoadStoreImm12(Instr) || 179 aarch64::getPageOffset12Shift(Instr) != 0) 180 return make_error<JITLinkError>( 181 "R_AARCH64_LDST8_ABS_LO12_NC target is not a " 182 "LDRB/STRB (imm12) instruction"); 183 184 Kind = aarch64::PageOffset12; 185 break; 186 } 187 case ELFLdSt16Abs12: { 188 uint32_t Instr = *(const ulittle32_t *)FixupContent; 189 if (!aarch64::isLoadStoreImm12(Instr) || 190 aarch64::getPageOffset12Shift(Instr) != 1) 191 return make_error<JITLinkError>( 192 "R_AARCH64_LDST16_ABS_LO12_NC target is not a " 193 "LDRH/STRH (imm12) instruction"); 194 195 Kind = aarch64::PageOffset12; 196 break; 197 } 198 case ELFLdSt32Abs12: { 199 uint32_t Instr = *(const ulittle32_t *)FixupContent; 200 if (!aarch64::isLoadStoreImm12(Instr) || 201 aarch64::getPageOffset12Shift(Instr) != 2) 202 return make_error<JITLinkError>( 203 "R_AARCH64_LDST32_ABS_LO12_NC target is not a " 204 "LDR/STR (imm12, 32 bit) instruction"); 205 206 Kind = aarch64::PageOffset12; 207 break; 208 } 209 case ELFLdSt64Abs12: { 210 uint32_t Instr = *(const ulittle32_t *)FixupContent; 211 if (!aarch64::isLoadStoreImm12(Instr) || 212 aarch64::getPageOffset12Shift(Instr) != 3) 213 return make_error<JITLinkError>( 214 "R_AARCH64_LDST64_ABS_LO12_NC target is not a " 215 "LDR/STR (imm12, 64 bit) instruction"); 216 217 Kind = aarch64::PageOffset12; 218 break; 219 } 220 case ELFLdSt128Abs12: { 221 uint32_t Instr = *(const ulittle32_t *)FixupContent; 222 if (!aarch64::isLoadStoreImm12(Instr) || 223 aarch64::getPageOffset12Shift(Instr) != 4) 224 return make_error<JITLinkError>( 225 "R_AARCH64_LDST128_ABS_LO12_NC target is not a " 226 "LDR/STR (imm12, 128 bit) instruction"); 227 228 Kind = aarch64::PageOffset12; 229 break; 230 } 231 case ELFMovwAbsG0: { 232 uint32_t Instr = *(const ulittle32_t *)FixupContent; 233 if (!aarch64::isMoveWideImm16(Instr) || 234 aarch64::getMoveWide16Shift(Instr) != 0) 235 return make_error<JITLinkError>( 236 "R_AARCH64_MOVW_UABS_G0_NC target is not a " 237 "MOVK/MOVZ (imm16, LSL #0) instruction"); 238 239 Kind = aarch64::MoveWide16; 240 break; 241 } 242 case ELFMovwAbsG1: { 243 uint32_t Instr = *(const ulittle32_t *)FixupContent; 244 if (!aarch64::isMoveWideImm16(Instr) || 245 aarch64::getMoveWide16Shift(Instr) != 16) 246 return make_error<JITLinkError>( 247 "R_AARCH64_MOVW_UABS_G1_NC target is not a " 248 "MOVK/MOVZ (imm16, LSL #16) instruction"); 249 250 Kind = aarch64::MoveWide16; 251 break; 252 } 253 case ELFMovwAbsG2: { 254 uint32_t Instr = *(const ulittle32_t *)FixupContent; 255 if (!aarch64::isMoveWideImm16(Instr) || 256 aarch64::getMoveWide16Shift(Instr) != 32) 257 return make_error<JITLinkError>( 258 "R_AARCH64_MOVW_UABS_G2_NC target is not a " 259 "MOVK/MOVZ (imm16, LSL #32) instruction"); 260 261 Kind = aarch64::MoveWide16; 262 break; 263 } 264 case ELFMovwAbsG3: { 265 uint32_t Instr = *(const ulittle32_t *)FixupContent; 266 if (!aarch64::isMoveWideImm16(Instr) || 267 aarch64::getMoveWide16Shift(Instr) != 48) 268 return make_error<JITLinkError>( 269 "R_AARCH64_MOVW_UABS_G3 target is not a " 270 "MOVK/MOVZ (imm16, LSL #48) instruction"); 271 272 Kind = aarch64::MoveWide16; 273 break; 274 } 275 case ELFAbs64: { 276 Kind = aarch64::Pointer64; 277 break; 278 } 279 case ELFPrel32: { 280 Kind = aarch64::Delta32; 281 break; 282 } 283 case ELFPrel64: { 284 Kind = aarch64::Delta64; 285 break; 286 } 287 case ELFAdrGOTPage21: { 288 Kind = aarch64::GOTPage21; 289 break; 290 } 291 case ELFLd64GOTLo12: { 292 Kind = aarch64::GOTPageOffset12; 293 break; 294 } 295 }; 296 297 Edge GE(Kind, Offset, *GraphSymbol, Addend); 298 LLVM_DEBUG({ 299 dbgs() << " "; 300 printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind)); 301 dbgs() << "\n"; 302 }); 303 304 BlockToFix.addEdge(std::move(GE)); 305 return Error::success(); 306 } 307 308 /// Return the string name of the given ELF aarch64 edge kind. 309 const char *getELFAArch64RelocationKindName(Edge::Kind R) { 310 switch (R) { 311 case ELFCall26: 312 return "ELFCall26"; 313 case ELFAdrPage21: 314 return "ELFAdrPage21"; 315 case ELFAddAbs12: 316 return "ELFAddAbs12"; 317 case ELFLdSt8Abs12: 318 return "ELFLdSt8Abs12"; 319 case ELFLdSt16Abs12: 320 return "ELFLdSt16Abs12"; 321 case ELFLdSt32Abs12: 322 return "ELFLdSt32Abs12"; 323 case ELFLdSt64Abs12: 324 return "ELFLdSt64Abs12"; 325 case ELFLdSt128Abs12: 326 return "ELFLdSt128Abs12"; 327 case ELFMovwAbsG0: 328 return "ELFMovwAbsG0"; 329 case ELFMovwAbsG1: 330 return "ELFMovwAbsG1"; 331 case ELFMovwAbsG2: 332 return "ELFMovwAbsG2"; 333 case ELFMovwAbsG3: 334 return "ELFMovwAbsG3"; 335 case ELFAbs64: 336 return "ELFAbs64"; 337 case ELFPrel32: 338 return "ELFPrel32"; 339 case ELFPrel64: 340 return "ELFPrel64"; 341 case ELFAdrGOTPage21: 342 return "ELFAdrGOTPage21"; 343 case ELFLd64GOTLo12: 344 return "ELFLd64GOTLo12"; 345 default: 346 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 347 } 348 } 349 350 public: 351 ELFLinkGraphBuilder_aarch64(StringRef FileName, 352 const object::ELFFile<ELFT> &Obj, const Triple T) 353 : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, 354 aarch64::getEdgeKindName) {} 355 }; 356 357 Error buildTables_ELF_aarch64(LinkGraph &G) { 358 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); 359 360 aarch64::GOTTableManager GOT; 361 aarch64::PLTTableManager PLT(GOT); 362 visitExistingEdges(G, GOT, PLT); 363 return Error::success(); 364 } 365 366 } // namespace 367 368 namespace llvm { 369 namespace jitlink { 370 371 Expected<std::unique_ptr<LinkGraph>> 372 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { 373 LLVM_DEBUG({ 374 dbgs() << "Building jitlink graph for new input " 375 << ObjectBuffer.getBufferIdentifier() << "...\n"; 376 }); 377 378 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 379 if (!ELFObj) 380 return ELFObj.takeError(); 381 382 assert((*ELFObj)->getArch() == Triple::aarch64 && 383 "Only AArch64 (little endian) is supported for now"); 384 385 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 386 return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(), 387 ELFObjFile.getELFFile(), 388 (*ELFObj)->makeTriple()) 389 .buildGraph(); 390 } 391 392 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, 393 std::unique_ptr<JITLinkContext> Ctx) { 394 PassConfiguration Config; 395 const Triple &TT = G->getTargetTriple(); 396 if (Ctx->shouldAddDefaultTargetPasses(TT)) { 397 // Add eh-frame passses. 398 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); 399 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 400 ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, 401 aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); 402 403 // Add a mark-live pass. 404 if (auto MarkLive = Ctx->getMarkLivePass(TT)) 405 Config.PrePrunePasses.push_back(std::move(MarkLive)); 406 else 407 Config.PrePrunePasses.push_back(markAllSymbolsLive); 408 409 // Add an in-place GOT/Stubs build pass. 410 Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); 411 } 412 413 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 414 return Ctx->notifyFailed(std::move(Err)); 415 416 ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config)); 417 } 418 419 } // namespace jitlink 420 } // namespace llvm 421