1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Symbols.h" 10 #include "SyntheticSections.h" 11 #include "Target.h" 12 #include "Thunks.h" 13 #include "lld/Common/ErrorHandler.h" 14 #include "llvm/Object/ELF.h" 15 #include "llvm/Support/Endian.h" 16 17 using namespace llvm; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 21 namespace lld { 22 namespace elf { 23 24 // Page(Expr) is the page address of the expression Expr, defined 25 // as (Expr & ~0xFFF). (This applies even if the machine page size 26 // supported by the platform has a different value.) 27 uint64_t getAArch64Page(uint64_t expr) { 28 return expr & ~static_cast<uint64_t>(0xFFF); 29 } 30 31 namespace { 32 class AArch64 : public TargetInfo { 33 public: 34 AArch64(); 35 RelExpr getRelExpr(RelType type, const Symbol &s, 36 const uint8_t *loc) const override; 37 RelType getDynRel(RelType type) const override; 38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 39 void writePltHeader(uint8_t *buf) const override; 40 void writePlt(uint8_t *buf, const Symbol &sym, 41 uint64_t pltEntryAddr) const override; 42 bool needsThunk(RelExpr expr, RelType type, const InputFile *file, 43 uint64_t branchAddr, const Symbol &s, 44 int64_t a) const override; 45 uint32_t getThunkSectionSpacing() const override; 46 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; 47 bool usesOnlyLowPageBits(RelType type) const override; 48 void relocateOne(uint8_t *loc, RelType type, uint64_t val) const override; 49 RelExpr adjustRelaxExpr(RelType type, const uint8_t *data, 50 RelExpr expr) const override; 51 void relaxTlsGdToLe(uint8_t *loc, RelType type, uint64_t val) const override; 52 void relaxTlsGdToIe(uint8_t *loc, RelType type, uint64_t val) const override; 53 void relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const override; 54 }; 55 } // namespace 56 57 AArch64::AArch64() { 58 copyRel = R_AARCH64_COPY; 59 relativeRel = R_AARCH64_RELATIVE; 60 iRelativeRel = R_AARCH64_IRELATIVE; 61 gotRel = R_AARCH64_GLOB_DAT; 62 noneRel = R_AARCH64_NONE; 63 pltRel = R_AARCH64_JUMP_SLOT; 64 symbolicRel = R_AARCH64_ABS64; 65 tlsDescRel = R_AARCH64_TLSDESC; 66 tlsGotRel = R_AARCH64_TLS_TPREL64; 67 pltHeaderSize = 32; 68 pltEntrySize = 16; 69 ipltEntrySize = 16; 70 defaultMaxPageSize = 65536; 71 72 // Align to the 2 MiB page size (known as a superpage or huge page). 73 // FreeBSD automatically promotes 2 MiB-aligned allocations. 74 defaultImageBase = 0x200000; 75 76 needsThunks = true; 77 } 78 79 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, 80 const uint8_t *loc) const { 81 switch (type) { 82 case R_AARCH64_ABS16: 83 case R_AARCH64_ABS32: 84 case R_AARCH64_ABS64: 85 case R_AARCH64_ADD_ABS_LO12_NC: 86 case R_AARCH64_LDST128_ABS_LO12_NC: 87 case R_AARCH64_LDST16_ABS_LO12_NC: 88 case R_AARCH64_LDST32_ABS_LO12_NC: 89 case R_AARCH64_LDST64_ABS_LO12_NC: 90 case R_AARCH64_LDST8_ABS_LO12_NC: 91 case R_AARCH64_MOVW_SABS_G0: 92 case R_AARCH64_MOVW_SABS_G1: 93 case R_AARCH64_MOVW_SABS_G2: 94 case R_AARCH64_MOVW_UABS_G0: 95 case R_AARCH64_MOVW_UABS_G0_NC: 96 case R_AARCH64_MOVW_UABS_G1: 97 case R_AARCH64_MOVW_UABS_G1_NC: 98 case R_AARCH64_MOVW_UABS_G2: 99 case R_AARCH64_MOVW_UABS_G2_NC: 100 case R_AARCH64_MOVW_UABS_G3: 101 return R_ABS; 102 case R_AARCH64_TLSDESC_ADR_PAGE21: 103 return R_AARCH64_TLSDESC_PAGE; 104 case R_AARCH64_TLSDESC_LD64_LO12: 105 case R_AARCH64_TLSDESC_ADD_LO12: 106 return R_TLSDESC; 107 case R_AARCH64_TLSDESC_CALL: 108 return R_TLSDESC_CALL; 109 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 110 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 111 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 112 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 113 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 114 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 115 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 116 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 117 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 118 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 119 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 120 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 121 return R_TLS; 122 case R_AARCH64_CALL26: 123 case R_AARCH64_CONDBR19: 124 case R_AARCH64_JUMP26: 125 case R_AARCH64_TSTBR14: 126 return R_PLT_PC; 127 case R_AARCH64_PREL16: 128 case R_AARCH64_PREL32: 129 case R_AARCH64_PREL64: 130 case R_AARCH64_ADR_PREL_LO21: 131 case R_AARCH64_LD_PREL_LO19: 132 case R_AARCH64_MOVW_PREL_G0: 133 case R_AARCH64_MOVW_PREL_G0_NC: 134 case R_AARCH64_MOVW_PREL_G1: 135 case R_AARCH64_MOVW_PREL_G1_NC: 136 case R_AARCH64_MOVW_PREL_G2: 137 case R_AARCH64_MOVW_PREL_G2_NC: 138 case R_AARCH64_MOVW_PREL_G3: 139 return R_PC; 140 case R_AARCH64_ADR_PREL_PG_HI21: 141 case R_AARCH64_ADR_PREL_PG_HI21_NC: 142 return R_AARCH64_PAGE_PC; 143 case R_AARCH64_LD64_GOT_LO12_NC: 144 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 145 return R_GOT; 146 case R_AARCH64_ADR_GOT_PAGE: 147 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 148 return R_AARCH64_GOT_PAGE_PC; 149 case R_AARCH64_NONE: 150 return R_NONE; 151 default: 152 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 153 ") against symbol " + toString(s)); 154 return R_NONE; 155 } 156 } 157 158 RelExpr AArch64::adjustRelaxExpr(RelType type, const uint8_t *data, 159 RelExpr expr) const { 160 if (expr == R_RELAX_TLS_GD_TO_IE) { 161 if (type == R_AARCH64_TLSDESC_ADR_PAGE21) 162 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 163 return R_RELAX_TLS_GD_TO_IE_ABS; 164 } 165 return expr; 166 } 167 168 bool AArch64::usesOnlyLowPageBits(RelType type) const { 169 switch (type) { 170 default: 171 return false; 172 case R_AARCH64_ADD_ABS_LO12_NC: 173 case R_AARCH64_LD64_GOT_LO12_NC: 174 case R_AARCH64_LDST128_ABS_LO12_NC: 175 case R_AARCH64_LDST16_ABS_LO12_NC: 176 case R_AARCH64_LDST32_ABS_LO12_NC: 177 case R_AARCH64_LDST64_ABS_LO12_NC: 178 case R_AARCH64_LDST8_ABS_LO12_NC: 179 case R_AARCH64_TLSDESC_ADD_LO12: 180 case R_AARCH64_TLSDESC_LD64_LO12: 181 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 182 return true; 183 } 184 } 185 186 RelType AArch64::getDynRel(RelType type) const { 187 if (type == R_AARCH64_ABS64) 188 return type; 189 return R_AARCH64_NONE; 190 } 191 192 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { 193 write64le(buf, in.plt->getVA()); 194 } 195 196 void AArch64::writePltHeader(uint8_t *buf) const { 197 const uint8_t pltData[] = { 198 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 199 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 200 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 201 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 202 0x20, 0x02, 0x1f, 0xd6, // br x17 203 0x1f, 0x20, 0x03, 0xd5, // nop 204 0x1f, 0x20, 0x03, 0xd5, // nop 205 0x1f, 0x20, 0x03, 0xd5 // nop 206 }; 207 memcpy(buf, pltData, sizeof(pltData)); 208 209 uint64_t got = in.gotPlt->getVA(); 210 uint64_t plt = in.plt->getVA(); 211 relocateOne(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 212 getAArch64Page(got + 16) - getAArch64Page(plt + 4)); 213 relocateOne(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 214 relocateOne(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 215 } 216 217 void AArch64::writePlt(uint8_t *buf, const Symbol &sym, 218 uint64_t pltEntryAddr) const { 219 const uint8_t inst[] = { 220 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 221 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 222 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 223 0x20, 0x02, 0x1f, 0xd6 // br x17 224 }; 225 memcpy(buf, inst, sizeof(inst)); 226 227 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 228 relocateOne(buf, R_AARCH64_ADR_PREL_PG_HI21, 229 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 230 relocateOne(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 231 relocateOne(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 232 } 233 234 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, 235 uint64_t branchAddr, const Symbol &s, 236 int64_t a) const { 237 // If s is an undefined weak symbol and does not have a PLT entry then it 238 // will be resolved as a branch to the next instruction. 239 if (s.isUndefWeak() && !s.isInPlt()) 240 return false; 241 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 242 // only permits range extension thunks for R_AARCH64_CALL26 and 243 // R_AARCH64_JUMP26 relocation types. 244 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26) 245 return false; 246 uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a); 247 return !inBranchRange(type, branchAddr, dst); 248 } 249 250 uint32_t AArch64::getThunkSectionSpacing() const { 251 // See comment in Arch/ARM.cpp for a more detailed explanation of 252 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 253 // Thunk have a range of +/- 128 MiB 254 return (128 * 1024 * 1024) - 0x30000; 255 } 256 257 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { 258 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26) 259 return true; 260 // The AArch64 call and unconditional branch instructions have a range of 261 // +/- 128 MiB. 262 uint64_t range = 128 * 1024 * 1024; 263 if (dst > src) { 264 // Immediate of branch is signed. 265 range -= 4; 266 return dst - src <= range; 267 } 268 return src - dst <= range; 269 } 270 271 static void write32AArch64Addr(uint8_t *l, uint64_t imm) { 272 uint32_t immLo = (imm & 0x3) << 29; 273 uint32_t immHi = (imm & 0x1FFFFC) << 3; 274 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); 275 write32le(l, (read32le(l) & ~mask) | immLo | immHi); 276 } 277 278 // Return the bits [Start, End] from Val shifted Start bits. 279 // For instance, getBits(0xF0, 4, 8) returns 0xF. 280 static uint64_t getBits(uint64_t val, int start, int end) { 281 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; 282 return (val >> start) & mask; 283 } 284 285 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } 286 287 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 288 static void or32AArch64Imm(uint8_t *l, uint64_t imm) { 289 or32le(l, (imm & 0xFFF) << 10); 290 } 291 292 // Update the immediate field in an AArch64 movk, movn or movz instruction 293 // for a signed relocation, and update the opcode of a movn or movz instruction 294 // to match the sign of the operand. 295 static void writeSMovWImm(uint8_t *loc, uint32_t imm) { 296 uint32_t inst = read32le(loc); 297 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. 298 if (!(inst & (1 << 29))) { 299 // movn or movz. 300 if (imm & 0x10000) { 301 // Change opcode to movn, which takes an inverted operand. 302 imm ^= 0xFFFF; 303 inst &= ~(1 << 30); 304 } else { 305 // Change opcode to movz. 306 inst |= 1 << 30; 307 } 308 } 309 write32le(loc, inst | ((imm & 0xFFFF) << 5)); 310 } 311 312 void AArch64::relocateOne(uint8_t *loc, RelType type, uint64_t val) const { 313 switch (type) { 314 case R_AARCH64_ABS16: 315 case R_AARCH64_PREL16: 316 checkIntUInt(loc, val, 16, type); 317 write16le(loc, val); 318 break; 319 case R_AARCH64_ABS32: 320 case R_AARCH64_PREL32: 321 checkIntUInt(loc, val, 32, type); 322 write32le(loc, val); 323 break; 324 case R_AARCH64_ABS64: 325 case R_AARCH64_PREL64: 326 write64le(loc, val); 327 break; 328 case R_AARCH64_ADD_ABS_LO12_NC: 329 or32AArch64Imm(loc, val); 330 break; 331 case R_AARCH64_ADR_GOT_PAGE: 332 case R_AARCH64_ADR_PREL_PG_HI21: 333 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 334 case R_AARCH64_TLSDESC_ADR_PAGE21: 335 checkInt(loc, val, 33, type); 336 LLVM_FALLTHROUGH; 337 case R_AARCH64_ADR_PREL_PG_HI21_NC: 338 write32AArch64Addr(loc, val >> 12); 339 break; 340 case R_AARCH64_ADR_PREL_LO21: 341 checkInt(loc, val, 21, type); 342 write32AArch64Addr(loc, val); 343 break; 344 case R_AARCH64_JUMP26: 345 // Normally we would just write the bits of the immediate field, however 346 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 347 // we want to replace a non-branch instruction with a branch immediate 348 // instruction. By writing all the bits of the instruction including the 349 // opcode and the immediate (0 001 | 01 imm26) we can do this 350 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 351 // the instruction we want to patch. 352 write32le(loc, 0x14000000); 353 LLVM_FALLTHROUGH; 354 case R_AARCH64_CALL26: 355 checkInt(loc, val, 28, type); 356 or32le(loc, (val & 0x0FFFFFFC) >> 2); 357 break; 358 case R_AARCH64_CONDBR19: 359 case R_AARCH64_LD_PREL_LO19: 360 checkAlignment(loc, val, 4, type); 361 checkInt(loc, val, 21, type); 362 or32le(loc, (val & 0x1FFFFC) << 3); 363 break; 364 case R_AARCH64_LDST8_ABS_LO12_NC: 365 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 366 or32AArch64Imm(loc, getBits(val, 0, 11)); 367 break; 368 case R_AARCH64_LDST16_ABS_LO12_NC: 369 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 370 checkAlignment(loc, val, 2, type); 371 or32AArch64Imm(loc, getBits(val, 1, 11)); 372 break; 373 case R_AARCH64_LDST32_ABS_LO12_NC: 374 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 375 checkAlignment(loc, val, 4, type); 376 or32AArch64Imm(loc, getBits(val, 2, 11)); 377 break; 378 case R_AARCH64_LDST64_ABS_LO12_NC: 379 case R_AARCH64_LD64_GOT_LO12_NC: 380 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 381 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 382 case R_AARCH64_TLSDESC_LD64_LO12: 383 checkAlignment(loc, val, 8, type); 384 or32AArch64Imm(loc, getBits(val, 3, 11)); 385 break; 386 case R_AARCH64_LDST128_ABS_LO12_NC: 387 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 388 checkAlignment(loc, val, 16, type); 389 or32AArch64Imm(loc, getBits(val, 4, 11)); 390 break; 391 case R_AARCH64_MOVW_UABS_G0: 392 checkUInt(loc, val, 16, type); 393 LLVM_FALLTHROUGH; 394 case R_AARCH64_MOVW_UABS_G0_NC: 395 or32le(loc, (val & 0xFFFF) << 5); 396 break; 397 case R_AARCH64_MOVW_UABS_G1: 398 checkUInt(loc, val, 32, type); 399 LLVM_FALLTHROUGH; 400 case R_AARCH64_MOVW_UABS_G1_NC: 401 or32le(loc, (val & 0xFFFF0000) >> 11); 402 break; 403 case R_AARCH64_MOVW_UABS_G2: 404 checkUInt(loc, val, 48, type); 405 LLVM_FALLTHROUGH; 406 case R_AARCH64_MOVW_UABS_G2_NC: 407 or32le(loc, (val & 0xFFFF00000000) >> 27); 408 break; 409 case R_AARCH64_MOVW_UABS_G3: 410 or32le(loc, (val & 0xFFFF000000000000) >> 43); 411 break; 412 case R_AARCH64_MOVW_PREL_G0: 413 case R_AARCH64_MOVW_SABS_G0: 414 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 415 checkInt(loc, val, 17, type); 416 LLVM_FALLTHROUGH; 417 case R_AARCH64_MOVW_PREL_G0_NC: 418 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 419 writeSMovWImm(loc, val); 420 break; 421 case R_AARCH64_MOVW_PREL_G1: 422 case R_AARCH64_MOVW_SABS_G1: 423 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 424 checkInt(loc, val, 33, type); 425 LLVM_FALLTHROUGH; 426 case R_AARCH64_MOVW_PREL_G1_NC: 427 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 428 writeSMovWImm(loc, val >> 16); 429 break; 430 case R_AARCH64_MOVW_PREL_G2: 431 case R_AARCH64_MOVW_SABS_G2: 432 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 433 checkInt(loc, val, 49, type); 434 LLVM_FALLTHROUGH; 435 case R_AARCH64_MOVW_PREL_G2_NC: 436 writeSMovWImm(loc, val >> 32); 437 break; 438 case R_AARCH64_MOVW_PREL_G3: 439 writeSMovWImm(loc, val >> 48); 440 break; 441 case R_AARCH64_TSTBR14: 442 checkInt(loc, val, 16, type); 443 or32le(loc, (val & 0xFFFC) << 3); 444 break; 445 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 446 checkUInt(loc, val, 24, type); 447 or32AArch64Imm(loc, val >> 12); 448 break; 449 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 450 case R_AARCH64_TLSDESC_ADD_LO12: 451 or32AArch64Imm(loc, val); 452 break; 453 default: 454 llvm_unreachable("unknown relocation"); 455 } 456 } 457 458 void AArch64::relaxTlsGdToLe(uint8_t *loc, RelType type, uint64_t val) const { 459 // TLSDESC Global-Dynamic relocation are in the form: 460 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 461 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 462 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 463 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 464 // blr x1 465 // And it can optimized to: 466 // movz x0, #0x0, lsl #16 467 // movk x0, #0x10 468 // nop 469 // nop 470 checkUInt(loc, val, 32, type); 471 472 switch (type) { 473 case R_AARCH64_TLSDESC_ADD_LO12: 474 case R_AARCH64_TLSDESC_CALL: 475 write32le(loc, 0xd503201f); // nop 476 return; 477 case R_AARCH64_TLSDESC_ADR_PAGE21: 478 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz 479 return; 480 case R_AARCH64_TLSDESC_LD64_LO12: 481 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk 482 return; 483 default: 484 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 485 } 486 } 487 488 void AArch64::relaxTlsGdToIe(uint8_t *loc, RelType type, uint64_t val) const { 489 // TLSDESC Global-Dynamic relocation are in the form: 490 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 491 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 492 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 493 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 494 // blr x1 495 // And it can optimized to: 496 // adrp x0, :gottprel:v 497 // ldr x0, [x0, :gottprel_lo12:v] 498 // nop 499 // nop 500 501 switch (type) { 502 case R_AARCH64_TLSDESC_ADD_LO12: 503 case R_AARCH64_TLSDESC_CALL: 504 write32le(loc, 0xd503201f); // nop 505 break; 506 case R_AARCH64_TLSDESC_ADR_PAGE21: 507 write32le(loc, 0x90000000); // adrp 508 relocateOne(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); 509 break; 510 case R_AARCH64_TLSDESC_LD64_LO12: 511 write32le(loc, 0xf9400000); // ldr 512 relocateOne(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); 513 break; 514 default: 515 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 516 } 517 } 518 519 void AArch64::relaxTlsIeToLe(uint8_t *loc, RelType type, uint64_t val) const { 520 checkUInt(loc, val, 32, type); 521 522 if (type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 523 // Generate MOVZ. 524 uint32_t regNo = read32le(loc) & 0x1f; 525 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); 526 return; 527 } 528 if (type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 529 // Generate MOVK. 530 uint32_t regNo = read32le(loc) & 0x1f; 531 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5)); 532 return; 533 } 534 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 535 } 536 537 // AArch64 may use security features in variant PLT sequences. These are: 538 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target 539 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used 540 // in the variant Plt sequences are encoded in the Hint space so they can be 541 // deployed on older architectures, which treat the instructions as a nop. 542 // PAC and BTI can be combined leading to the following combinations: 543 // writePltHeader 544 // writePltHeaderBti (no PAC Header needed) 545 // writePlt 546 // writePltBti (BTI only) 547 // writePltPac (PAC only) 548 // writePltBtiPac (BTI and PAC) 549 // 550 // When PAC is enabled the dynamic loader encrypts the address that it places 551 // in the .got.plt using the pacia1716 instruction which encrypts the value in 552 // x17 using the modifier in x16. The static linker places autia1716 before the 553 // indirect branch to x17 to authenticate the address in x17 with the modifier 554 // in x16. This makes it more difficult for an attacker to modify the value in 555 // the .got.plt. 556 // 557 // When BTI is enabled all indirect branches must land on a bti instruction. 558 // The static linker must place a bti instruction at the start of any PLT entry 559 // that may be the target of an indirect branch. As the PLT entries call the 560 // lazy resolver indirectly this must have a bti instruction at start. In 561 // general a bti instruction is not needed for a PLT entry as indirect calls 562 // are resolved to the function address and not the PLT entry for the function. 563 // There are a small number of cases where the PLT address can escape, such as 564 // taking the address of a function or ifunc via a non got-generating 565 // relocation, and a shared library refers to that symbol. 566 // 567 // We use the bti c variant of the instruction which permits indirect branches 568 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI 569 // guarantees that all indirect branches from code requiring BTI protection 570 // will go via x16/x17 571 572 namespace { 573 class AArch64BtiPac final : public AArch64 { 574 public: 575 AArch64BtiPac(); 576 void writePltHeader(uint8_t *buf) const override; 577 void writePlt(uint8_t *buf, const Symbol &sym, 578 uint64_t pltEntryAddr) const override; 579 580 private: 581 bool btiHeader; // bti instruction needed in PLT Header 582 bool btiEntry; // bti instruction needed in PLT Entry 583 bool pacEntry; // autia1716 instruction needed in PLT Entry 584 }; 585 } // namespace 586 587 AArch64BtiPac::AArch64BtiPac() { 588 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); 589 // A BTI (Branch Target Indicator) Plt Entry is only required if the 590 // address of the PLT entry can be taken by the program, which permits an 591 // indirect jump to the PLT entry. This can happen when the address 592 // of the PLT entry for a function is canonicalised due to the address of 593 // the function in an executable being taken by a shared library. 594 // FIXME: There is a potential optimization to omit the BTI if we detect 595 // that the address of the PLT entry isn't taken. 596 btiEntry = btiHeader && !config->shared; 597 pacEntry = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_PAC); 598 599 if (btiEntry || pacEntry) { 600 pltEntrySize = 24; 601 ipltEntrySize = 24; 602 } 603 } 604 605 void AArch64BtiPac::writePltHeader(uint8_t *buf) const { 606 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 607 const uint8_t pltData[] = { 608 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 609 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 610 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 611 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 612 0x20, 0x02, 0x1f, 0xd6, // br x17 613 0x1f, 0x20, 0x03, 0xd5, // nop 614 0x1f, 0x20, 0x03, 0xd5 // nop 615 }; 616 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 617 618 uint64_t got = in.gotPlt->getVA(); 619 uint64_t plt = in.plt->getVA(); 620 621 if (btiHeader) { 622 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C 623 // instruction. 624 memcpy(buf, btiData, sizeof(btiData)); 625 buf += sizeof(btiData); 626 plt += sizeof(btiData); 627 } 628 memcpy(buf, pltData, sizeof(pltData)); 629 630 relocateOne(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 631 getAArch64Page(got + 16) - getAArch64Page(plt + 8)); 632 relocateOne(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 633 relocateOne(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 634 if (!btiHeader) 635 // We didn't add the BTI c instruction so round out size with NOP. 636 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData)); 637 } 638 639 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym, 640 uint64_t pltEntryAddr) const { 641 // The PLT entry is of the form: 642 // [btiData] addrInst (pacBr | stdBr) [nopData] 643 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 644 const uint8_t addrInst[] = { 645 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 646 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 647 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.plt.got[n])) 648 }; 649 const uint8_t pacBr[] = { 650 0x9f, 0x21, 0x03, 0xd5, // autia1716 651 0x20, 0x02, 0x1f, 0xd6 // br x17 652 }; 653 const uint8_t stdBr[] = { 654 0x20, 0x02, 0x1f, 0xd6, // br x17 655 0x1f, 0x20, 0x03, 0xd5 // nop 656 }; 657 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 658 659 if (btiEntry) { 660 memcpy(buf, btiData, sizeof(btiData)); 661 buf += sizeof(btiData); 662 pltEntryAddr += sizeof(btiData); 663 } 664 665 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 666 memcpy(buf, addrInst, sizeof(addrInst)); 667 relocateOne(buf, R_AARCH64_ADR_PREL_PG_HI21, 668 getAArch64Page(gotPltEntryAddr) - 669 getAArch64Page(pltEntryAddr)); 670 relocateOne(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 671 relocateOne(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 672 673 if (pacEntry) 674 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr)); 675 else 676 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr)); 677 if (!btiEntry) 678 // We didn't add the BTI c instruction so round out size with NOP. 679 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData)); 680 } 681 682 static TargetInfo *getTargetInfo() { 683 if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI | 684 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) { 685 static AArch64BtiPac t; 686 return &t; 687 } 688 static AArch64 t; 689 return &t; 690 } 691 692 TargetInfo *getAArch64TargetInfo() { return getTargetInfo(); } 693 694 } // namespace elf 695 } // namespace lld 696