1 //===- AArch64.cpp --------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "Symbols.h"
10 #include "SyntheticSections.h"
11 #include "Target.h"
12 #include "Thunks.h"
13 #include "lld/Common/ErrorHandler.h"
14 #include "llvm/Object/ELF.h"
15 #include "llvm/Support/Endian.h"
16
17 using namespace llvm;
18 using namespace llvm::support::endian;
19 using namespace llvm::ELF;
20 using namespace lld;
21 using namespace lld::elf;
22
23 // Page(Expr) is the page address of the expression Expr, defined
24 // as (Expr & ~0xFFF). (This applies even if the machine page size
25 // supported by the platform has a different value.)
getAArch64Page(uint64_t expr)26 uint64_t elf::getAArch64Page(uint64_t expr) {
27 return expr & ~static_cast<uint64_t>(0xFFF);
28 }
29
30 namespace {
31 class AArch64 : public TargetInfo {
32 public:
33 AArch64();
34 RelExpr getRelExpr(RelType type, const Symbol &s,
35 const uint8_t *loc) const override;
36 RelType getDynRel(RelType type) const override;
37 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
39 void writePltHeader(uint8_t *buf) const override;
40 void writePlt(uint8_t *buf, const Symbol &sym,
41 uint64_t pltEntryAddr) const override;
42 bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
43 uint64_t branchAddr, const Symbol &s,
44 int64_t a) const override;
45 uint32_t getThunkSectionSpacing() const override;
46 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;
47 bool usesOnlyLowPageBits(RelType type) const override;
48 void relocate(uint8_t *loc, const Relocation &rel,
49 uint64_t val) const override;
50 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
51 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
52 uint64_t val) const override;
53 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
54 uint64_t val) const override;
55 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
56 uint64_t val) const override;
57 };
58 } // namespace
59
AArch64()60 AArch64::AArch64() {
61 copyRel = R_AARCH64_COPY;
62 relativeRel = R_AARCH64_RELATIVE;
63 iRelativeRel = R_AARCH64_IRELATIVE;
64 gotRel = R_AARCH64_GLOB_DAT;
65 noneRel = R_AARCH64_NONE;
66 pltRel = R_AARCH64_JUMP_SLOT;
67 symbolicRel = R_AARCH64_ABS64;
68 tlsDescRel = R_AARCH64_TLSDESC;
69 tlsGotRel = R_AARCH64_TLS_TPREL64;
70 pltHeaderSize = 32;
71 pltEntrySize = 16;
72 ipltEntrySize = 16;
73 defaultMaxPageSize = 65536;
74 gotBaseSymInGotPlt = false;
75
76 // Align to the 2 MiB page size (known as a superpage or huge page).
77 // FreeBSD automatically promotes 2 MiB-aligned allocations.
78 defaultImageBase = 0x200000;
79
80 needsThunks = true;
81 }
82
getRelExpr(RelType type,const Symbol & s,const uint8_t * loc) const83 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
84 const uint8_t *loc) const {
85 switch (type) {
86 case R_AARCH64_ABS16:
87 case R_AARCH64_ABS32:
88 case R_AARCH64_ABS64:
89 case R_AARCH64_ADD_ABS_LO12_NC:
90 case R_AARCH64_LDST128_ABS_LO12_NC:
91 case R_AARCH64_LDST16_ABS_LO12_NC:
92 case R_AARCH64_LDST32_ABS_LO12_NC:
93 case R_AARCH64_LDST64_ABS_LO12_NC:
94 case R_AARCH64_LDST8_ABS_LO12_NC:
95 case R_AARCH64_MOVW_SABS_G0:
96 case R_AARCH64_MOVW_SABS_G1:
97 case R_AARCH64_MOVW_SABS_G2:
98 case R_AARCH64_MOVW_UABS_G0:
99 case R_AARCH64_MOVW_UABS_G0_NC:
100 case R_AARCH64_MOVW_UABS_G1:
101 case R_AARCH64_MOVW_UABS_G1_NC:
102 case R_AARCH64_MOVW_UABS_G2:
103 case R_AARCH64_MOVW_UABS_G2_NC:
104 case R_AARCH64_MOVW_UABS_G3:
105 return R_ABS;
106 case R_AARCH64_TLSDESC_ADR_PAGE21:
107 return R_AARCH64_TLSDESC_PAGE;
108 case R_AARCH64_TLSDESC_LD64_LO12:
109 case R_AARCH64_TLSDESC_ADD_LO12:
110 return R_TLSDESC;
111 case R_AARCH64_TLSDESC_CALL:
112 return R_TLSDESC_CALL;
113 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
114 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
115 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
116 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
117 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
118 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
119 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
120 case R_AARCH64_TLSLE_MOVW_TPREL_G0:
121 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
122 case R_AARCH64_TLSLE_MOVW_TPREL_G1:
123 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
124 case R_AARCH64_TLSLE_MOVW_TPREL_G2:
125 return R_TPREL;
126 case R_AARCH64_CALL26:
127 case R_AARCH64_CONDBR19:
128 case R_AARCH64_JUMP26:
129 case R_AARCH64_TSTBR14:
130 case R_AARCH64_PLT32:
131 return R_PLT_PC;
132 case R_AARCH64_PREL16:
133 case R_AARCH64_PREL32:
134 case R_AARCH64_PREL64:
135 case R_AARCH64_ADR_PREL_LO21:
136 case R_AARCH64_LD_PREL_LO19:
137 case R_AARCH64_MOVW_PREL_G0:
138 case R_AARCH64_MOVW_PREL_G0_NC:
139 case R_AARCH64_MOVW_PREL_G1:
140 case R_AARCH64_MOVW_PREL_G1_NC:
141 case R_AARCH64_MOVW_PREL_G2:
142 case R_AARCH64_MOVW_PREL_G2_NC:
143 case R_AARCH64_MOVW_PREL_G3:
144 return R_PC;
145 case R_AARCH64_ADR_PREL_PG_HI21:
146 case R_AARCH64_ADR_PREL_PG_HI21_NC:
147 return R_AARCH64_PAGE_PC;
148 case R_AARCH64_LD64_GOT_LO12_NC:
149 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
150 return R_GOT;
151 case R_AARCH64_LD64_GOTPAGE_LO15:
152 return R_AARCH64_GOT_PAGE;
153 case R_AARCH64_ADR_GOT_PAGE:
154 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
155 return R_AARCH64_GOT_PAGE_PC;
156 case R_AARCH64_NONE:
157 return R_NONE;
158 default:
159 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
160 ") against symbol " + toString(s));
161 return R_NONE;
162 }
163 }
164
adjustTlsExpr(RelType type,RelExpr expr) const165 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {
166 if (expr == R_RELAX_TLS_GD_TO_IE) {
167 if (type == R_AARCH64_TLSDESC_ADR_PAGE21)
168 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
169 return R_RELAX_TLS_GD_TO_IE_ABS;
170 }
171 return expr;
172 }
173
usesOnlyLowPageBits(RelType type) const174 bool AArch64::usesOnlyLowPageBits(RelType type) const {
175 switch (type) {
176 default:
177 return false;
178 case R_AARCH64_ADD_ABS_LO12_NC:
179 case R_AARCH64_LD64_GOT_LO12_NC:
180 case R_AARCH64_LDST128_ABS_LO12_NC:
181 case R_AARCH64_LDST16_ABS_LO12_NC:
182 case R_AARCH64_LDST32_ABS_LO12_NC:
183 case R_AARCH64_LDST64_ABS_LO12_NC:
184 case R_AARCH64_LDST8_ABS_LO12_NC:
185 case R_AARCH64_TLSDESC_ADD_LO12:
186 case R_AARCH64_TLSDESC_LD64_LO12:
187 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
188 return true;
189 }
190 }
191
getDynRel(RelType type) const192 RelType AArch64::getDynRel(RelType type) const {
193 if (type == R_AARCH64_ABS64)
194 return type;
195 return R_AARCH64_NONE;
196 }
197
getImplicitAddend(const uint8_t * buf,RelType type) const198 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
199 switch (type) {
200 case R_AARCH64_TLSDESC:
201 return read64(buf + 8);
202 default:
203 internalLinkerError(getErrorLocation(buf),
204 "cannot read addend for relocation " + toString(type));
205 return 0;
206 }
207 }
208
writeGotPlt(uint8_t * buf,const Symbol &) const209 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {
210 write64(buf, in.plt->getVA());
211 }
212
writePltHeader(uint8_t * buf) const213 void AArch64::writePltHeader(uint8_t *buf) const {
214 const uint8_t pltData[] = {
215 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!
216 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2]))
217 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))]
218 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2]))
219 0x20, 0x02, 0x1f, 0xd6, // br x17
220 0x1f, 0x20, 0x03, 0xd5, // nop
221 0x1f, 0x20, 0x03, 0xd5, // nop
222 0x1f, 0x20, 0x03, 0xd5 // nop
223 };
224 memcpy(buf, pltData, sizeof(pltData));
225
226 uint64_t got = in.gotPlt->getVA();
227 uint64_t plt = in.plt->getVA();
228 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
229 getAArch64Page(got + 16) - getAArch64Page(plt + 4));
230 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
231 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
232 }
233
writePlt(uint8_t * buf,const Symbol & sym,uint64_t pltEntryAddr) const234 void AArch64::writePlt(uint8_t *buf, const Symbol &sym,
235 uint64_t pltEntryAddr) const {
236 const uint8_t inst[] = {
237 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n]))
238 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))]
239 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n]))
240 0x20, 0x02, 0x1f, 0xd6 // br x17
241 };
242 memcpy(buf, inst, sizeof(inst));
243
244 uint64_t gotPltEntryAddr = sym.getGotPltVA();
245 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
246 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
247 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
248 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
249 }
250
needsThunk(RelExpr expr,RelType type,const InputFile * file,uint64_t branchAddr,const Symbol & s,int64_t a) const251 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
252 uint64_t branchAddr, const Symbol &s,
253 int64_t a) const {
254 // If s is an undefined weak symbol and does not have a PLT entry then it
255 // will be resolved as a branch to the next instruction.
256 if (s.isUndefWeak() && !s.isInPlt())
257 return false;
258 // ELF for the ARM 64-bit architecture, section Call and Jump relocations
259 // only permits range extension thunks for R_AARCH64_CALL26 and
260 // R_AARCH64_JUMP26 relocation types.
261 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
262 type != R_AARCH64_PLT32)
263 return false;
264 uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a);
265 return !inBranchRange(type, branchAddr, dst);
266 }
267
getThunkSectionSpacing() const268 uint32_t AArch64::getThunkSectionSpacing() const {
269 // See comment in Arch/ARM.cpp for a more detailed explanation of
270 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to
271 // Thunk have a range of +/- 128 MiB
272 return (128 * 1024 * 1024) - 0x30000;
273 }
274
inBranchRange(RelType type,uint64_t src,uint64_t dst) const275 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
276 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
277 type != R_AARCH64_PLT32)
278 return true;
279 // The AArch64 call and unconditional branch instructions have a range of
280 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.
281 uint64_t range =
282 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);
283 if (dst > src) {
284 // Immediate of branch is signed.
285 range -= 4;
286 return dst - src <= range;
287 }
288 return src - dst <= range;
289 }
290
write32AArch64Addr(uint8_t * l,uint64_t imm)291 static void write32AArch64Addr(uint8_t *l, uint64_t imm) {
292 uint32_t immLo = (imm & 0x3) << 29;
293 uint32_t immHi = (imm & 0x1FFFFC) << 3;
294 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3);
295 write32le(l, (read32le(l) & ~mask) | immLo | immHi);
296 }
297
298 // Return the bits [Start, End] from Val shifted Start bits.
299 // For instance, getBits(0xF0, 4, 8) returns 0xF.
getBits(uint64_t val,int start,int end)300 static uint64_t getBits(uint64_t val, int start, int end) {
301 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1;
302 return (val >> start) & mask;
303 }
304
or32le(uint8_t * p,int32_t v)305 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); }
306
307 // Update the immediate field in a AARCH64 ldr, str, and add instruction.
or32AArch64Imm(uint8_t * l,uint64_t imm)308 static void or32AArch64Imm(uint8_t *l, uint64_t imm) {
309 or32le(l, (imm & 0xFFF) << 10);
310 }
311
312 // Update the immediate field in an AArch64 movk, movn or movz instruction
313 // for a signed relocation, and update the opcode of a movn or movz instruction
314 // to match the sign of the operand.
writeSMovWImm(uint8_t * loc,uint32_t imm)315 static void writeSMovWImm(uint8_t *loc, uint32_t imm) {
316 uint32_t inst = read32le(loc);
317 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk.
318 if (!(inst & (1 << 29))) {
319 // movn or movz.
320 if (imm & 0x10000) {
321 // Change opcode to movn, which takes an inverted operand.
322 imm ^= 0xFFFF;
323 inst &= ~(1 << 30);
324 } else {
325 // Change opcode to movz.
326 inst |= 1 << 30;
327 }
328 }
329 write32le(loc, inst | ((imm & 0xFFFF) << 5));
330 }
331
relocate(uint8_t * loc,const Relocation & rel,uint64_t val) const332 void AArch64::relocate(uint8_t *loc, const Relocation &rel,
333 uint64_t val) const {
334 switch (rel.type) {
335 case R_AARCH64_ABS16:
336 case R_AARCH64_PREL16:
337 checkIntUInt(loc, val, 16, rel);
338 write16(loc, val);
339 break;
340 case R_AARCH64_ABS32:
341 case R_AARCH64_PREL32:
342 checkIntUInt(loc, val, 32, rel);
343 write32(loc, val);
344 break;
345 case R_AARCH64_PLT32:
346 checkInt(loc, val, 32, rel);
347 write32(loc, val);
348 break;
349 case R_AARCH64_ABS64:
350 case R_AARCH64_PREL64:
351 write64(loc, val);
352 break;
353 case R_AARCH64_ADD_ABS_LO12_NC:
354 or32AArch64Imm(loc, val);
355 break;
356 case R_AARCH64_ADR_GOT_PAGE:
357 case R_AARCH64_ADR_PREL_PG_HI21:
358 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
359 case R_AARCH64_TLSDESC_ADR_PAGE21:
360 checkInt(loc, val, 33, rel);
361 LLVM_FALLTHROUGH;
362 case R_AARCH64_ADR_PREL_PG_HI21_NC:
363 write32AArch64Addr(loc, val >> 12);
364 break;
365 case R_AARCH64_ADR_PREL_LO21:
366 checkInt(loc, val, 21, rel);
367 write32AArch64Addr(loc, val);
368 break;
369 case R_AARCH64_JUMP26:
370 // Normally we would just write the bits of the immediate field, however
371 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419
372 // we want to replace a non-branch instruction with a branch immediate
373 // instruction. By writing all the bits of the instruction including the
374 // opcode and the immediate (0 001 | 01 imm26) we can do this
375 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of
376 // the instruction we want to patch.
377 write32le(loc, 0x14000000);
378 LLVM_FALLTHROUGH;
379 case R_AARCH64_CALL26:
380 checkInt(loc, val, 28, rel);
381 or32le(loc, (val & 0x0FFFFFFC) >> 2);
382 break;
383 case R_AARCH64_CONDBR19:
384 case R_AARCH64_LD_PREL_LO19:
385 checkAlignment(loc, val, 4, rel);
386 checkInt(loc, val, 21, rel);
387 or32le(loc, (val & 0x1FFFFC) << 3);
388 break;
389 case R_AARCH64_LDST8_ABS_LO12_NC:
390 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
391 or32AArch64Imm(loc, getBits(val, 0, 11));
392 break;
393 case R_AARCH64_LDST16_ABS_LO12_NC:
394 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
395 checkAlignment(loc, val, 2, rel);
396 or32AArch64Imm(loc, getBits(val, 1, 11));
397 break;
398 case R_AARCH64_LDST32_ABS_LO12_NC:
399 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
400 checkAlignment(loc, val, 4, rel);
401 or32AArch64Imm(loc, getBits(val, 2, 11));
402 break;
403 case R_AARCH64_LDST64_ABS_LO12_NC:
404 case R_AARCH64_LD64_GOT_LO12_NC:
405 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
406 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
407 case R_AARCH64_TLSDESC_LD64_LO12:
408 checkAlignment(loc, val, 8, rel);
409 or32AArch64Imm(loc, getBits(val, 3, 11));
410 break;
411 case R_AARCH64_LDST128_ABS_LO12_NC:
412 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
413 checkAlignment(loc, val, 16, rel);
414 or32AArch64Imm(loc, getBits(val, 4, 11));
415 break;
416 case R_AARCH64_LD64_GOTPAGE_LO15:
417 checkAlignment(loc, val, 8, rel);
418 or32AArch64Imm(loc, getBits(val, 3, 14));
419 break;
420 case R_AARCH64_MOVW_UABS_G0:
421 checkUInt(loc, val, 16, rel);
422 LLVM_FALLTHROUGH;
423 case R_AARCH64_MOVW_UABS_G0_NC:
424 or32le(loc, (val & 0xFFFF) << 5);
425 break;
426 case R_AARCH64_MOVW_UABS_G1:
427 checkUInt(loc, val, 32, rel);
428 LLVM_FALLTHROUGH;
429 case R_AARCH64_MOVW_UABS_G1_NC:
430 or32le(loc, (val & 0xFFFF0000) >> 11);
431 break;
432 case R_AARCH64_MOVW_UABS_G2:
433 checkUInt(loc, val, 48, rel);
434 LLVM_FALLTHROUGH;
435 case R_AARCH64_MOVW_UABS_G2_NC:
436 or32le(loc, (val & 0xFFFF00000000) >> 27);
437 break;
438 case R_AARCH64_MOVW_UABS_G3:
439 or32le(loc, (val & 0xFFFF000000000000) >> 43);
440 break;
441 case R_AARCH64_MOVW_PREL_G0:
442 case R_AARCH64_MOVW_SABS_G0:
443 case R_AARCH64_TLSLE_MOVW_TPREL_G0:
444 checkInt(loc, val, 17, rel);
445 LLVM_FALLTHROUGH;
446 case R_AARCH64_MOVW_PREL_G0_NC:
447 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
448 writeSMovWImm(loc, val);
449 break;
450 case R_AARCH64_MOVW_PREL_G1:
451 case R_AARCH64_MOVW_SABS_G1:
452 case R_AARCH64_TLSLE_MOVW_TPREL_G1:
453 checkInt(loc, val, 33, rel);
454 LLVM_FALLTHROUGH;
455 case R_AARCH64_MOVW_PREL_G1_NC:
456 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
457 writeSMovWImm(loc, val >> 16);
458 break;
459 case R_AARCH64_MOVW_PREL_G2:
460 case R_AARCH64_MOVW_SABS_G2:
461 case R_AARCH64_TLSLE_MOVW_TPREL_G2:
462 checkInt(loc, val, 49, rel);
463 LLVM_FALLTHROUGH;
464 case R_AARCH64_MOVW_PREL_G2_NC:
465 writeSMovWImm(loc, val >> 32);
466 break;
467 case R_AARCH64_MOVW_PREL_G3:
468 writeSMovWImm(loc, val >> 48);
469 break;
470 case R_AARCH64_TSTBR14:
471 checkInt(loc, val, 16, rel);
472 or32le(loc, (val & 0xFFFC) << 3);
473 break;
474 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
475 checkUInt(loc, val, 24, rel);
476 or32AArch64Imm(loc, val >> 12);
477 break;
478 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
479 case R_AARCH64_TLSDESC_ADD_LO12:
480 or32AArch64Imm(loc, val);
481 break;
482 case R_AARCH64_TLSDESC:
483 // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
484 write64(loc + 8, val);
485 break;
486 default:
487 llvm_unreachable("unknown relocation");
488 }
489 }
490
relaxTlsGdToLe(uint8_t * loc,const Relocation & rel,uint64_t val) const491 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
492 uint64_t val) const {
493 // TLSDESC Global-Dynamic relocation are in the form:
494 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
495 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
496 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
497 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
498 // blr x1
499 // And it can optimized to:
500 // movz x0, #0x0, lsl #16
501 // movk x0, #0x10
502 // nop
503 // nop
504 checkUInt(loc, val, 32, rel);
505
506 switch (rel.type) {
507 case R_AARCH64_TLSDESC_ADD_LO12:
508 case R_AARCH64_TLSDESC_CALL:
509 write32le(loc, 0xd503201f); // nop
510 return;
511 case R_AARCH64_TLSDESC_ADR_PAGE21:
512 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz
513 return;
514 case R_AARCH64_TLSDESC_LD64_LO12:
515 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk
516 return;
517 default:
518 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
519 }
520 }
521
relaxTlsGdToIe(uint8_t * loc,const Relocation & rel,uint64_t val) const522 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
523 uint64_t val) const {
524 // TLSDESC Global-Dynamic relocation are in the form:
525 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
526 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
527 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
528 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
529 // blr x1
530 // And it can optimized to:
531 // adrp x0, :gottprel:v
532 // ldr x0, [x0, :gottprel_lo12:v]
533 // nop
534 // nop
535
536 switch (rel.type) {
537 case R_AARCH64_TLSDESC_ADD_LO12:
538 case R_AARCH64_TLSDESC_CALL:
539 write32le(loc, 0xd503201f); // nop
540 break;
541 case R_AARCH64_TLSDESC_ADR_PAGE21:
542 write32le(loc, 0x90000000); // adrp
543 relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val);
544 break;
545 case R_AARCH64_TLSDESC_LD64_LO12:
546 write32le(loc, 0xf9400000); // ldr
547 relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val);
548 break;
549 default:
550 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
551 }
552 }
553
relaxTlsIeToLe(uint8_t * loc,const Relocation & rel,uint64_t val) const554 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
555 uint64_t val) const {
556 checkUInt(loc, val, 32, rel);
557
558 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
559 // Generate MOVZ.
560 uint32_t regNo = read32le(loc) & 0x1f;
561 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5));
562 return;
563 }
564 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {
565 // Generate MOVK.
566 uint32_t regNo = read32le(loc) & 0x1f;
567 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5));
568 return;
569 }
570 llvm_unreachable("invalid relocation for TLS IE to LE relaxation");
571 }
572
573 // AArch64 may use security features in variant PLT sequences. These are:
574 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
575 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used
576 // in the variant Plt sequences are encoded in the Hint space so they can be
577 // deployed on older architectures, which treat the instructions as a nop.
578 // PAC and BTI can be combined leading to the following combinations:
579 // writePltHeader
580 // writePltHeaderBti (no PAC Header needed)
581 // writePlt
582 // writePltBti (BTI only)
583 // writePltPac (PAC only)
584 // writePltBtiPac (BTI and PAC)
585 //
586 // When PAC is enabled the dynamic loader encrypts the address that it places
587 // in the .got.plt using the pacia1716 instruction which encrypts the value in
588 // x17 using the modifier in x16. The static linker places autia1716 before the
589 // indirect branch to x17 to authenticate the address in x17 with the modifier
590 // in x16. This makes it more difficult for an attacker to modify the value in
591 // the .got.plt.
592 //
593 // When BTI is enabled all indirect branches must land on a bti instruction.
594 // The static linker must place a bti instruction at the start of any PLT entry
595 // that may be the target of an indirect branch. As the PLT entries call the
596 // lazy resolver indirectly this must have a bti instruction at start. In
597 // general a bti instruction is not needed for a PLT entry as indirect calls
598 // are resolved to the function address and not the PLT entry for the function.
599 // There are a small number of cases where the PLT address can escape, such as
600 // taking the address of a function or ifunc via a non got-generating
601 // relocation, and a shared library refers to that symbol.
602 //
603 // We use the bti c variant of the instruction which permits indirect branches
604 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI
605 // guarantees that all indirect branches from code requiring BTI protection
606 // will go via x16/x17
607
608 namespace {
609 class AArch64BtiPac final : public AArch64 {
610 public:
611 AArch64BtiPac();
612 void writePltHeader(uint8_t *buf) const override;
613 void writePlt(uint8_t *buf, const Symbol &sym,
614 uint64_t pltEntryAddr) const override;
615
616 private:
617 bool btiHeader; // bti instruction needed in PLT Header
618 bool btiEntry; // bti instruction needed in PLT Entry
619 bool pacEntry; // autia1716 instruction needed in PLT Entry
620 };
621 } // namespace
622
AArch64BtiPac()623 AArch64BtiPac::AArch64BtiPac() {
624 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI);
625 // A BTI (Branch Target Indicator) Plt Entry is only required if the
626 // address of the PLT entry can be taken by the program, which permits an
627 // indirect jump to the PLT entry. This can happen when the address
628 // of the PLT entry for a function is canonicalised due to the address of
629 // the function in an executable being taken by a shared library.
630 // FIXME: There is a potential optimization to omit the BTI if we detect
631 // that the address of the PLT entry isn't taken.
632 // The PAC PLT entries require dynamic loader support and this isn't known
633 // from properties in the objects, so we use the command line flag.
634 btiEntry = btiHeader && !config->shared;
635 pacEntry = config->zPacPlt;
636
637 if (btiEntry || pacEntry) {
638 pltEntrySize = 24;
639 ipltEntrySize = 24;
640 }
641 }
642
writePltHeader(uint8_t * buf) const643 void AArch64BtiPac::writePltHeader(uint8_t *buf) const {
644 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
645 const uint8_t pltData[] = {
646 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!
647 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2]))
648 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))]
649 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2]))
650 0x20, 0x02, 0x1f, 0xd6, // br x17
651 0x1f, 0x20, 0x03, 0xd5, // nop
652 0x1f, 0x20, 0x03, 0xd5 // nop
653 };
654 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
655
656 uint64_t got = in.gotPlt->getVA();
657 uint64_t plt = in.plt->getVA();
658
659 if (btiHeader) {
660 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C
661 // instruction.
662 memcpy(buf, btiData, sizeof(btiData));
663 buf += sizeof(btiData);
664 plt += sizeof(btiData);
665 }
666 memcpy(buf, pltData, sizeof(pltData));
667
668 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
669 getAArch64Page(got + 16) - getAArch64Page(plt + 8));
670 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
671 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
672 if (!btiHeader)
673 // We didn't add the BTI c instruction so round out size with NOP.
674 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData));
675 }
676
writePlt(uint8_t * buf,const Symbol & sym,uint64_t pltEntryAddr) const677 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,
678 uint64_t pltEntryAddr) const {
679 // The PLT entry is of the form:
680 // [btiData] addrInst (pacBr | stdBr) [nopData]
681 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
682 const uint8_t addrInst[] = {
683 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n]))
684 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))]
685 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.plt.got[n]))
686 };
687 const uint8_t pacBr[] = {
688 0x9f, 0x21, 0x03, 0xd5, // autia1716
689 0x20, 0x02, 0x1f, 0xd6 // br x17
690 };
691 const uint8_t stdBr[] = {
692 0x20, 0x02, 0x1f, 0xd6, // br x17
693 0x1f, 0x20, 0x03, 0xd5 // nop
694 };
695 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
696
697 if (btiEntry) {
698 memcpy(buf, btiData, sizeof(btiData));
699 buf += sizeof(btiData);
700 pltEntryAddr += sizeof(btiData);
701 }
702
703 uint64_t gotPltEntryAddr = sym.getGotPltVA();
704 memcpy(buf, addrInst, sizeof(addrInst));
705 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
706 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
707 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
708 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
709
710 if (pacEntry)
711 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr));
712 else
713 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr));
714 if (!btiEntry)
715 // We didn't add the BTI c instruction so round out size with NOP.
716 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData));
717 }
718
getTargetInfo()719 static TargetInfo *getTargetInfo() {
720 if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
721 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) {
722 static AArch64BtiPac t;
723 return &t;
724 }
725 static AArch64 t;
726 return &t;
727 }
728
getAArch64TargetInfo()729 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); }
730