1 //===- AArch64.cpp --------------------------------------------------------===//
2 //
3 // The LLVM Linker
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "Symbols.h"
11 #include "SyntheticSections.h"
12 #include "Target.h"
13 #include "Thunks.h"
14 #include "lld/Common/ErrorHandler.h"
15 #include "llvm/Object/ELF.h"
16 #include "llvm/Support/Endian.h"
17
18 using namespace llvm;
19 using namespace llvm::support::endian;
20 using namespace llvm::ELF;
21 using namespace lld;
22 using namespace lld::elf;
23
24 // Page(Expr) is the page address of the expression Expr, defined
25 // as (Expr & ~0xFFF). (This applies even if the machine page size
26 // supported by the platform has a different value.)
getAArch64Page(uint64_t Expr)27 uint64_t elf::getAArch64Page(uint64_t Expr) {
28 return Expr & ~static_cast<uint64_t>(0xFFF);
29 }
30
31 namespace {
32 class AArch64 final : public TargetInfo {
33 public:
34 AArch64();
35 RelExpr getRelExpr(RelType Type, const Symbol &S,
36 const uint8_t *Loc) const override;
37 RelType getDynRel(RelType Type) const override;
38 void writeGotPlt(uint8_t *Buf, const Symbol &S) const override;
39 void writePltHeader(uint8_t *Buf) const override;
40 void writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr, uint64_t PltEntryAddr,
41 int32_t Index, unsigned RelOff) const override;
42 bool needsThunk(RelExpr Expr, RelType Type, const InputFile *File,
43 uint64_t BranchAddr, const Symbol &S) const override;
44 bool inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const override;
45 bool usesOnlyLowPageBits(RelType Type) const override;
46 void relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const override;
47 RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data,
48 RelExpr Expr) const override;
49 void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
50 void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
51 void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const override;
52 };
53 } // namespace
54
AArch64()55 AArch64::AArch64() {
56 CopyRel = R_AARCH64_COPY;
57 RelativeRel = R_AARCH64_RELATIVE;
58 IRelativeRel = R_AARCH64_IRELATIVE;
59 GotRel = R_AARCH64_GLOB_DAT;
60 PltRel = R_AARCH64_JUMP_SLOT;
61 TlsDescRel = R_AARCH64_TLSDESC;
62 TlsGotRel = R_AARCH64_TLS_TPREL64;
63 GotEntrySize = 8;
64 GotPltEntrySize = 8;
65 PltEntrySize = 16;
66 PltHeaderSize = 32;
67 DefaultMaxPageSize = 65536;
68
69 // Align to the 2 MiB page size (known as a superpage or huge page).
70 // FreeBSD automatically promotes 2 MiB-aligned allocations.
71 DefaultImageBase = 0x200000;
72
73 // It doesn't seem to be documented anywhere, but tls on aarch64 uses variant
74 // 1 of the tls structures and the tcb size is 16.
75 TcbSize = 16;
76 NeedsThunks = true;
77
78 // See comment in Arch/ARM.cpp for a more detailed explanation of
79 // ThunkSectionSpacing. For AArch64 the only branches we are permitted to
80 // Thunk have a range of +/- 128 MiB
81 ThunkSectionSpacing = (128 * 1024 * 1024) - 0x30000;
82 }
83
getRelExpr(RelType Type,const Symbol & S,const uint8_t * Loc) const84 RelExpr AArch64::getRelExpr(RelType Type, const Symbol &S,
85 const uint8_t *Loc) const {
86 switch (Type) {
87 case R_AARCH64_TLSDESC_ADR_PAGE21:
88 return R_TLSDESC_PAGE;
89 case R_AARCH64_TLSDESC_LD64_LO12:
90 case R_AARCH64_TLSDESC_ADD_LO12:
91 return R_TLSDESC;
92 case R_AARCH64_TLSDESC_CALL:
93 return R_TLSDESC_CALL;
94 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
95 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
96 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
97 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
98 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
99 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
100 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
101 return R_TLS;
102 case R_AARCH64_CALL26:
103 case R_AARCH64_CONDBR19:
104 case R_AARCH64_JUMP26:
105 case R_AARCH64_TSTBR14:
106 return R_PLT_PC;
107 case R_AARCH64_PREL16:
108 case R_AARCH64_PREL32:
109 case R_AARCH64_PREL64:
110 case R_AARCH64_ADR_PREL_LO21:
111 case R_AARCH64_LD_PREL_LO19:
112 return R_PC;
113 case R_AARCH64_ADR_PREL_PG_HI21:
114 return R_PAGE_PC;
115 case R_AARCH64_LD64_GOT_LO12_NC:
116 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
117 return R_GOT;
118 case R_AARCH64_ADR_GOT_PAGE:
119 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
120 return R_GOT_PAGE_PC;
121 case R_AARCH64_NONE:
122 return R_NONE;
123 default:
124 return R_ABS;
125 }
126 }
127
adjustRelaxExpr(RelType Type,const uint8_t * Data,RelExpr Expr) const128 RelExpr AArch64::adjustRelaxExpr(RelType Type, const uint8_t *Data,
129 RelExpr Expr) const {
130 if (Expr == R_RELAX_TLS_GD_TO_IE) {
131 if (Type == R_AARCH64_TLSDESC_ADR_PAGE21)
132 return R_RELAX_TLS_GD_TO_IE_PAGE_PC;
133 return R_RELAX_TLS_GD_TO_IE_ABS;
134 }
135 return Expr;
136 }
137
usesOnlyLowPageBits(RelType Type) const138 bool AArch64::usesOnlyLowPageBits(RelType Type) const {
139 switch (Type) {
140 default:
141 return false;
142 case R_AARCH64_ADD_ABS_LO12_NC:
143 case R_AARCH64_LD64_GOT_LO12_NC:
144 case R_AARCH64_LDST128_ABS_LO12_NC:
145 case R_AARCH64_LDST16_ABS_LO12_NC:
146 case R_AARCH64_LDST32_ABS_LO12_NC:
147 case R_AARCH64_LDST64_ABS_LO12_NC:
148 case R_AARCH64_LDST8_ABS_LO12_NC:
149 case R_AARCH64_TLSDESC_ADD_LO12:
150 case R_AARCH64_TLSDESC_LD64_LO12:
151 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
152 return true;
153 }
154 }
155
getDynRel(RelType Type) const156 RelType AArch64::getDynRel(RelType Type) const {
157 if (Type == R_AARCH64_ABS32 || Type == R_AARCH64_ABS64)
158 return Type;
159 return R_AARCH64_NONE;
160 }
161
writeGotPlt(uint8_t * Buf,const Symbol &) const162 void AArch64::writeGotPlt(uint8_t *Buf, const Symbol &) const {
163 write64le(Buf, InX::Plt->getVA());
164 }
165
writePltHeader(uint8_t * Buf) const166 void AArch64::writePltHeader(uint8_t *Buf) const {
167 const uint8_t PltData[] = {
168 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!
169 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2]))
170 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))]
171 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2]))
172 0x20, 0x02, 0x1f, 0xd6, // br x17
173 0x1f, 0x20, 0x03, 0xd5, // nop
174 0x1f, 0x20, 0x03, 0xd5, // nop
175 0x1f, 0x20, 0x03, 0xd5 // nop
176 };
177 memcpy(Buf, PltData, sizeof(PltData));
178
179 uint64_t Got = InX::GotPlt->getVA();
180 uint64_t Plt = InX::Plt->getVA();
181 relocateOne(Buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
182 getAArch64Page(Got + 16) - getAArch64Page(Plt + 4));
183 relocateOne(Buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, Got + 16);
184 relocateOne(Buf + 12, R_AARCH64_ADD_ABS_LO12_NC, Got + 16);
185 }
186
writePlt(uint8_t * Buf,uint64_t GotPltEntryAddr,uint64_t PltEntryAddr,int32_t Index,unsigned RelOff) const187 void AArch64::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr,
188 uint64_t PltEntryAddr, int32_t Index,
189 unsigned RelOff) const {
190 const uint8_t Inst[] = {
191 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n]))
192 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))]
193 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n]))
194 0x20, 0x02, 0x1f, 0xd6 // br x17
195 };
196 memcpy(Buf, Inst, sizeof(Inst));
197
198 relocateOne(Buf, R_AARCH64_ADR_PREL_PG_HI21,
199 getAArch64Page(GotPltEntryAddr) - getAArch64Page(PltEntryAddr));
200 relocateOne(Buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, GotPltEntryAddr);
201 relocateOne(Buf + 8, R_AARCH64_ADD_ABS_LO12_NC, GotPltEntryAddr);
202 }
203
needsThunk(RelExpr Expr,RelType Type,const InputFile * File,uint64_t BranchAddr,const Symbol & S) const204 bool AArch64::needsThunk(RelExpr Expr, RelType Type, const InputFile *File,
205 uint64_t BranchAddr, const Symbol &S) const {
206 // ELF for the ARM 64-bit architecture, section Call and Jump relocations
207 // only permits range extension thunks for R_AARCH64_CALL26 and
208 // R_AARCH64_JUMP26 relocation types.
209 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26)
210 return false;
211 uint64_t Dst = (Expr == R_PLT_PC) ? S.getPltVA() : S.getVA();
212 return !inBranchRange(Type, BranchAddr, Dst);
213 }
214
inBranchRange(RelType Type,uint64_t Src,uint64_t Dst) const215 bool AArch64::inBranchRange(RelType Type, uint64_t Src, uint64_t Dst) const {
216 if (Type != R_AARCH64_CALL26 && Type != R_AARCH64_JUMP26)
217 return true;
218 // The AArch64 call and unconditional branch instructions have a range of
219 // +/- 128 MiB.
220 uint64_t Range = 128 * 1024 * 1024;
221 if (Dst > Src) {
222 // Immediate of branch is signed.
223 Range -= 4;
224 return Dst - Src <= Range;
225 }
226 return Src - Dst <= Range;
227 }
228
write32AArch64Addr(uint8_t * L,uint64_t Imm)229 static void write32AArch64Addr(uint8_t *L, uint64_t Imm) {
230 uint32_t ImmLo = (Imm & 0x3) << 29;
231 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;
232 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);
233 write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi);
234 }
235
236 // Return the bits [Start, End] from Val shifted Start bits.
237 // For instance, getBits(0xF0, 4, 8) returns 0xF.
getBits(uint64_t Val,int Start,int End)238 static uint64_t getBits(uint64_t Val, int Start, int End) {
239 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1;
240 return (Val >> Start) & Mask;
241 }
242
or32le(uint8_t * P,int32_t V)243 static void or32le(uint8_t *P, int32_t V) { write32le(P, read32le(P) | V); }
244
245 // Update the immediate field in a AARCH64 ldr, str, and add instruction.
or32AArch64Imm(uint8_t * L,uint64_t Imm)246 static void or32AArch64Imm(uint8_t *L, uint64_t Imm) {
247 or32le(L, (Imm & 0xFFF) << 10);
248 }
249
relocateOne(uint8_t * Loc,RelType Type,uint64_t Val) const250 void AArch64::relocateOne(uint8_t *Loc, RelType Type, uint64_t Val) const {
251 switch (Type) {
252 case R_AARCH64_ABS16:
253 case R_AARCH64_PREL16:
254 checkIntUInt(Loc, Val, 16, Type);
255 write16le(Loc, Val);
256 break;
257 case R_AARCH64_ABS32:
258 case R_AARCH64_PREL32:
259 checkIntUInt(Loc, Val, 32, Type);
260 write32le(Loc, Val);
261 break;
262 case R_AARCH64_ABS64:
263 case R_AARCH64_GLOB_DAT:
264 case R_AARCH64_PREL64:
265 write64le(Loc, Val);
266 break;
267 case R_AARCH64_ADD_ABS_LO12_NC:
268 or32AArch64Imm(Loc, Val);
269 break;
270 case R_AARCH64_ADR_GOT_PAGE:
271 case R_AARCH64_ADR_PREL_PG_HI21:
272 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
273 case R_AARCH64_TLSDESC_ADR_PAGE21:
274 checkInt(Loc, Val, 33, Type);
275 write32AArch64Addr(Loc, Val >> 12);
276 break;
277 case R_AARCH64_ADR_PREL_LO21:
278 checkInt(Loc, Val, 21, Type);
279 write32AArch64Addr(Loc, Val);
280 break;
281 case R_AARCH64_JUMP26:
282 // Normally we would just write the bits of the immediate field, however
283 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419
284 // we want to replace a non-branch instruction with a branch immediate
285 // instruction. By writing all the bits of the instruction including the
286 // opcode and the immediate (0 001 | 01 imm26) we can do this
287 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of
288 // the instruction we want to patch.
289 write32le(Loc, 0x14000000);
290 LLVM_FALLTHROUGH;
291 case R_AARCH64_CALL26:
292 checkInt(Loc, Val, 28, Type);
293 or32le(Loc, (Val & 0x0FFFFFFC) >> 2);
294 break;
295 case R_AARCH64_CONDBR19:
296 case R_AARCH64_LD_PREL_LO19:
297 checkAlignment(Loc, Val, 4, Type);
298 checkInt(Loc, Val, 21, Type);
299 or32le(Loc, (Val & 0x1FFFFC) << 3);
300 break;
301 case R_AARCH64_LDST8_ABS_LO12_NC:
302 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
303 or32AArch64Imm(Loc, getBits(Val, 0, 11));
304 break;
305 case R_AARCH64_LDST16_ABS_LO12_NC:
306 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
307 checkAlignment(Loc, Val, 2, Type);
308 or32AArch64Imm(Loc, getBits(Val, 1, 11));
309 break;
310 case R_AARCH64_LDST32_ABS_LO12_NC:
311 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
312 checkAlignment(Loc, Val, 4, Type);
313 or32AArch64Imm(Loc, getBits(Val, 2, 11));
314 break;
315 case R_AARCH64_LDST64_ABS_LO12_NC:
316 case R_AARCH64_LD64_GOT_LO12_NC:
317 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
318 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
319 case R_AARCH64_TLSDESC_LD64_LO12:
320 checkAlignment(Loc, Val, 8, Type);
321 or32AArch64Imm(Loc, getBits(Val, 3, 11));
322 break;
323 case R_AARCH64_LDST128_ABS_LO12_NC:
324 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
325 checkAlignment(Loc, Val, 16, Type);
326 or32AArch64Imm(Loc, getBits(Val, 4, 11));
327 break;
328 case R_AARCH64_MOVW_UABS_G0_NC:
329 or32le(Loc, (Val & 0xFFFF) << 5);
330 break;
331 case R_AARCH64_MOVW_UABS_G1_NC:
332 or32le(Loc, (Val & 0xFFFF0000) >> 11);
333 break;
334 case R_AARCH64_MOVW_UABS_G2_NC:
335 or32le(Loc, (Val & 0xFFFF00000000) >> 27);
336 break;
337 case R_AARCH64_MOVW_UABS_G3:
338 or32le(Loc, (Val & 0xFFFF000000000000) >> 43);
339 break;
340 case R_AARCH64_TSTBR14:
341 checkInt(Loc, Val, 16, Type);
342 or32le(Loc, (Val & 0xFFFC) << 3);
343 break;
344 case R_AARCH64_TLSLE_ADD_TPREL_HI12:
345 checkInt(Loc, Val, 24, Type);
346 or32AArch64Imm(Loc, Val >> 12);
347 break;
348 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
349 case R_AARCH64_TLSDESC_ADD_LO12:
350 or32AArch64Imm(Loc, Val);
351 break;
352 default:
353 error(getErrorLocation(Loc) + "unrecognized reloc " + Twine(Type));
354 }
355 }
356
relaxTlsGdToLe(uint8_t * Loc,RelType Type,uint64_t Val) const357 void AArch64::relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const {
358 // TLSDESC Global-Dynamic relocation are in the form:
359 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
360 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
361 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
362 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
363 // blr x1
364 // And it can optimized to:
365 // movz x0, #0x0, lsl #16
366 // movk x0, #0x10
367 // nop
368 // nop
369 checkUInt(Loc, Val, 32, Type);
370
371 switch (Type) {
372 case R_AARCH64_TLSDESC_ADD_LO12:
373 case R_AARCH64_TLSDESC_CALL:
374 write32le(Loc, 0xd503201f); // nop
375 return;
376 case R_AARCH64_TLSDESC_ADR_PAGE21:
377 write32le(Loc, 0xd2a00000 | (((Val >> 16) & 0xffff) << 5)); // movz
378 return;
379 case R_AARCH64_TLSDESC_LD64_LO12:
380 write32le(Loc, 0xf2800000 | ((Val & 0xffff) << 5)); // movk
381 return;
382 default:
383 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
384 }
385 }
386
relaxTlsGdToIe(uint8_t * Loc,RelType Type,uint64_t Val) const387 void AArch64::relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const {
388 // TLSDESC Global-Dynamic relocation are in the form:
389 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]
390 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]
391 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]
392 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]
393 // blr x1
394 // And it can optimized to:
395 // adrp x0, :gottprel:v
396 // ldr x0, [x0, :gottprel_lo12:v]
397 // nop
398 // nop
399
400 switch (Type) {
401 case R_AARCH64_TLSDESC_ADD_LO12:
402 case R_AARCH64_TLSDESC_CALL:
403 write32le(Loc, 0xd503201f); // nop
404 break;
405 case R_AARCH64_TLSDESC_ADR_PAGE21:
406 write32le(Loc, 0x90000000); // adrp
407 relocateOne(Loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, Val);
408 break;
409 case R_AARCH64_TLSDESC_LD64_LO12:
410 write32le(Loc, 0xf9400000); // ldr
411 relocateOne(Loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, Val);
412 break;
413 default:
414 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
415 }
416 }
417
relaxTlsIeToLe(uint8_t * Loc,RelType Type,uint64_t Val) const418 void AArch64::relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const {
419 checkUInt(Loc, Val, 32, Type);
420
421 if (Type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
422 // Generate MOVZ.
423 uint32_t RegNo = read32le(Loc) & 0x1f;
424 write32le(Loc, (0xd2a00000 | RegNo) | (((Val >> 16) & 0xffff) << 5));
425 return;
426 }
427 if (Type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {
428 // Generate MOVK.
429 uint32_t RegNo = read32le(Loc) & 0x1f;
430 write32le(Loc, (0xf2800000 | RegNo) | ((Val & 0xffff) << 5));
431 return;
432 }
433 llvm_unreachable("invalid relocation for TLS IE to LE relaxation");
434 }
435
getAArch64TargetInfo()436 TargetInfo *elf::getAArch64TargetInfo() {
437 static AArch64 Target;
438 return &Target;
439 }
440