1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_arm64_Architecture_arm64_h
8 #define jit_arm64_Architecture_arm64_h
9
10 #include "mozilla/Assertions.h"
11 #include "mozilla/MathAlgorithms.h"
12
13 #include "jit/arm64/vixl/Instructions-vixl.h"
14 #include "jit/shared/Architecture-shared.h"
15
16 #include "js/Utility.h"
17
18 #define JS_HAS_HIDDEN_SP
19 static const uint32_t HiddenSPEncoding = vixl::kSPRegInternalCode;
20
21 namespace js {
22 namespace jit {
23
24 // AArch64 has 32 64-bit integer registers, x0 though x31.
25 // x31 is special and functions as both the stack pointer and a zero register.
26 // The bottom 32 bits of each of the X registers is accessible as w0 through
27 // w31. The program counter is no longer accessible as a register.
28 // SIMD and scalar floating-point registers share a register bank.
29 // 32 bit float registers are s0 through s31.
30 // 64 bit double registers are d0 through d31.
31 // 128 bit SIMD registers are v0 through v31.
32 // e.g., s0 is the bottom 32 bits of d0, which is the bottom 64 bits of v0.
33
34 // AArch64 Calling Convention:
35 // x0 - x7: arguments and return value
36 // x8: indirect result (struct) location
37 // x9 - x15: temporary registers
38 // x16 - x17: intra-call-use registers (PLT, linker)
39 // x18: platform specific use (TLS)
40 // x19 - x28: callee-saved registers
41 // x29: frame pointer
42 // x30: link register
43
44 // AArch64 Calling Convention for Floats:
45 // d0 - d7: arguments and return value
46 // d8 - d15: callee-saved registers
47 // Bits 64:128 are not saved for v8-v15.
48 // d16 - d31: temporary registers
49
50 // AArch64 does not have soft float.
51
52 class Registers {
53 public:
54 enum RegisterID {
55 w0 = 0,
56 x0 = 0,
57 w1 = 1,
58 x1 = 1,
59 w2 = 2,
60 x2 = 2,
61 w3 = 3,
62 x3 = 3,
63 w4 = 4,
64 x4 = 4,
65 w5 = 5,
66 x5 = 5,
67 w6 = 6,
68 x6 = 6,
69 w7 = 7,
70 x7 = 7,
71 w8 = 8,
72 x8 = 8,
73 w9 = 9,
74 x9 = 9,
75 w10 = 10,
76 x10 = 10,
77 w11 = 11,
78 x11 = 11,
79 w12 = 12,
80 x12 = 12,
81 w13 = 13,
82 x13 = 13,
83 w14 = 14,
84 x14 = 14,
85 w15 = 15,
86 x15 = 15,
87 w16 = 16,
88 x16 = 16,
89 ip0 = 16, // MacroAssembler scratch register 1.
90 w17 = 17,
91 x17 = 17,
92 ip1 = 17, // MacroAssembler scratch register 2.
93 w18 = 18,
94 x18 = 18,
95 tls = 18, // Platform-specific use (TLS).
96 w19 = 19,
97 x19 = 19,
98 w20 = 20,
99 x20 = 20,
100 w21 = 21,
101 x21 = 21,
102 w22 = 22,
103 x22 = 22,
104 w23 = 23,
105 x23 = 23,
106 w24 = 24,
107 x24 = 24,
108 w25 = 25,
109 x25 = 25,
110 w26 = 26,
111 x26 = 26,
112 w27 = 27,
113 x27 = 27,
114 w28 = 28,
115 x28 = 28,
116 w29 = 29,
117 x29 = 29,
118 fp = 29,
119 w30 = 30,
120 x30 = 30,
121 lr = 30,
122 w31 = 31,
123 x31 = 31,
124 wzr = 31,
125 xzr = 31,
126 sp = 31, // Special: both stack pointer and a zero register.
127 invalid_reg
128 };
129 typedef uint8_t Code;
130 typedef uint32_t Encoding;
131 typedef uint32_t SetType;
132
133 union RegisterContent {
134 uintptr_t r;
135 };
136
SetSize(SetType x)137 static uint32_t SetSize(SetType x) {
138 static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
139 return mozilla::CountPopulation32(x);
140 }
FirstBit(SetType x)141 static uint32_t FirstBit(SetType x) {
142 return mozilla::CountTrailingZeroes32(x);
143 }
LastBit(SetType x)144 static uint32_t LastBit(SetType x) {
145 return 31 - mozilla::CountLeadingZeroes32(x);
146 }
147
GetName(Code code)148 static const char* GetName(Code code) {
149 static const char* const Names[] = {
150 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8",
151 "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17",
152 "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26",
153 "x27", "x28", "x29", "lr", "sp", "invalid"};
154 return Names[code];
155 }
GetName(uint32_t i)156 static const char* GetName(uint32_t i) {
157 MOZ_ASSERT(i < Total);
158 return GetName(Code(i));
159 }
160
161 static Code FromName(const char* name);
162
163 // If SP is used as the base register for a memory load or store, then the
164 // value of the stack pointer prior to adding any offset must be quadword (16
165 // byte) aligned, or else a stack aligment exception will be generated.
166 static const Code StackPointer = sp;
167
168 static const Code Invalid = invalid_reg;
169
170 static const uint32_t Total = 32;
171 static const uint32_t TotalPhys = 32;
172 static const uint32_t Allocatable =
173 27; // No named special-function registers.
174
175 static const SetType AllMask = 0xFFFFFFFF;
176
177 static const SetType ArgRegMask =
178 (1 << Registers::x0) | (1 << Registers::x1) | (1 << Registers::x2) |
179 (1 << Registers::x3) | (1 << Registers::x4) | (1 << Registers::x5) |
180 (1 << Registers::x6) | (1 << Registers::x7) | (1 << Registers::x8);
181
182 static const SetType VolatileMask =
183 (1 << Registers::x0) | (1 << Registers::x1) | (1 << Registers::x2) |
184 (1 << Registers::x3) | (1 << Registers::x4) | (1 << Registers::x5) |
185 (1 << Registers::x6) | (1 << Registers::x7) | (1 << Registers::x8) |
186 (1 << Registers::x9) | (1 << Registers::x10) | (1 << Registers::x11) |
187 (1 << Registers::x11) | (1 << Registers::x12) | (1 << Registers::x13) |
188 (1 << Registers::x14) | (1 << Registers::x14) | (1 << Registers::x15) |
189 (1 << Registers::x16) | (1 << Registers::x17) | (1 << Registers::x18);
190
191 static const SetType NonVolatileMask =
192 (1 << Registers::x19) | (1 << Registers::x20) | (1 << Registers::x21) |
193 (1 << Registers::x22) | (1 << Registers::x23) | (1 << Registers::x24) |
194 (1 << Registers::x25) | (1 << Registers::x26) | (1 << Registers::x27) |
195 (1 << Registers::x28) | (1 << Registers::x29) | (1 << Registers::x30);
196
197 static const SetType SingleByteRegs = VolatileMask | NonVolatileMask;
198
199 static const SetType NonAllocatableMask =
200 (1 << Registers::x28) | // PseudoStackPointer.
201 (1 << Registers::ip0) | // First scratch register.
202 (1 << Registers::ip1) | // Second scratch register.
203 (1 << Registers::tls) | (1 << Registers::lr) | (1 << Registers::sp);
204
205 static const SetType WrapperMask = VolatileMask;
206
207 // Registers returned from a JS -> JS call.
208 static const SetType JSCallMask = (1 << Registers::x2);
209
210 // Registers returned from a JS -> C call.
211 static const SetType CallMask = (1 << Registers::x0);
212
213 static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
214 };
215
216 // Smallest integer type that can hold a register bitmask.
217 typedef uint32_t PackedRegisterMask;
218
219 template <typename T>
220 class TypedRegisterSet;
221
222 class FloatRegisters {
223 public:
224 enum FPRegisterID {
225 s0 = 0,
226 d0 = 0,
227 v0 = 0,
228 s1 = 1,
229 d1 = 1,
230 v1 = 1,
231 s2 = 2,
232 d2 = 2,
233 v2 = 2,
234 s3 = 3,
235 d3 = 3,
236 v3 = 3,
237 s4 = 4,
238 d4 = 4,
239 v4 = 4,
240 s5 = 5,
241 d5 = 5,
242 v5 = 5,
243 s6 = 6,
244 d6 = 6,
245 v6 = 6,
246 s7 = 7,
247 d7 = 7,
248 v7 = 7,
249 s8 = 8,
250 d8 = 8,
251 v8 = 8,
252 s9 = 9,
253 d9 = 9,
254 v9 = 9,
255 s10 = 10,
256 d10 = 10,
257 v10 = 10,
258 s11 = 11,
259 d11 = 11,
260 v11 = 11,
261 s12 = 12,
262 d12 = 12,
263 v12 = 12,
264 s13 = 13,
265 d13 = 13,
266 v13 = 13,
267 s14 = 14,
268 d14 = 14,
269 v14 = 14,
270 s15 = 15,
271 d15 = 15,
272 v15 = 15,
273 s16 = 16,
274 d16 = 16,
275 v16 = 16,
276 s17 = 17,
277 d17 = 17,
278 v17 = 17,
279 s18 = 18,
280 d18 = 18,
281 v18 = 18,
282 s19 = 19,
283 d19 = 19,
284 v19 = 19,
285 s20 = 20,
286 d20 = 20,
287 v20 = 20,
288 s21 = 21,
289 d21 = 21,
290 v21 = 21,
291 s22 = 22,
292 d22 = 22,
293 v22 = 22,
294 s23 = 23,
295 d23 = 23,
296 v23 = 23,
297 s24 = 24,
298 d24 = 24,
299 v24 = 24,
300 s25 = 25,
301 d25 = 25,
302 v25 = 25,
303 s26 = 26,
304 d26 = 26,
305 v26 = 26,
306 s27 = 27,
307 d27 = 27,
308 v27 = 27,
309 s28 = 28,
310 d28 = 28,
311 v28 = 28,
312 s29 = 29,
313 d29 = 29,
314 v29 = 29,
315 s30 = 30,
316 d30 = 30,
317 v30 = 30,
318 s31 = 31,
319 d31 = 31,
320 v31 = 31, // Scratch register.
321 invalid_fpreg
322 };
323 typedef uint8_t Code;
324 typedef FPRegisterID Encoding;
325 typedef uint64_t SetType;
326
GetName(Code code)327 static const char* GetName(Code code) {
328 static const char* const Names[] = {
329 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8",
330 "d9", "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17",
331 "d18", "d19", "d20", "d21", "d22", "d23", "d24", "d25", "d26",
332 "d27", "d28", "d29", "d30", "d31", "invalid"};
333 return Names[code];
334 }
335
GetName(uint32_t i)336 static const char* GetName(uint32_t i) {
337 MOZ_ASSERT(i < TotalPhys);
338 return GetName(Code(i));
339 }
340
341 static Code FromName(const char* name);
342
343 static const Code Invalid = invalid_fpreg;
344
345 static const uint32_t Total = 64;
346 static const uint32_t TotalPhys = 32;
347 static const SetType AllMask = 0xFFFFFFFFFFFFFFFFULL;
348 static const SetType AllPhysMask = 0xFFFFFFFFULL;
349 static const SetType SpreadCoefficient = 0x100000001ULL;
350
351 static const uint32_t Allocatable = 31; // Without d31, the scratch register.
352
353 // d31 is the ScratchFloatReg.
354 static const SetType NonVolatileMask =
355 SetType((1 << FloatRegisters::d8) | (1 << FloatRegisters::d9) |
356 (1 << FloatRegisters::d10) | (1 << FloatRegisters::d11) |
357 (1 << FloatRegisters::d12) | (1 << FloatRegisters::d13) |
358 (1 << FloatRegisters::d14) | (1 << FloatRegisters::d15) |
359 (1 << FloatRegisters::d16) | (1 << FloatRegisters::d17) |
360 (1 << FloatRegisters::d18) | (1 << FloatRegisters::d19) |
361 (1 << FloatRegisters::d20) | (1 << FloatRegisters::d21) |
362 (1 << FloatRegisters::d22) | (1 << FloatRegisters::d23) |
363 (1 << FloatRegisters::d24) | (1 << FloatRegisters::d25) |
364 (1 << FloatRegisters::d26) | (1 << FloatRegisters::d27) |
365 (1 << FloatRegisters::d28) | (1 << FloatRegisters::d29) |
366 (1 << FloatRegisters::d30)) *
367 SpreadCoefficient;
368
369 static const SetType VolatileMask = AllMask & ~NonVolatileMask;
370 static const SetType AllDoubleMask = AllPhysMask << TotalPhys;
371 static const SetType AllSingleMask = AllPhysMask;
372
373 static const SetType WrapperMask = VolatileMask;
374
375 // d31 is the ScratchFloatReg.
376 static const SetType NonAllocatableMask =
377 (SetType(1) << FloatRegisters::d31) * SpreadCoefficient;
378
379 static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
380 union RegisterContent {
381 float s;
382 double d;
383 };
384 enum Kind { Double, Single };
385 };
386
387 // In bytes: slots needed for potential memory->memory move spills.
388 // +8 for cycles
389 // +8 for gpr spills
390 // +8 for double spills
391 static const uint32_t ION_FRAME_SLACK_SIZE = 24;
392
393 static const uint32_t ShadowStackSpace = 0;
394
395 // TODO:
396 // This constant needs to be updated to account for whatever near/far branching
397 // strategy is used by ARM64.
398 static const uint32_t JumpImmediateRange = UINT32_MAX;
399
400 static const uint32_t ABIStackAlignment = 16;
401 static const uint32_t CodeAlignment = 16;
402 static const bool StackKeptAligned = false;
403
404 // Although sp is only usable if 16-byte alignment is kept,
405 // the Pseudo-StackPointer enables use of 8-byte alignment.
406 static const uint32_t StackAlignment = 8;
407 static const uint32_t NativeFrameSize = 8;
408
409 struct FloatRegister {
410 typedef FloatRegisters Codes;
411 typedef Codes::Code Code;
412 typedef Codes::Encoding Encoding;
413 typedef Codes::SetType SetType;
414
415 union RegisterContent {
416 float s;
417 double d;
418 };
419
FloatRegisterFloatRegister420 constexpr FloatRegister(uint32_t code, FloatRegisters::Kind k)
421 : code_(FloatRegisters::Code(code & 31)), k_(k) {}
422
FloatRegisterFloatRegister423 explicit constexpr FloatRegister(uint32_t code)
424 : code_(FloatRegisters::Code(code & 31)),
425 k_(FloatRegisters::Kind(code >> 5)) {}
426
FloatRegisterFloatRegister427 constexpr FloatRegister()
428 : code_(FloatRegisters::Code(-1)), k_(FloatRegisters::Double) {}
429
SetSizeFloatRegister430 static uint32_t SetSize(SetType x) {
431 static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
432 x |= x >> FloatRegisters::TotalPhys;
433 x &= FloatRegisters::AllPhysMask;
434 return mozilla::CountPopulation32(x);
435 }
436
FromCodeFloatRegister437 static FloatRegister FromCode(uint32_t i) {
438 MOZ_ASSERT(i < FloatRegisters::Total);
439 FloatRegister r(i);
440 return r;
441 }
codeFloatRegister442 Code code() const {
443 MOZ_ASSERT((uint32_t)code_ < FloatRegisters::Total);
444 return Code(code_ | (k_ << 5));
445 }
encodingFloatRegister446 Encoding encoding() const { return Encoding(code_); }
447
nameFloatRegister448 const char* name() const { return FloatRegisters::GetName(code()); }
volatile_FloatRegister449 bool volatile_() const {
450 return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
451 }
452 bool operator!=(FloatRegister other) const {
453 return other.code_ != code_ || other.k_ != k_;
454 }
455 bool operator==(FloatRegister other) const {
456 return other.code_ == code_ && other.k_ == k_;
457 }
aliasesFloatRegister458 bool aliases(FloatRegister other) const { return other.code_ == code_; }
numAliasedFloatRegister459 uint32_t numAliased() const { return 2; }
otherkindFloatRegister460 static FloatRegisters::Kind otherkind(FloatRegisters::Kind k) {
461 if (k == FloatRegisters::Double) return FloatRegisters::Single;
462 return FloatRegisters::Double;
463 }
aliasedFloatRegister464 void aliased(uint32_t aliasIdx, FloatRegister* ret) {
465 if (aliasIdx == 0)
466 *ret = *this;
467 else
468 *ret = FloatRegister(code_, otherkind(k_));
469 }
470 // This function mostly exists for the ARM backend. It is to ensure that two
471 // floating point registers' types are equivalent. e.g. S0 is not equivalent
472 // to D16, since S0 holds a float32, and D16 holds a Double.
473 // Since all floating point registers on x86 and x64 are equivalent, it is
474 // reasonable for this function to do the same.
equivFloatRegister475 bool equiv(FloatRegister other) const { return k_ == other.k_; }
sizeFloatRegister476 constexpr uint32_t size() const {
477 return k_ == FloatRegisters::Double ? sizeof(double) : sizeof(float);
478 }
numAlignedAliasedFloatRegister479 uint32_t numAlignedAliased() { return numAliased(); }
alignedAliasedFloatRegister480 void alignedAliased(uint32_t aliasIdx, FloatRegister* ret) {
481 MOZ_ASSERT(aliasIdx < numAliased());
482 aliased(aliasIdx, ret);
483 }
alignedOrDominatedAliasedSetFloatRegister484 SetType alignedOrDominatedAliasedSet() const {
485 return Codes::SpreadCoefficient << code_;
486 }
487
isSingleFloatRegister488 bool isSingle() const { return k_ == FloatRegisters::Single; }
isDoubleFloatRegister489 bool isDouble() const { return k_ == FloatRegisters::Double; }
isSimd128FloatRegister490 bool isSimd128() const { return false; }
491
asSingleFloatRegister492 FloatRegister asSingle() const {
493 return FloatRegister(code_, FloatRegisters::Single);
494 }
asDoubleFloatRegister495 FloatRegister asDouble() const {
496 return FloatRegister(code_, FloatRegisters::Double);
497 }
asSimd128FloatRegister498 FloatRegister asSimd128() const { MOZ_CRASH(); }
499
FirstBitFloatRegister500 static uint32_t FirstBit(SetType x) {
501 JS_STATIC_ASSERT(sizeof(SetType) == 8);
502 return mozilla::CountTrailingZeroes64(x);
503 }
LastBitFloatRegister504 static uint32_t LastBit(SetType x) {
505 JS_STATIC_ASSERT(sizeof(SetType) == 8);
506 return 63 - mozilla::CountLeadingZeroes64(x);
507 }
508
509 static constexpr RegTypeName DefaultType = RegTypeName::Float64;
510
511 template <RegTypeName Name = DefaultType>
LiveAsIndexableSetFloatRegister512 static SetType LiveAsIndexableSet(SetType s) {
513 return SetType(0);
514 }
515
516 template <RegTypeName Name = DefaultType>
AllocatableAsIndexableSetFloatRegister517 static SetType AllocatableAsIndexableSet(SetType s) {
518 static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
519 return LiveAsIndexableSet<Name>(s);
520 }
521
522 static TypedRegisterSet<FloatRegister> ReduceSetForPush(
523 const TypedRegisterSet<FloatRegister>& s);
524 static uint32_t GetSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
525 static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
526 uint32_t getRegisterDumpOffsetInBytes();
527
528 public:
529 Code code_ : 8;
530 FloatRegisters::Kind k_ : 1;
531 };
532
533 template <>
534 inline FloatRegister::SetType
535 FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
536 return set & FloatRegisters::AllSingleMask;
537 }
538
539 template <>
540 inline FloatRegister::SetType
541 FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
542 return set & FloatRegisters::AllDoubleMask;
543 }
544
545 template <>
546 inline FloatRegister::SetType
547 FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
548 return set;
549 }
550
551 // ARM/D32 has double registers that cannot be treated as float32.
552 // Luckily, ARMv8 doesn't have the same misfortune.
hasUnaliasedDouble()553 inline bool hasUnaliasedDouble() { return false; }
554
555 // ARM prior to ARMv8 also has doubles that alias multiple floats.
556 // Again, ARMv8 is in the clear.
hasMultiAlias()557 inline bool hasMultiAlias() { return false; }
558
559 uint32_t GetARM64Flags();
560
561 } // namespace jit
562 } // namespace js
563
564 #endif // jit_arm64_Architecture_arm64_h
565