1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #if V8_TARGET_ARCH_ARM64
6
7 #include "src/base/bits.h"
8 #include "src/base/division-by-constant.h"
9 #include "src/codegen/assembler.h"
10 #include "src/codegen/callable.h"
11 #include "src/codegen/code-factory.h"
12 #include "src/codegen/external-reference-table.h"
13 #include "src/codegen/macro-assembler-inl.h"
14 #include "src/codegen/register-configuration.h"
15 #include "src/debug/debug.h"
16 #include "src/deoptimizer/deoptimizer.h"
17 #include "src/execution/frame-constants.h"
18 #include "src/execution/frames-inl.h"
19 #include "src/heap/memory-chunk.h"
20 #include "src/init/bootstrapper.h"
21 #include "src/logging/counters.h"
22 #include "src/runtime/runtime.h"
23 #include "src/snapshot/embedded/embedded-data.h"
24 #include "src/snapshot/snapshot.h"
25 #include "src/wasm/wasm-code-manager.h"
26
27 // Satisfy cpplint check, but don't include platform-specific header. It is
28 // included recursively via macro-assembler.h.
29 #if 0
30 #include "src/codegen/arm64/macro-assembler-arm64.h"
31 #endif
32
33 namespace v8 {
34 namespace internal {
35
DefaultTmpList()36 CPURegList TurboAssembler::DefaultTmpList() { return CPURegList(ip0, ip1); }
37
DefaultFPTmpList()38 CPURegList TurboAssembler::DefaultFPTmpList() {
39 return CPURegList(fp_scratch1, fp_scratch2);
40 }
41
RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,Register exclusion) const42 int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
43 Register exclusion) const {
44 auto list = kCallerSaved;
45 list.Remove(exclusion);
46 list.Align();
47
48 int bytes = list.Count() * kXRegSizeInBits / 8;
49
50 if (fp_mode == kSaveFPRegs) {
51 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
52 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
53 }
54 return bytes;
55 }
56
PushCallerSaved(SaveFPRegsMode fp_mode,Register exclusion)57 int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
58 Register exclusion) {
59 auto list = kCallerSaved;
60 list.Remove(exclusion);
61 list.Align();
62
63 PushCPURegList<kDontStoreLR>(list);
64
65 int bytes = list.Count() * kXRegSizeInBits / 8;
66
67 if (fp_mode == kSaveFPRegs) {
68 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
69 PushCPURegList(kCallerSavedV);
70 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
71 }
72 return bytes;
73 }
74
PopCallerSaved(SaveFPRegsMode fp_mode,Register exclusion)75 int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion) {
76 int bytes = 0;
77 if (fp_mode == kSaveFPRegs) {
78 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
79 PopCPURegList(kCallerSavedV);
80 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
81 }
82
83 auto list = kCallerSaved;
84 list.Remove(exclusion);
85 list.Align();
86
87 PopCPURegList<kDontLoadLR>(list);
88 bytes += list.Count() * kXRegSizeInBits / 8;
89
90 return bytes;
91 }
92
LogicalMacro(const Register & rd,const Register & rn,const Operand & operand,LogicalOp op)93 void TurboAssembler::LogicalMacro(const Register& rd, const Register& rn,
94 const Operand& operand, LogicalOp op) {
95 UseScratchRegisterScope temps(this);
96
97 if (operand.NeedsRelocation(this)) {
98 Register temp = temps.AcquireX();
99 Ldr(temp, operand.immediate());
100 Logical(rd, rn, temp, op);
101
102 } else if (operand.IsImmediate()) {
103 int64_t immediate = operand.ImmediateValue();
104 unsigned reg_size = rd.SizeInBits();
105
106 // If the operation is NOT, invert the operation and immediate.
107 if ((op & NOT) == NOT) {
108 op = static_cast<LogicalOp>(op & ~NOT);
109 immediate = ~immediate;
110 }
111
112 // Ignore the top 32 bits of an immediate if we're moving to a W register.
113 if (rd.Is32Bits()) {
114 // Check that the top 32 bits are consistent.
115 DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
116 ((immediate >> kWRegSizeInBits) == -1));
117 immediate &= kWRegMask;
118 }
119
120 DCHECK(rd.Is64Bits() || is_uint32(immediate));
121
122 // Special cases for all set or all clear immediates.
123 if (immediate == 0) {
124 switch (op) {
125 case AND:
126 Mov(rd, 0);
127 return;
128 case ORR: // Fall through.
129 case EOR:
130 Mov(rd, rn);
131 return;
132 case ANDS: // Fall through.
133 case BICS:
134 break;
135 default:
136 UNREACHABLE();
137 }
138 } else if ((rd.Is64Bits() && (immediate == -1L)) ||
139 (rd.Is32Bits() && (immediate == 0xFFFFFFFFL))) {
140 switch (op) {
141 case AND:
142 Mov(rd, rn);
143 return;
144 case ORR:
145 Mov(rd, immediate);
146 return;
147 case EOR:
148 Mvn(rd, rn);
149 return;
150 case ANDS: // Fall through.
151 case BICS:
152 break;
153 default:
154 UNREACHABLE();
155 }
156 }
157
158 unsigned n, imm_s, imm_r;
159 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
160 // Immediate can be encoded in the instruction.
161 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
162 } else {
163 // Immediate can't be encoded: synthesize using move immediate.
164 Register temp = temps.AcquireSameSizeAs(rn);
165
166 // If the left-hand input is the stack pointer, we can't pre-shift the
167 // immediate, as the encoding won't allow the subsequent post shift.
168 PreShiftImmMode mode = rn == sp ? kNoShift : kAnyShift;
169 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);
170
171 if (rd.IsSP()) {
172 // If rd is the stack pointer we cannot use it as the destination
173 // register so we use the temp register as an intermediate again.
174 Logical(temp, rn, imm_operand, op);
175 Mov(sp, temp);
176 } else {
177 Logical(rd, rn, imm_operand, op);
178 }
179 }
180
181 } else if (operand.IsExtendedRegister()) {
182 DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
183 // Add/sub extended supports shift <= 4. We want to support exactly the
184 // same modes here.
185 DCHECK_LE(operand.shift_amount(), 4);
186 DCHECK(operand.reg().Is64Bits() ||
187 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
188 Register temp = temps.AcquireSameSizeAs(rn);
189 EmitExtendShift(temp, operand.reg(), operand.extend(),
190 operand.shift_amount());
191 Logical(rd, rn, temp, op);
192
193 } else {
194 // The operand can be encoded in the instruction.
195 DCHECK(operand.IsShiftedRegister());
196 Logical(rd, rn, operand, op);
197 }
198 }
199
Mov(const Register & rd,uint64_t imm)200 void TurboAssembler::Mov(const Register& rd, uint64_t imm) {
201 DCHECK(allow_macro_instructions());
202 DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
203 DCHECK(!rd.IsZero());
204
205 // TODO(all) extend to support more immediates.
206 //
207 // Immediates on Aarch64 can be produced using an initial value, and zero to
208 // three move keep operations.
209 //
210 // Initial values can be generated with:
211 // 1. 64-bit move zero (movz).
212 // 2. 32-bit move inverted (movn).
213 // 3. 64-bit move inverted.
214 // 4. 32-bit orr immediate.
215 // 5. 64-bit orr immediate.
216 // Move-keep may then be used to modify each of the 16-bit half-words.
217 //
218 // The code below supports all five initial value generators, and
219 // applying move-keep operations to move-zero and move-inverted initial
220 // values.
221
222 // Try to move the immediate in one instruction, and if that fails, switch to
223 // using multiple instructions.
224 if (!TryOneInstrMoveImmediate(rd, imm)) {
225 unsigned reg_size = rd.SizeInBits();
226
227 // Generic immediate case. Imm will be represented by
228 // [imm3, imm2, imm1, imm0], where each imm is 16 bits.
229 // A move-zero or move-inverted is generated for the first non-zero or
230 // non-0xFFFF immX, and a move-keep for subsequent non-zero immX.
231
232 uint64_t ignored_halfword = 0;
233 bool invert_move = false;
234 // If the number of 0xFFFF halfwords is greater than the number of 0x0000
235 // halfwords, it's more efficient to use move-inverted.
236 if (CountClearHalfWords(~imm, reg_size) >
237 CountClearHalfWords(imm, reg_size)) {
238 ignored_halfword = 0xFFFFL;
239 invert_move = true;
240 }
241
242 // Mov instructions can't move immediate values into the stack pointer, so
243 // set up a temporary register, if needed.
244 UseScratchRegisterScope temps(this);
245 Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
246
247 // Iterate through the halfwords. Use movn/movz for the first non-ignored
248 // halfword, and movk for subsequent halfwords.
249 DCHECK_EQ(reg_size % 16, 0);
250 bool first_mov_done = false;
251 for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
252 uint64_t imm16 = (imm >> (16 * i)) & 0xFFFFL;
253 if (imm16 != ignored_halfword) {
254 if (!first_mov_done) {
255 if (invert_move) {
256 movn(temp, (~imm16) & 0xFFFFL, 16 * i);
257 } else {
258 movz(temp, imm16, 16 * i);
259 }
260 first_mov_done = true;
261 } else {
262 // Construct a wider constant.
263 movk(temp, imm16, 16 * i);
264 }
265 }
266 }
267 DCHECK(first_mov_done);
268
269 // Move the temporary if the original destination register was the stack
270 // pointer.
271 if (rd.IsSP()) {
272 mov(rd, temp);
273 }
274 }
275 }
276
Mov(const Register & rd,const Operand & operand,DiscardMoveMode discard_mode)277 void TurboAssembler::Mov(const Register& rd, const Operand& operand,
278 DiscardMoveMode discard_mode) {
279 DCHECK(allow_macro_instructions());
280 DCHECK(!rd.IsZero());
281
282 // Provide a swap register for instructions that need to write into the
283 // system stack pointer (and can't do this inherently).
284 UseScratchRegisterScope temps(this);
285 Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
286
287 if (operand.NeedsRelocation(this)) {
288 // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
289 // non-isolate-independent code. In many cases it might be cheaper than
290 // embedding the relocatable value.
291 if (root_array_available_ && options().isolate_independent_code) {
292 if (operand.ImmediateRMode() == RelocInfo::EXTERNAL_REFERENCE) {
293 Address addr = static_cast<Address>(operand.ImmediateValue());
294 ExternalReference reference = bit_cast<ExternalReference>(addr);
295 IndirectLoadExternalReference(rd, reference);
296 return;
297 } else if (RelocInfo::IsEmbeddedObjectMode(operand.ImmediateRMode())) {
298 Handle<HeapObject> x(
299 reinterpret_cast<Address*>(operand.ImmediateValue()));
300 // TODO(v8:9706): Fix-it! This load will always uncompress the value
301 // even when we are loading a compressed embedded object.
302 IndirectLoadConstant(rd.X(), x);
303 return;
304 }
305 }
306 Ldr(dst, operand);
307 } else if (operand.IsImmediate()) {
308 // Call the macro assembler for generic immediates.
309 Mov(dst, operand.ImmediateValue());
310 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
311 // Emit a shift instruction if moving a shifted register. This operation
312 // could also be achieved using an orr instruction (like orn used by Mvn),
313 // but using a shift instruction makes the disassembly clearer.
314 EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
315 } else if (operand.IsExtendedRegister()) {
316 // Emit an extend instruction if moving an extended register. This handles
317 // extend with post-shift operations, too.
318 EmitExtendShift(dst, operand.reg(), operand.extend(),
319 operand.shift_amount());
320 } else {
321 // Otherwise, emit a register move only if the registers are distinct, or
322 // if they are not X registers.
323 //
324 // Note that mov(w0, w0) is not a no-op because it clears the top word of
325 // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
326 // registers is not required to clear the top word of the X register. In
327 // this case, the instruction is discarded.
328 //
329 // If sp is an operand, add #0 is emitted, otherwise, orr #0.
330 if (rd != operand.reg() ||
331 (rd.Is32Bits() && (discard_mode == kDontDiscardForSameWReg))) {
332 Assembler::mov(rd, operand.reg());
333 }
334 // This case can handle writes into the system stack pointer directly.
335 dst = rd;
336 }
337
338 // Copy the result to the system stack pointer.
339 if (dst != rd) {
340 DCHECK(rd.IsSP());
341 Assembler::mov(rd, dst);
342 }
343 }
344
Mov(const Register & rd,Smi smi)345 void TurboAssembler::Mov(const Register& rd, Smi smi) {
346 return Mov(rd, Operand(smi));
347 }
348
Movi16bitHelper(const VRegister & vd,uint64_t imm)349 void TurboAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
350 DCHECK(is_uint16(imm));
351 int byte1 = (imm & 0xFF);
352 int byte2 = ((imm >> 8) & 0xFF);
353 if (byte1 == byte2) {
354 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
355 } else if (byte1 == 0) {
356 movi(vd, byte2, LSL, 8);
357 } else if (byte2 == 0) {
358 movi(vd, byte1);
359 } else if (byte1 == 0xFF) {
360 mvni(vd, ~byte2 & 0xFF, LSL, 8);
361 } else if (byte2 == 0xFF) {
362 mvni(vd, ~byte1 & 0xFF);
363 } else {
364 UseScratchRegisterScope temps(this);
365 Register temp = temps.AcquireW();
366 movz(temp, imm);
367 dup(vd, temp);
368 }
369 }
370
Movi32bitHelper(const VRegister & vd,uint64_t imm)371 void TurboAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
372 DCHECK(is_uint32(imm));
373
374 uint8_t bytes[sizeof(imm)];
375 memcpy(bytes, &imm, sizeof(imm));
376
377 // All bytes are either 0x00 or 0xFF.
378 {
379 bool all0orff = true;
380 for (int i = 0; i < 4; ++i) {
381 if ((bytes[i] != 0) && (bytes[i] != 0xFF)) {
382 all0orff = false;
383 break;
384 }
385 }
386
387 if (all0orff == true) {
388 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
389 return;
390 }
391 }
392
393 // Of the 4 bytes, only one byte is non-zero.
394 for (int i = 0; i < 4; i++) {
395 if ((imm & (0xFF << (i * 8))) == imm) {
396 movi(vd, bytes[i], LSL, i * 8);
397 return;
398 }
399 }
400
401 // Of the 4 bytes, only one byte is not 0xFF.
402 for (int i = 0; i < 4; i++) {
403 uint32_t mask = ~(0xFF << (i * 8));
404 if ((imm & mask) == mask) {
405 mvni(vd, ~bytes[i] & 0xFF, LSL, i * 8);
406 return;
407 }
408 }
409
410 // Immediate is of the form 0x00MMFFFF.
411 if ((imm & 0xFF00FFFF) == 0x0000FFFF) {
412 movi(vd, bytes[2], MSL, 16);
413 return;
414 }
415
416 // Immediate is of the form 0x0000MMFF.
417 if ((imm & 0xFFFF00FF) == 0x000000FF) {
418 movi(vd, bytes[1], MSL, 8);
419 return;
420 }
421
422 // Immediate is of the form 0xFFMM0000.
423 if ((imm & 0xFF00FFFF) == 0xFF000000) {
424 mvni(vd, ~bytes[2] & 0xFF, MSL, 16);
425 return;
426 }
427 // Immediate is of the form 0xFFFFMM00.
428 if ((imm & 0xFFFF00FF) == 0xFFFF0000) {
429 mvni(vd, ~bytes[1] & 0xFF, MSL, 8);
430 return;
431 }
432
433 // Top and bottom 16-bits are equal.
434 if (((imm >> 16) & 0xFFFF) == (imm & 0xFFFF)) {
435 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xFFFF);
436 return;
437 }
438
439 // Default case.
440 {
441 UseScratchRegisterScope temps(this);
442 Register temp = temps.AcquireW();
443 Mov(temp, imm);
444 dup(vd, temp);
445 }
446 }
447
Movi64bitHelper(const VRegister & vd,uint64_t imm)448 void TurboAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
449 // All bytes are either 0x00 or 0xFF.
450 {
451 bool all0orff = true;
452 for (int i = 0; i < 8; ++i) {
453 int byteval = (imm >> (i * 8)) & 0xFF;
454 if (byteval != 0 && byteval != 0xFF) {
455 all0orff = false;
456 break;
457 }
458 }
459 if (all0orff == true) {
460 movi(vd, imm);
461 return;
462 }
463 }
464
465 // Top and bottom 32-bits are equal.
466 if (((imm >> 32) & 0xFFFFFFFF) == (imm & 0xFFFFFFFF)) {
467 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xFFFFFFFF);
468 return;
469 }
470
471 // Default case.
472 {
473 UseScratchRegisterScope temps(this);
474 Register temp = temps.AcquireX();
475 Mov(temp, imm);
476 if (vd.Is1D()) {
477 mov(vd.D(), 0, temp);
478 } else {
479 dup(vd.V2D(), temp);
480 }
481 }
482 }
483
Movi(const VRegister & vd,uint64_t imm,Shift shift,int shift_amount)484 void TurboAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
485 int shift_amount) {
486 DCHECK(allow_macro_instructions());
487 if (shift_amount != 0 || shift != LSL) {
488 movi(vd, imm, shift, shift_amount);
489 } else if (vd.Is8B() || vd.Is16B()) {
490 // 8-bit immediate.
491 DCHECK(is_uint8(imm));
492 movi(vd, imm);
493 } else if (vd.Is4H() || vd.Is8H()) {
494 // 16-bit immediate.
495 Movi16bitHelper(vd, imm);
496 } else if (vd.Is2S() || vd.Is4S()) {
497 // 32-bit immediate.
498 Movi32bitHelper(vd, imm);
499 } else {
500 // 64-bit immediate.
501 Movi64bitHelper(vd, imm);
502 }
503 }
504
Movi(const VRegister & vd,uint64_t hi,uint64_t lo)505 void TurboAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
506 // TODO(v8:11033): Move 128-bit values in a more efficient way.
507 DCHECK(vd.Is128Bits());
508 Movi(vd.V2D(), lo);
509 if (lo != hi) {
510 UseScratchRegisterScope temps(this);
511 Register temp = temps.AcquireX();
512 Mov(temp, hi);
513 Ins(vd.V2D(), 1, temp);
514 }
515 }
516
Mvn(const Register & rd,const Operand & operand)517 void TurboAssembler::Mvn(const Register& rd, const Operand& operand) {
518 DCHECK(allow_macro_instructions());
519
520 if (operand.NeedsRelocation(this)) {
521 Ldr(rd, operand.immediate());
522 mvn(rd, rd);
523
524 } else if (operand.IsImmediate()) {
525 // Call the macro assembler for generic immediates.
526 Mov(rd, ~operand.ImmediateValue());
527
528 } else if (operand.IsExtendedRegister()) {
529 // Emit two instructions for the extend case. This differs from Mov, as
530 // the extend and invert can't be achieved in one instruction.
531 EmitExtendShift(rd, operand.reg(), operand.extend(),
532 operand.shift_amount());
533 mvn(rd, rd);
534
535 } else {
536 mvn(rd, operand);
537 }
538 }
539
CountClearHalfWords(uint64_t imm,unsigned reg_size)540 unsigned TurboAssembler::CountClearHalfWords(uint64_t imm, unsigned reg_size) {
541 DCHECK_EQ(reg_size % 8, 0);
542 int count = 0;
543 for (unsigned i = 0; i < (reg_size / 16); i++) {
544 if ((imm & 0xFFFF) == 0) {
545 count++;
546 }
547 imm >>= 16;
548 }
549 return count;
550 }
551
552 // The movz instruction can generate immediates containing an arbitrary 16-bit
553 // half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)554 bool TurboAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
555 DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
556 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
557 }
558
559 // The movn instruction can generate immediates containing an arbitrary 16-bit
560 // half-word, with remaining bits set, eg. 0xFFFF1234, 0xFFFF1234FFFFFFFF.
IsImmMovn(uint64_t imm,unsigned reg_size)561 bool TurboAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
562 return IsImmMovz(~imm, reg_size);
563 }
564
ConditionalCompareMacro(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)565 void TurboAssembler::ConditionalCompareMacro(const Register& rn,
566 const Operand& operand,
567 StatusFlags nzcv, Condition cond,
568 ConditionalCompareOp op) {
569 DCHECK((cond != al) && (cond != nv));
570 if (operand.NeedsRelocation(this)) {
571 UseScratchRegisterScope temps(this);
572 Register temp = temps.AcquireX();
573 Ldr(temp, operand.immediate());
574 ConditionalCompareMacro(rn, temp, nzcv, cond, op);
575
576 } else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
577 (operand.IsImmediate() &&
578 IsImmConditionalCompare(operand.ImmediateValue()))) {
579 // The immediate can be encoded in the instruction, or the operand is an
580 // unshifted register: call the assembler.
581 ConditionalCompare(rn, operand, nzcv, cond, op);
582
583 } else {
584 // The operand isn't directly supported by the instruction: perform the
585 // operation on a temporary register.
586 UseScratchRegisterScope temps(this);
587 Register temp = temps.AcquireSameSizeAs(rn);
588 Mov(temp, operand);
589 ConditionalCompare(rn, temp, nzcv, cond, op);
590 }
591 }
592
Csel(const Register & rd,const Register & rn,const Operand & operand,Condition cond)593 void TurboAssembler::Csel(const Register& rd, const Register& rn,
594 const Operand& operand, Condition cond) {
595 DCHECK(allow_macro_instructions());
596 DCHECK(!rd.IsZero());
597 DCHECK((cond != al) && (cond != nv));
598 if (operand.IsImmediate()) {
599 // Immediate argument. Handle special cases of 0, 1 and -1 using zero
600 // register.
601 int64_t imm = operand.ImmediateValue();
602 Register zr = AppropriateZeroRegFor(rn);
603 if (imm == 0) {
604 csel(rd, rn, zr, cond);
605 } else if (imm == 1) {
606 csinc(rd, rn, zr, cond);
607 } else if (imm == -1) {
608 csinv(rd, rn, zr, cond);
609 } else {
610 UseScratchRegisterScope temps(this);
611 Register temp = temps.AcquireSameSizeAs(rn);
612 Mov(temp, imm);
613 csel(rd, rn, temp, cond);
614 }
615 } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
616 // Unshifted register argument.
617 csel(rd, rn, operand.reg(), cond);
618 } else {
619 // All other arguments.
620 UseScratchRegisterScope temps(this);
621 Register temp = temps.AcquireSameSizeAs(rn);
622 Mov(temp, operand);
623 csel(rd, rn, temp, cond);
624 }
625 }
626
TryOneInstrMoveImmediate(const Register & dst,int64_t imm)627 bool TurboAssembler::TryOneInstrMoveImmediate(const Register& dst,
628 int64_t imm) {
629 unsigned n, imm_s, imm_r;
630 int reg_size = dst.SizeInBits();
631 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
632 // Immediate can be represented in a move zero instruction. Movz can't write
633 // to the stack pointer.
634 movz(dst, imm);
635 return true;
636 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
637 // Immediate can be represented in a move not instruction. Movn can't write
638 // to the stack pointer.
639 movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
640 return true;
641 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
642 // Immediate can be represented in a logical orr instruction.
643 LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
644 return true;
645 }
646 return false;
647 }
648
MoveImmediateForShiftedOp(const Register & dst,int64_t imm,PreShiftImmMode mode)649 Operand TurboAssembler::MoveImmediateForShiftedOp(const Register& dst,
650 int64_t imm,
651 PreShiftImmMode mode) {
652 int reg_size = dst.SizeInBits();
653 // Encode the immediate in a single move instruction, if possible.
654 if (TryOneInstrMoveImmediate(dst, imm)) {
655 // The move was successful; nothing to do here.
656 } else {
657 // Pre-shift the immediate to the least-significant bits of the register.
658 int shift_low;
659 if (reg_size == 64) {
660 shift_low = base::bits::CountTrailingZeros(imm);
661 } else {
662 DCHECK_EQ(reg_size, 32);
663 shift_low = base::bits::CountTrailingZeros(static_cast<uint32_t>(imm));
664 }
665
666 if (mode == kLimitShiftForSP) {
667 // When applied to the stack pointer, the subsequent arithmetic operation
668 // can use the extend form to shift left by a maximum of four bits. Right
669 // shifts are not allowed, so we filter them out later before the new
670 // immediate is tested.
671 shift_low = std::min(shift_low, 4);
672 }
673 int64_t imm_low = imm >> shift_low;
674
675 // Pre-shift the immediate to the most-significant bits of the register. We
676 // insert set bits in the least-significant bits, as this creates a
677 // different immediate that may be encodable using movn or orr-immediate.
678 // If this new immediate is encodable, the set bits will be eliminated by
679 // the post shift on the following instruction.
680 int shift_high = CountLeadingZeros(imm, reg_size);
681 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
682
683 if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
684 // The new immediate has been moved into the destination's low bits:
685 // return a new leftward-shifting operand.
686 return Operand(dst, LSL, shift_low);
687 } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
688 // The new immediate has been moved into the destination's high bits:
689 // return a new rightward-shifting operand.
690 return Operand(dst, LSR, shift_high);
691 } else {
692 // Use the generic move operation to set up the immediate.
693 Mov(dst, imm);
694 }
695 }
696 return Operand(dst);
697 }
698
AddSubMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)699 void TurboAssembler::AddSubMacro(const Register& rd, const Register& rn,
700 const Operand& operand, FlagsUpdate S,
701 AddSubOp op) {
702 if (operand.IsZero() && rd == rn && rd.Is64Bits() && rn.Is64Bits() &&
703 !operand.NeedsRelocation(this) && (S == LeaveFlags)) {
704 // The instruction would be a nop. Avoid generating useless code.
705 return;
706 }
707
708 if (operand.NeedsRelocation(this)) {
709 UseScratchRegisterScope temps(this);
710 Register temp = temps.AcquireX();
711 Ldr(temp, operand.immediate());
712 AddSubMacro(rd, rn, temp, S, op);
713 } else if ((operand.IsImmediate() &&
714 !IsImmAddSub(operand.ImmediateValue())) ||
715 (rn.IsZero() && !operand.IsShiftedRegister()) ||
716 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
717 UseScratchRegisterScope temps(this);
718 Register temp = temps.AcquireSameSizeAs(rn);
719 if (operand.IsImmediate()) {
720 PreShiftImmMode mode = kAnyShift;
721
722 // If the destination or source register is the stack pointer, we can
723 // only pre-shift the immediate right by values supported in the add/sub
724 // extend encoding.
725 if (rd == sp) {
726 // If the destination is SP and flags will be set, we can't pre-shift
727 // the immediate at all.
728 mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
729 } else if (rn == sp) {
730 mode = kLimitShiftForSP;
731 }
732
733 Operand imm_operand =
734 MoveImmediateForShiftedOp(temp, operand.ImmediateValue(), mode);
735 AddSub(rd, rn, imm_operand, S, op);
736 } else {
737 Mov(temp, operand);
738 AddSub(rd, rn, temp, S, op);
739 }
740 } else {
741 AddSub(rd, rn, operand, S, op);
742 }
743 }
744
AddSubWithCarryMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)745 void TurboAssembler::AddSubWithCarryMacro(const Register& rd,
746 const Register& rn,
747 const Operand& operand, FlagsUpdate S,
748 AddSubWithCarryOp op) {
749 DCHECK(rd.SizeInBits() == rn.SizeInBits());
750 UseScratchRegisterScope temps(this);
751
752 if (operand.NeedsRelocation(this)) {
753 Register temp = temps.AcquireX();
754 Ldr(temp, operand.immediate());
755 AddSubWithCarryMacro(rd, rn, temp, S, op);
756
757 } else if (operand.IsImmediate() ||
758 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
759 // Add/sub with carry (immediate or ROR shifted register.)
760 Register temp = temps.AcquireSameSizeAs(rn);
761 Mov(temp, operand);
762 AddSubWithCarry(rd, rn, temp, S, op);
763
764 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
765 // Add/sub with carry (shifted register).
766 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
767 DCHECK(operand.shift() != ROR);
768 DCHECK(is_uintn(operand.shift_amount(), rd.SizeInBits() == kXRegSizeInBits
769 ? kXRegSizeInBitsLog2
770 : kWRegSizeInBitsLog2));
771 Register temp = temps.AcquireSameSizeAs(rn);
772 EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
773 AddSubWithCarry(rd, rn, temp, S, op);
774
775 } else if (operand.IsExtendedRegister()) {
776 // Add/sub with carry (extended register).
777 DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
778 // Add/sub extended supports a shift <= 4. We want to support exactly the
779 // same modes.
780 DCHECK_LE(operand.shift_amount(), 4);
781 DCHECK(operand.reg().Is64Bits() ||
782 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
783 Register temp = temps.AcquireSameSizeAs(rn);
784 EmitExtendShift(temp, operand.reg(), operand.extend(),
785 operand.shift_amount());
786 AddSubWithCarry(rd, rn, temp, S, op);
787
788 } else {
789 // The addressing mode is directly supported by the instruction.
790 AddSubWithCarry(rd, rn, operand, S, op);
791 }
792 }
793
LoadStoreMacro(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op)794 void TurboAssembler::LoadStoreMacro(const CPURegister& rt,
795 const MemOperand& addr, LoadStoreOp op) {
796 int64_t offset = addr.offset();
797 unsigned size = CalcLSDataSize(op);
798
799 // Check if an immediate offset fits in the immediate field of the
800 // appropriate instruction. If not, emit two instructions to perform
801 // the operation.
802 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
803 !IsImmLSUnscaled(offset)) {
804 // Immediate offset that can't be encoded using unsigned or unscaled
805 // addressing modes.
806 UseScratchRegisterScope temps(this);
807 Register temp = temps.AcquireSameSizeAs(addr.base());
808 Mov(temp, addr.offset());
809 LoadStore(rt, MemOperand(addr.base(), temp), op);
810 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
811 // Post-index beyond unscaled addressing range.
812 LoadStore(rt, MemOperand(addr.base()), op);
813 add(addr.base(), addr.base(), offset);
814 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
815 // Pre-index beyond unscaled addressing range.
816 add(addr.base(), addr.base(), offset);
817 LoadStore(rt, MemOperand(addr.base()), op);
818 } else {
819 // Encodable in one load/store instruction.
820 LoadStore(rt, addr, op);
821 }
822 }
823
LoadStorePairMacro(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)824 void TurboAssembler::LoadStorePairMacro(const CPURegister& rt,
825 const CPURegister& rt2,
826 const MemOperand& addr,
827 LoadStorePairOp op) {
828 // TODO(all): Should we support register offset for load-store-pair?
829 DCHECK(!addr.IsRegisterOffset());
830
831 int64_t offset = addr.offset();
832 unsigned size = CalcLSPairDataSize(op);
833
834 // Check if the offset fits in the immediate field of the appropriate
835 // instruction. If not, emit two instructions to perform the operation.
836 if (IsImmLSPair(offset, size)) {
837 // Encodable in one load/store pair instruction.
838 LoadStorePair(rt, rt2, addr, op);
839 } else {
840 Register base = addr.base();
841 if (addr.IsImmediateOffset()) {
842 UseScratchRegisterScope temps(this);
843 Register temp = temps.AcquireSameSizeAs(base);
844 Add(temp, base, offset);
845 LoadStorePair(rt, rt2, MemOperand(temp), op);
846 } else if (addr.IsPostIndex()) {
847 LoadStorePair(rt, rt2, MemOperand(base), op);
848 Add(base, base, offset);
849 } else {
850 DCHECK(addr.IsPreIndex());
851 Add(base, base, offset);
852 LoadStorePair(rt, rt2, MemOperand(base), op);
853 }
854 }
855 }
856
NeedExtraInstructionsOrRegisterBranch(Label * label,ImmBranchType b_type)857 bool TurboAssembler::NeedExtraInstructionsOrRegisterBranch(
858 Label* label, ImmBranchType b_type) {
859 bool need_longer_range = false;
860 // There are two situations in which we care about the offset being out of
861 // range:
862 // - The label is bound but too far away.
863 // - The label is not bound but linked, and the previous branch
864 // instruction in the chain is too far away.
865 if (label->is_bound() || label->is_linked()) {
866 need_longer_range =
867 !Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
868 }
869 if (!need_longer_range && !label->is_bound()) {
870 int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
871 unresolved_branches_.insert(std::pair<int, FarBranchInfo>(
872 max_reachable_pc, FarBranchInfo(pc_offset(), label)));
873 // Also maintain the next pool check.
874 next_veneer_pool_check_ = std::min(
875 next_veneer_pool_check_, max_reachable_pc - kVeneerDistanceCheckMargin);
876 }
877 return need_longer_range;
878 }
879
Adr(const Register & rd,Label * label,AdrHint hint)880 void TurboAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
881 DCHECK(allow_macro_instructions());
882 DCHECK(!rd.IsZero());
883
884 if (hint == kAdrNear) {
885 adr(rd, label);
886 return;
887 }
888
889 DCHECK_EQ(hint, kAdrFar);
890 if (label->is_bound()) {
891 int label_offset = label->pos() - pc_offset();
892 if (Instruction::IsValidPCRelOffset(label_offset)) {
893 adr(rd, label);
894 } else {
895 DCHECK_LE(label_offset, 0);
896 int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
897 adr(rd, min_adr_offset);
898 Add(rd, rd, label_offset - min_adr_offset);
899 }
900 } else {
901 UseScratchRegisterScope temps(this);
902 Register scratch = temps.AcquireX();
903
904 InstructionAccurateScope scope(this,
905 PatchingAssembler::kAdrFarPatchableNInstrs);
906 adr(rd, label);
907 for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
908 nop(ADR_FAR_NOP);
909 }
910 movz(scratch, 0);
911 }
912 }
913
B(Label * label,BranchType type,Register reg,int bit)914 void TurboAssembler::B(Label* label, BranchType type, Register reg, int bit) {
915 DCHECK((reg == NoReg || type >= kBranchTypeFirstUsingReg) &&
916 (bit == -1 || type >= kBranchTypeFirstUsingBit));
917 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
918 B(static_cast<Condition>(type), label);
919 } else {
920 switch (type) {
921 case always:
922 B(label);
923 break;
924 case never:
925 break;
926 case reg_zero:
927 Cbz(reg, label);
928 break;
929 case reg_not_zero:
930 Cbnz(reg, label);
931 break;
932 case reg_bit_clear:
933 Tbz(reg, bit, label);
934 break;
935 case reg_bit_set:
936 Tbnz(reg, bit, label);
937 break;
938 default:
939 UNREACHABLE();
940 }
941 }
942 }
943
B(Label * label,Condition cond)944 void TurboAssembler::B(Label* label, Condition cond) {
945 DCHECK(allow_macro_instructions());
946 DCHECK((cond != al) && (cond != nv));
947
948 Label done;
949 bool need_extra_instructions =
950 NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
951
952 if (need_extra_instructions) {
953 b(&done, NegateCondition(cond));
954 B(label);
955 } else {
956 b(label, cond);
957 }
958 bind(&done);
959 }
960
Tbnz(const Register & rt,unsigned bit_pos,Label * label)961 void TurboAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
962 DCHECK(allow_macro_instructions());
963
964 Label done;
965 bool need_extra_instructions =
966 NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
967
968 if (need_extra_instructions) {
969 tbz(rt, bit_pos, &done);
970 B(label);
971 } else {
972 tbnz(rt, bit_pos, label);
973 }
974 bind(&done);
975 }
976
Tbz(const Register & rt,unsigned bit_pos,Label * label)977 void TurboAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
978 DCHECK(allow_macro_instructions());
979
980 Label done;
981 bool need_extra_instructions =
982 NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
983
984 if (need_extra_instructions) {
985 tbnz(rt, bit_pos, &done);
986 B(label);
987 } else {
988 tbz(rt, bit_pos, label);
989 }
990 bind(&done);
991 }
992
Cbnz(const Register & rt,Label * label)993 void TurboAssembler::Cbnz(const Register& rt, Label* label) {
994 DCHECK(allow_macro_instructions());
995
996 Label done;
997 bool need_extra_instructions =
998 NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
999
1000 if (need_extra_instructions) {
1001 cbz(rt, &done);
1002 B(label);
1003 } else {
1004 cbnz(rt, label);
1005 }
1006 bind(&done);
1007 }
1008
Cbz(const Register & rt,Label * label)1009 void TurboAssembler::Cbz(const Register& rt, Label* label) {
1010 DCHECK(allow_macro_instructions());
1011
1012 Label done;
1013 bool need_extra_instructions =
1014 NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1015
1016 if (need_extra_instructions) {
1017 cbnz(rt, &done);
1018 B(label);
1019 } else {
1020 cbz(rt, label);
1021 }
1022 bind(&done);
1023 }
1024
1025 // Pseudo-instructions.
1026
Abs(const Register & rd,const Register & rm,Label * is_not_representable,Label * is_representable)1027 void TurboAssembler::Abs(const Register& rd, const Register& rm,
1028 Label* is_not_representable, Label* is_representable) {
1029 DCHECK(allow_macro_instructions());
1030 DCHECK(AreSameSizeAndType(rd, rm));
1031
1032 Cmp(rm, 1);
1033 Cneg(rd, rm, lt);
1034
1035 // If the comparison sets the v flag, the input was the smallest value
1036 // representable by rm, and the mathematical result of abs(rm) is not
1037 // representable using two's complement.
1038 if ((is_not_representable != nullptr) && (is_representable != nullptr)) {
1039 B(is_not_representable, vs);
1040 B(is_representable);
1041 } else if (is_not_representable != nullptr) {
1042 B(is_not_representable, vs);
1043 } else if (is_representable != nullptr) {
1044 B(is_representable, vc);
1045 }
1046 }
1047
1048 // Abstracted stack operations.
1049
Push(const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3,const CPURegister & src4,const CPURegister & src5,const CPURegister & src6,const CPURegister & src7)1050 void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1051 const CPURegister& src2, const CPURegister& src3,
1052 const CPURegister& src4, const CPURegister& src5,
1053 const CPURegister& src6, const CPURegister& src7) {
1054 DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
1055
1056 int count = 5 + src5.is_valid() + src6.is_valid() + src6.is_valid();
1057 int size = src0.SizeInBytes();
1058 DCHECK_EQ(0, (size * count) % 16);
1059
1060 PushHelper(4, size, src0, src1, src2, src3);
1061 PushHelper(count - 4, size, src4, src5, src6, src7);
1062 }
1063
Pop(const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3,const CPURegister & dst4,const CPURegister & dst5,const CPURegister & dst6,const CPURegister & dst7)1064 void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1065 const CPURegister& dst2, const CPURegister& dst3,
1066 const CPURegister& dst4, const CPURegister& dst5,
1067 const CPURegister& dst6, const CPURegister& dst7) {
1068 // It is not valid to pop into the same register more than once in one
1069 // instruction, not even into the zero register.
1070 DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1071 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1072 DCHECK(dst0.is_valid());
1073
1074 int count = 5 + dst5.is_valid() + dst6.is_valid() + dst7.is_valid();
1075 int size = dst0.SizeInBytes();
1076 DCHECK_EQ(0, (size * count) % 16);
1077
1078 PopHelper(4, size, dst0, dst1, dst2, dst3);
1079 PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
1080 }
1081
PushMultipleTimes(CPURegister src,Register count)1082 void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
1083 UseScratchRegisterScope temps(this);
1084 Register temp = temps.AcquireSameSizeAs(count);
1085
1086 Label loop, leftover2, leftover1, done;
1087
1088 Subs(temp, count, 4);
1089 B(mi, &leftover2);
1090
1091 // Push groups of four first.
1092 Bind(&loop);
1093 Subs(temp, temp, 4);
1094 PushHelper(4, src.SizeInBytes(), src, src, src, src);
1095 B(pl, &loop);
1096
1097 // Push groups of two.
1098 Bind(&leftover2);
1099 Tbz(count, 1, &leftover1);
1100 PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
1101
1102 // Push the last one (if required).
1103 Bind(&leftover1);
1104 Tbz(count, 0, &done);
1105 PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
1106
1107 Bind(&done);
1108 }
1109
PushHelper(int count,int size,const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1110 void TurboAssembler::PushHelper(int count, int size, const CPURegister& src0,
1111 const CPURegister& src1,
1112 const CPURegister& src2,
1113 const CPURegister& src3) {
1114 // Ensure that we don't unintentially modify scratch or debug registers.
1115 InstructionAccurateScope scope(this);
1116
1117 DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
1118 DCHECK(size == src0.SizeInBytes());
1119
1120 // When pushing multiple registers, the store order is chosen such that
1121 // Push(a, b) is equivalent to Push(a) followed by Push(b).
1122 switch (count) {
1123 case 1:
1124 DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
1125 str(src0, MemOperand(sp, -1 * size, PreIndex));
1126 break;
1127 case 2:
1128 DCHECK(src2.IsNone() && src3.IsNone());
1129 stp(src1, src0, MemOperand(sp, -2 * size, PreIndex));
1130 break;
1131 case 3:
1132 DCHECK(src3.IsNone());
1133 stp(src2, src1, MemOperand(sp, -3 * size, PreIndex));
1134 str(src0, MemOperand(sp, 2 * size));
1135 break;
1136 case 4:
1137 // Skip over 4 * size, then fill in the gap. This allows four W registers
1138 // to be pushed using sp, whilst maintaining 16-byte alignment for sp
1139 // at all times.
1140 stp(src3, src2, MemOperand(sp, -4 * size, PreIndex));
1141 stp(src1, src0, MemOperand(sp, 2 * size));
1142 break;
1143 default:
1144 UNREACHABLE();
1145 }
1146 }
1147
PopHelper(int count,int size,const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1148 void TurboAssembler::PopHelper(int count, int size, const CPURegister& dst0,
1149 const CPURegister& dst1, const CPURegister& dst2,
1150 const CPURegister& dst3) {
1151 // Ensure that we don't unintentially modify scratch or debug registers.
1152 InstructionAccurateScope scope(this);
1153
1154 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1155 DCHECK(size == dst0.SizeInBytes());
1156
1157 // When popping multiple registers, the load order is chosen such that
1158 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
1159 switch (count) {
1160 case 1:
1161 DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
1162 ldr(dst0, MemOperand(sp, 1 * size, PostIndex));
1163 break;
1164 case 2:
1165 DCHECK(dst2.IsNone() && dst3.IsNone());
1166 ldp(dst0, dst1, MemOperand(sp, 2 * size, PostIndex));
1167 break;
1168 case 3:
1169 DCHECK(dst3.IsNone());
1170 ldr(dst2, MemOperand(sp, 2 * size));
1171 ldp(dst0, dst1, MemOperand(sp, 3 * size, PostIndex));
1172 break;
1173 case 4:
1174 // Load the higher addresses first, then load the lower addresses and
1175 // skip the whole block in the second instruction. This allows four W
1176 // registers to be popped using sp, whilst maintaining 16-byte alignment
1177 // for sp at all times.
1178 ldp(dst2, dst3, MemOperand(sp, 2 * size));
1179 ldp(dst0, dst1, MemOperand(sp, 4 * size, PostIndex));
1180 break;
1181 default:
1182 UNREACHABLE();
1183 }
1184 }
1185
PokePair(const CPURegister & src1,const CPURegister & src2,int offset)1186 void TurboAssembler::PokePair(const CPURegister& src1, const CPURegister& src2,
1187 int offset) {
1188 DCHECK(AreSameSizeAndType(src1, src2));
1189 DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
1190 Stp(src1, src2, MemOperand(sp, offset));
1191 }
1192
PeekPair(const CPURegister & dst1,const CPURegister & dst2,int offset)1193 void MacroAssembler::PeekPair(const CPURegister& dst1, const CPURegister& dst2,
1194 int offset) {
1195 DCHECK(AreSameSizeAndType(dst1, dst2));
1196 DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
1197 Ldp(dst1, dst2, MemOperand(sp, offset));
1198 }
1199
PushCalleeSavedRegisters()1200 void MacroAssembler::PushCalleeSavedRegisters() {
1201 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1202 Pacibsp();
1203 #endif
1204
1205 {
1206 // Ensure that the macro-assembler doesn't use any scratch registers.
1207 InstructionAccurateScope scope(this);
1208
1209 MemOperand tos(sp, -2 * static_cast<int>(kXRegSize), PreIndex);
1210
1211 stp(d14, d15, tos);
1212 stp(d12, d13, tos);
1213 stp(d10, d11, tos);
1214 stp(d8, d9, tos);
1215
1216 STATIC_ASSERT(
1217 EntryFrameConstants::kCalleeSavedRegisterBytesPushedBeforeFpLrPair ==
1218 8 * kSystemPointerSize);
1219 stp(x29, x30, tos); // fp, lr
1220
1221 STATIC_ASSERT(
1222 EntryFrameConstants::kCalleeSavedRegisterBytesPushedAfterFpLrPair ==
1223 10 * kSystemPointerSize);
1224
1225 stp(x27, x28, tos);
1226 stp(x25, x26, tos);
1227 stp(x23, x24, tos);
1228 stp(x21, x22, tos);
1229 stp(x19, x20, tos);
1230 }
1231 }
1232
PopCalleeSavedRegisters()1233 void MacroAssembler::PopCalleeSavedRegisters() {
1234 {
1235 // Ensure that the macro-assembler doesn't use any scratch registers.
1236 InstructionAccurateScope scope(this);
1237
1238 MemOperand tos(sp, 2 * kXRegSize, PostIndex);
1239
1240 ldp(x19, x20, tos);
1241 ldp(x21, x22, tos);
1242 ldp(x23, x24, tos);
1243 ldp(x25, x26, tos);
1244 ldp(x27, x28, tos);
1245 ldp(x29, x30, tos);
1246
1247 ldp(d8, d9, tos);
1248 ldp(d10, d11, tos);
1249 ldp(d12, d13, tos);
1250 ldp(d14, d15, tos);
1251 }
1252
1253 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1254 Autibsp();
1255 #endif
1256 }
1257
AssertSpAligned()1258 void TurboAssembler::AssertSpAligned() {
1259 if (emit_debug_code()) {
1260 HardAbortScope hard_abort(this); // Avoid calls to Abort.
1261 // Arm64 requires the stack pointer to be 16-byte aligned prior to address
1262 // calculation.
1263 UseScratchRegisterScope scope(this);
1264 Register temp = scope.AcquireX();
1265 Mov(temp, sp);
1266 Tst(temp, 15);
1267 Check(eq, AbortReason::kUnexpectedStackPointer);
1268 }
1269 }
1270
CopySlots(int dst,Register src,Register slot_count)1271 void TurboAssembler::CopySlots(int dst, Register src, Register slot_count) {
1272 DCHECK(!src.IsZero());
1273 UseScratchRegisterScope scope(this);
1274 Register dst_reg = scope.AcquireX();
1275 SlotAddress(dst_reg, dst);
1276 SlotAddress(src, src);
1277 CopyDoubleWords(dst_reg, src, slot_count);
1278 }
1279
CopySlots(Register dst,Register src,Register slot_count)1280 void TurboAssembler::CopySlots(Register dst, Register src,
1281 Register slot_count) {
1282 DCHECK(!dst.IsZero() && !src.IsZero());
1283 SlotAddress(dst, dst);
1284 SlotAddress(src, src);
1285 CopyDoubleWords(dst, src, slot_count);
1286 }
1287
CopyDoubleWords(Register dst,Register src,Register count,CopyDoubleWordsMode mode)1288 void TurboAssembler::CopyDoubleWords(Register dst, Register src, Register count,
1289 CopyDoubleWordsMode mode) {
1290 DCHECK(!AreAliased(dst, src, count));
1291
1292 if (emit_debug_code()) {
1293 Register pointer1 = dst;
1294 Register pointer2 = src;
1295 if (mode == kSrcLessThanDst) {
1296 pointer1 = src;
1297 pointer2 = dst;
1298 }
1299 // Copy requires pointer1 < pointer2 || (pointer1 - pointer2) >= count.
1300 Label pointer1_below_pointer2;
1301 Subs(pointer1, pointer1, pointer2);
1302 B(lt, &pointer1_below_pointer2);
1303 Cmp(pointer1, count);
1304 Check(ge, AbortReason::kOffsetOutOfRange);
1305 Bind(&pointer1_below_pointer2);
1306 Add(pointer1, pointer1, pointer2);
1307 }
1308 static_assert(kSystemPointerSize == kDRegSize,
1309 "pointers must be the same size as doubles");
1310
1311 if (mode == kDstLessThanSrcAndReverse) {
1312 Add(src, src, Operand(count, LSL, kSystemPointerSizeLog2));
1313 Sub(src, src, kSystemPointerSize);
1314 }
1315
1316 int src_direction = (mode == kDstLessThanSrc) ? 1 : -1;
1317 int dst_direction = (mode == kSrcLessThanDst) ? -1 : 1;
1318
1319 UseScratchRegisterScope scope(this);
1320 VRegister temp0 = scope.AcquireD();
1321 VRegister temp1 = scope.AcquireD();
1322
1323 Label pairs, loop, done;
1324
1325 Tbz(count, 0, &pairs);
1326 Ldr(temp0, MemOperand(src, src_direction * kSystemPointerSize, PostIndex));
1327 Sub(count, count, 1);
1328 Str(temp0, MemOperand(dst, dst_direction * kSystemPointerSize, PostIndex));
1329
1330 Bind(&pairs);
1331 if (mode == kSrcLessThanDst) {
1332 // Adjust pointers for post-index ldp/stp with negative offset:
1333 Sub(dst, dst, kSystemPointerSize);
1334 Sub(src, src, kSystemPointerSize);
1335 } else if (mode == kDstLessThanSrcAndReverse) {
1336 Sub(src, src, kSystemPointerSize);
1337 }
1338 Bind(&loop);
1339 Cbz(count, &done);
1340 Ldp(temp0, temp1,
1341 MemOperand(src, 2 * src_direction * kSystemPointerSize, PostIndex));
1342 Sub(count, count, 2);
1343 if (mode == kDstLessThanSrcAndReverse) {
1344 Stp(temp1, temp0,
1345 MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
1346 } else {
1347 Stp(temp0, temp1,
1348 MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
1349 }
1350 B(&loop);
1351
1352 // TODO(all): large copies may benefit from using temporary Q registers
1353 // to copy four double words per iteration.
1354
1355 Bind(&done);
1356 }
1357
SlotAddress(Register dst,int slot_offset)1358 void TurboAssembler::SlotAddress(Register dst, int slot_offset) {
1359 Add(dst, sp, slot_offset << kSystemPointerSizeLog2);
1360 }
1361
SlotAddress(Register dst,Register slot_offset)1362 void TurboAssembler::SlotAddress(Register dst, Register slot_offset) {
1363 Add(dst, sp, Operand(slot_offset, LSL, kSystemPointerSizeLog2));
1364 }
1365
AssertFPCRState(Register fpcr)1366 void TurboAssembler::AssertFPCRState(Register fpcr) {
1367 if (emit_debug_code()) {
1368 Label unexpected_mode, done;
1369 UseScratchRegisterScope temps(this);
1370 if (fpcr.IsNone()) {
1371 fpcr = temps.AcquireX();
1372 Mrs(fpcr, FPCR);
1373 }
1374
1375 // Settings left to their default values:
1376 // - Assert that flush-to-zero is not set.
1377 Tbnz(fpcr, FZ_offset, &unexpected_mode);
1378 // - Assert that the rounding mode is nearest-with-ties-to-even.
1379 STATIC_ASSERT(FPTieEven == 0);
1380 Tst(fpcr, RMode_mask);
1381 B(eq, &done);
1382
1383 Bind(&unexpected_mode);
1384 Abort(AbortReason::kUnexpectedFPCRMode);
1385
1386 Bind(&done);
1387 }
1388 }
1389
CanonicalizeNaN(const VRegister & dst,const VRegister & src)1390 void TurboAssembler::CanonicalizeNaN(const VRegister& dst,
1391 const VRegister& src) {
1392 AssertFPCRState();
1393
1394 // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
1395 // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
1396 // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
1397 Fsub(dst, src, fp_zero);
1398 }
1399
LoadRoot(Register destination,RootIndex index)1400 void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
1401 // TODO(jbramley): Most root values are constants, and can be synthesized
1402 // without a load. Refer to the ARM back end for details.
1403 Ldr(destination,
1404 MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
1405 }
1406
Move(Register dst,Smi src)1407 void TurboAssembler::Move(Register dst, Smi src) { Mov(dst, src); }
1408
MovePair(Register dst0,Register src0,Register dst1,Register src1)1409 void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
1410 Register src1) {
1411 DCHECK_NE(dst0, dst1);
1412 if (dst0 != src1) {
1413 Mov(dst0, src0);
1414 Mov(dst1, src1);
1415 } else if (dst1 != src0) {
1416 // Swap the order of the moves to resolve the overlap.
1417 Mov(dst1, src1);
1418 Mov(dst0, src0);
1419 } else {
1420 // Worse case scenario, this is a swap.
1421 Swap(dst0, src0);
1422 }
1423 }
1424
Swap(Register lhs,Register rhs)1425 void TurboAssembler::Swap(Register lhs, Register rhs) {
1426 DCHECK(lhs.IsSameSizeAndType(rhs));
1427 DCHECK_NE(lhs, rhs);
1428 UseScratchRegisterScope temps(this);
1429 Register temp = temps.AcquireX();
1430 Mov(temp, rhs);
1431 Mov(rhs, lhs);
1432 Mov(lhs, temp);
1433 }
1434
Swap(VRegister lhs,VRegister rhs)1435 void TurboAssembler::Swap(VRegister lhs, VRegister rhs) {
1436 DCHECK(lhs.IsSameSizeAndType(rhs));
1437 DCHECK_NE(lhs, rhs);
1438 UseScratchRegisterScope temps(this);
1439 VRegister temp = VRegister::no_reg();
1440 if (lhs.IsS()) {
1441 temp = temps.AcquireS();
1442 } else if (lhs.IsD()) {
1443 temp = temps.AcquireD();
1444 } else {
1445 DCHECK(lhs.IsQ());
1446 temp = temps.AcquireQ();
1447 }
1448 Mov(temp, rhs);
1449 Mov(rhs, lhs);
1450 Mov(lhs, temp);
1451 }
1452
AssertSmi(Register object,AbortReason reason)1453 void TurboAssembler::AssertSmi(Register object, AbortReason reason) {
1454 if (emit_debug_code()) {
1455 STATIC_ASSERT(kSmiTag == 0);
1456 Tst(object, kSmiTagMask);
1457 Check(eq, reason);
1458 }
1459 }
1460
AssertNotSmi(Register object,AbortReason reason)1461 void MacroAssembler::AssertNotSmi(Register object, AbortReason reason) {
1462 if (emit_debug_code()) {
1463 STATIC_ASSERT(kSmiTag == 0);
1464 Tst(object, kSmiTagMask);
1465 Check(ne, reason);
1466 }
1467 }
1468
AssertConstructor(Register object)1469 void MacroAssembler::AssertConstructor(Register object) {
1470 if (emit_debug_code()) {
1471 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAConstructor);
1472
1473 UseScratchRegisterScope temps(this);
1474 Register temp = temps.AcquireX();
1475
1476 LoadMap(temp, object);
1477 Ldrb(temp, FieldMemOperand(temp, Map::kBitFieldOffset));
1478 Tst(temp, Operand(Map::Bits1::IsConstructorBit::kMask));
1479
1480 Check(ne, AbortReason::kOperandIsNotAConstructor);
1481 }
1482 }
1483
AssertFunction(Register object)1484 void MacroAssembler::AssertFunction(Register object) {
1485 if (emit_debug_code()) {
1486 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);
1487
1488 UseScratchRegisterScope temps(this);
1489 Register temp = temps.AcquireX();
1490
1491 CompareObjectType(object, temp, temp, JS_FUNCTION_TYPE);
1492 Check(eq, AbortReason::kOperandIsNotAFunction);
1493 }
1494 }
1495
AssertBoundFunction(Register object)1496 void MacroAssembler::AssertBoundFunction(Register object) {
1497 if (emit_debug_code()) {
1498 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotABoundFunction);
1499
1500 UseScratchRegisterScope temps(this);
1501 Register temp = temps.AcquireX();
1502
1503 CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
1504 Check(eq, AbortReason::kOperandIsNotABoundFunction);
1505 }
1506 }
1507
AssertGeneratorObject(Register object)1508 void MacroAssembler::AssertGeneratorObject(Register object) {
1509 if (!emit_debug_code()) return;
1510 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
1511
1512 // Load map
1513 UseScratchRegisterScope temps(this);
1514 Register temp = temps.AcquireX();
1515 LoadMap(temp, object);
1516
1517 Label do_check;
1518 // Load instance type and check if JSGeneratorObject
1519 CompareInstanceType(temp, temp, JS_GENERATOR_OBJECT_TYPE);
1520 B(eq, &do_check);
1521
1522 // Check if JSAsyncFunctionObject
1523 Cmp(temp, JS_ASYNC_FUNCTION_OBJECT_TYPE);
1524 B(eq, &do_check);
1525
1526 // Check if JSAsyncGeneratorObject
1527 Cmp(temp, JS_ASYNC_GENERATOR_OBJECT_TYPE);
1528
1529 bind(&do_check);
1530 // Restore generator object to register and perform assertion
1531 Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
1532 }
1533
AssertUndefinedOrAllocationSite(Register object)1534 void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
1535 if (emit_debug_code()) {
1536 UseScratchRegisterScope temps(this);
1537 Register scratch = temps.AcquireX();
1538 Label done_checking;
1539 AssertNotSmi(object);
1540 JumpIfRoot(object, RootIndex::kUndefinedValue, &done_checking);
1541 LoadMap(scratch, object);
1542 CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
1543 Assert(eq, AbortReason::kExpectedUndefinedOrCell);
1544 Bind(&done_checking);
1545 }
1546 }
1547
AssertPositiveOrZero(Register value)1548 void TurboAssembler::AssertPositiveOrZero(Register value) {
1549 if (emit_debug_code()) {
1550 Label done;
1551 int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
1552 Tbz(value, sign_bit, &done);
1553 Abort(AbortReason::kUnexpectedNegativeValue);
1554 Bind(&done);
1555 }
1556 }
1557
CallRuntime(const Runtime::Function * f,int num_arguments,SaveFPRegsMode save_doubles)1558 void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
1559 SaveFPRegsMode save_doubles) {
1560 // All arguments must be on the stack before this function is called.
1561 // x0 holds the return value after the call.
1562
1563 // Check that the number of arguments matches what the function expects.
1564 // If f->nargs is -1, the function can accept a variable number of arguments.
1565 CHECK(f->nargs < 0 || f->nargs == num_arguments);
1566
1567 // Place the necessary arguments.
1568 Mov(x0, num_arguments);
1569 Mov(x1, ExternalReference::Create(f));
1570
1571 Handle<Code> code =
1572 CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
1573 Call(code, RelocInfo::CODE_TARGET);
1574 }
1575
JumpToExternalReference(const ExternalReference & builtin,bool builtin_exit_frame)1576 void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
1577 bool builtin_exit_frame) {
1578 Mov(x1, builtin);
1579 Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
1580 kArgvOnStack, builtin_exit_frame);
1581 Jump(code, RelocInfo::CODE_TARGET);
1582 }
1583
JumpToInstructionStream(Address entry)1584 void MacroAssembler::JumpToInstructionStream(Address entry) {
1585 Ldr(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1586 Br(kOffHeapTrampolineRegister);
1587 }
1588
TailCallRuntime(Runtime::FunctionId fid)1589 void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
1590 const Runtime::Function* function = Runtime::FunctionForId(fid);
1591 DCHECK_EQ(1, function->result_size);
1592 if (function->nargs >= 0) {
1593 // TODO(1236192): Most runtime routines don't need the number of
1594 // arguments passed in because it is constant. At some point we
1595 // should remove this need and make the runtime routine entry code
1596 // smarter.
1597 Mov(x0, function->nargs);
1598 }
1599 JumpToExternalReference(ExternalReference::Create(fid));
1600 }
1601
ActivationFrameAlignment()1602 int TurboAssembler::ActivationFrameAlignment() {
1603 #if V8_HOST_ARCH_ARM64
1604 // Running on the real platform. Use the alignment as mandated by the local
1605 // environment.
1606 // Note: This will break if we ever start generating snapshots on one ARM
1607 // platform for another ARM platform with a different alignment.
1608 return base::OS::ActivationFrameAlignment();
1609 #else // V8_HOST_ARCH_ARM64
1610 // If we are using the simulator then we should always align to the expected
1611 // alignment. As the simulator is used to generate snapshots we do not know
1612 // if the target platform will need alignment, so this is controlled from a
1613 // flag.
1614 return FLAG_sim_stack_alignment;
1615 #endif // V8_HOST_ARCH_ARM64
1616 }
1617
CallCFunction(ExternalReference function,int num_of_reg_args)1618 void TurboAssembler::CallCFunction(ExternalReference function,
1619 int num_of_reg_args) {
1620 CallCFunction(function, num_of_reg_args, 0);
1621 }
1622
CallCFunction(ExternalReference function,int num_of_reg_args,int num_of_double_args)1623 void TurboAssembler::CallCFunction(ExternalReference function,
1624 int num_of_reg_args,
1625 int num_of_double_args) {
1626 UseScratchRegisterScope temps(this);
1627 Register temp = temps.AcquireX();
1628 Mov(temp, function);
1629 CallCFunction(temp, num_of_reg_args, num_of_double_args);
1630 }
1631
1632 static const int kRegisterPassedArguments = 8;
1633
CallCFunction(Register function,int num_of_reg_args,int num_of_double_args)1634 void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
1635 int num_of_double_args) {
1636 DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
1637 DCHECK(has_frame());
1638
1639 // If we're passing doubles, we're limited to the following prototypes
1640 // (defined by ExternalReference::Type):
1641 // BUILTIN_COMPARE_CALL: int f(double, double)
1642 // BUILTIN_FP_FP_CALL: double f(double, double)
1643 // BUILTIN_FP_CALL: double f(double)
1644 // BUILTIN_FP_INT_CALL: double f(double, int)
1645 if (num_of_double_args > 0) {
1646 DCHECK_LE(num_of_reg_args, 1);
1647 DCHECK_LE(num_of_double_args + num_of_reg_args, 2);
1648 }
1649
1650 // Save the frame pointer and PC so that the stack layout remains iterable,
1651 // even without an ExitFrame which normally exists between JS and C frames.
1652 Register pc_scratch = x4;
1653 Register addr_scratch = x5;
1654 Push(pc_scratch, addr_scratch);
1655
1656 Label get_pc;
1657 Bind(&get_pc);
1658 Adr(pc_scratch, &get_pc);
1659
1660 // See x64 code for reasoning about how to address the isolate data fields.
1661 if (root_array_available()) {
1662 Str(pc_scratch,
1663 MemOperand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset()));
1664 Str(fp,
1665 MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
1666 } else {
1667 DCHECK_NOT_NULL(isolate());
1668 Mov(addr_scratch,
1669 ExternalReference::fast_c_call_caller_pc_address(isolate()));
1670 Str(pc_scratch, MemOperand(addr_scratch));
1671 Mov(addr_scratch,
1672 ExternalReference::fast_c_call_caller_fp_address(isolate()));
1673 Str(fp, MemOperand(addr_scratch));
1674 }
1675
1676 Pop(addr_scratch, pc_scratch);
1677
1678 // Call directly. The function called cannot cause a GC, or allow preemption,
1679 // so the return address in the link register stays correct.
1680 Call(function);
1681
1682 // We don't unset the PC; the FP is the source of truth.
1683 if (root_array_available()) {
1684 Str(xzr,
1685 MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
1686 } else {
1687 DCHECK_NOT_NULL(isolate());
1688 Push(addr_scratch, xzr);
1689 Mov(addr_scratch,
1690 ExternalReference::fast_c_call_caller_fp_address(isolate()));
1691 Str(xzr, MemOperand(addr_scratch));
1692 Pop(xzr, addr_scratch);
1693 }
1694
1695 if (num_of_reg_args > kRegisterPassedArguments) {
1696 // Drop the register passed arguments.
1697 int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
1698 Drop(claim_slots);
1699 }
1700 }
1701
LoadFromConstantsTable(Register destination,int constant_index)1702 void TurboAssembler::LoadFromConstantsTable(Register destination,
1703 int constant_index) {
1704 DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
1705 LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
1706 LoadTaggedPointerField(
1707 destination, FieldMemOperand(destination, FixedArray::OffsetOfElementAt(
1708 constant_index)));
1709 }
1710
LoadRootRelative(Register destination,int32_t offset)1711 void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
1712 Ldr(destination, MemOperand(kRootRegister, offset));
1713 }
1714
LoadRootRegisterOffset(Register destination,intptr_t offset)1715 void TurboAssembler::LoadRootRegisterOffset(Register destination,
1716 intptr_t offset) {
1717 if (offset == 0) {
1718 Mov(destination, kRootRegister);
1719 } else {
1720 Add(destination, kRootRegister, offset);
1721 }
1722 }
1723
Jump(Register target,Condition cond)1724 void TurboAssembler::Jump(Register target, Condition cond) {
1725 if (cond == nv) return;
1726 Label done;
1727 if (cond != al) B(NegateCondition(cond), &done);
1728 Br(target);
1729 Bind(&done);
1730 }
1731
JumpHelper(int64_t offset,RelocInfo::Mode rmode,Condition cond)1732 void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
1733 Condition cond) {
1734 if (cond == nv) return;
1735 Label done;
1736 if (cond != al) B(NegateCondition(cond), &done);
1737 if (CanUseNearCallOrJump(rmode)) {
1738 DCHECK(IsNearCallOffset(offset));
1739 near_jump(static_cast<int>(offset), rmode);
1740 } else {
1741 UseScratchRegisterScope temps(this);
1742 Register temp = temps.AcquireX();
1743 uint64_t imm = reinterpret_cast<uint64_t>(pc_) + offset * kInstrSize;
1744 Mov(temp, Immediate(imm, rmode));
1745 Br(temp);
1746 }
1747 Bind(&done);
1748 }
1749
1750 namespace {
1751
1752 // The calculated offset is either:
1753 // * the 'target' input unmodified if this is a Wasm call, or
1754 // * the offset of the target from the current PC, in instructions, for any
1755 // other type of call.
CalculateTargetOffset(Address target,RelocInfo::Mode rmode,byte * pc)1756 static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
1757 byte* pc) {
1758 int64_t offset = static_cast<int64_t>(target);
1759 // The target of WebAssembly calls is still an index instead of an actual
1760 // address at this point, and needs to be encoded as-is.
1761 if (rmode != RelocInfo::WASM_CALL && rmode != RelocInfo::WASM_STUB_CALL) {
1762 offset -= reinterpret_cast<int64_t>(pc);
1763 DCHECK_EQ(offset % kInstrSize, 0);
1764 offset = offset / static_cast<int>(kInstrSize);
1765 }
1766 return offset;
1767 }
1768 } // namespace
1769
Jump(Address target,RelocInfo::Mode rmode,Condition cond)1770 void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
1771 Condition cond) {
1772 JumpHelper(CalculateTargetOffset(target, rmode, pc_), rmode, cond);
1773 }
1774
Jump(Handle<Code> code,RelocInfo::Mode rmode,Condition cond)1775 void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
1776 Condition cond) {
1777 DCHECK(RelocInfo::IsCodeTarget(rmode));
1778 DCHECK_IMPLIES(options().isolate_independent_code,
1779 Builtins::IsIsolateIndependentBuiltin(*code));
1780
1781 if (options().inline_offheap_trampolines) {
1782 int builtin_index = Builtins::kNoBuiltinId;
1783 if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) {
1784 // Inline the trampoline.
1785 RecordCommentForOffHeapTrampoline(builtin_index);
1786 CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
1787 UseScratchRegisterScope temps(this);
1788 Register scratch = temps.AcquireX();
1789 EmbeddedData d = EmbeddedData::FromBlob();
1790 Address entry = d.InstructionStartOfBuiltin(builtin_index);
1791 Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1792 Jump(scratch, cond);
1793 return;
1794 }
1795 }
1796
1797 if (CanUseNearCallOrJump(rmode)) {
1798 EmbeddedObjectIndex index = AddEmbeddedObject(code);
1799 DCHECK(is_int32(index));
1800 JumpHelper(static_cast<int64_t>(index), rmode, cond);
1801 } else {
1802 Jump(code.address(), rmode, cond);
1803 }
1804 }
1805
Jump(const ExternalReference & reference)1806 void TurboAssembler::Jump(const ExternalReference& reference) {
1807 UseScratchRegisterScope temps(this);
1808 Register scratch = temps.AcquireX();
1809 Mov(scratch, reference);
1810 Jump(scratch);
1811 }
1812
Call(Register target)1813 void TurboAssembler::Call(Register target) {
1814 BlockPoolsScope scope(this);
1815 Blr(target);
1816 }
1817
Call(Address target,RelocInfo::Mode rmode)1818 void TurboAssembler::Call(Address target, RelocInfo::Mode rmode) {
1819 BlockPoolsScope scope(this);
1820
1821 if (CanUseNearCallOrJump(rmode)) {
1822 int64_t offset = CalculateTargetOffset(target, rmode, pc_);
1823 DCHECK(IsNearCallOffset(offset));
1824 near_call(static_cast<int>(offset), rmode);
1825 } else {
1826 IndirectCall(target, rmode);
1827 }
1828 }
1829
Call(Handle<Code> code,RelocInfo::Mode rmode)1830 void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
1831 DCHECK_IMPLIES(options().isolate_independent_code,
1832 Builtins::IsIsolateIndependentBuiltin(*code));
1833 BlockPoolsScope scope(this);
1834
1835 if (options().inline_offheap_trampolines) {
1836 int builtin_index = Builtins::kNoBuiltinId;
1837 if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) {
1838 // Inline the trampoline.
1839 CallBuiltin(builtin_index);
1840 return;
1841 }
1842 }
1843
1844 DCHECK(code->IsExecutable());
1845 if (CanUseNearCallOrJump(rmode)) {
1846 EmbeddedObjectIndex index = AddEmbeddedObject(code);
1847 DCHECK(is_int32(index));
1848 near_call(static_cast<int32_t>(index), rmode);
1849 } else {
1850 IndirectCall(code.address(), rmode);
1851 }
1852 }
1853
Call(ExternalReference target)1854 void TurboAssembler::Call(ExternalReference target) {
1855 UseScratchRegisterScope temps(this);
1856 Register temp = temps.AcquireX();
1857 Mov(temp, target);
1858 Call(temp);
1859 }
1860
LoadEntryFromBuiltinIndex(Register builtin_index)1861 void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
1862 // The builtin_index register contains the builtin index as a Smi.
1863 // Untagging is folded into the indexing operand below.
1864 if (SmiValuesAre32Bits()) {
1865 Asr(builtin_index, builtin_index, kSmiShift - kSystemPointerSizeLog2);
1866 Add(builtin_index, builtin_index,
1867 IsolateData::builtin_entry_table_offset());
1868 Ldr(builtin_index, MemOperand(kRootRegister, builtin_index));
1869 } else {
1870 DCHECK(SmiValuesAre31Bits());
1871 if (COMPRESS_POINTERS_BOOL) {
1872 Add(builtin_index, kRootRegister,
1873 Operand(builtin_index.W(), SXTW, kSystemPointerSizeLog2 - kSmiShift));
1874 } else {
1875 Add(builtin_index, kRootRegister,
1876 Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiShift));
1877 }
1878 Ldr(builtin_index,
1879 MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
1880 }
1881 }
1882
LoadEntryFromBuiltinIndex(Builtins::Name builtin_index,Register destination)1883 void TurboAssembler::LoadEntryFromBuiltinIndex(Builtins::Name builtin_index,
1884 Register destination) {
1885 Ldr(destination,
1886 MemOperand(kRootRegister,
1887 IsolateData::builtin_entry_slot_offset(builtin_index)));
1888 }
1889
CallBuiltinByIndex(Register builtin_index)1890 void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
1891 LoadEntryFromBuiltinIndex(builtin_index);
1892 Call(builtin_index);
1893 }
1894
CallBuiltin(int builtin_index)1895 void TurboAssembler::CallBuiltin(int builtin_index) {
1896 DCHECK(Builtins::IsBuiltinId(builtin_index));
1897 RecordCommentForOffHeapTrampoline(builtin_index);
1898 CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
1899 UseScratchRegisterScope temps(this);
1900 Register scratch = temps.AcquireX();
1901 EmbeddedData d = EmbeddedData::FromBlob();
1902 Address entry = d.InstructionStartOfBuiltin(builtin_index);
1903 Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1904 Call(scratch);
1905 }
1906
LoadCodeObjectEntry(Register destination,Register code_object)1907 void TurboAssembler::LoadCodeObjectEntry(Register destination,
1908 Register code_object) {
1909 // Code objects are called differently depending on whether we are generating
1910 // builtin code (which will later be embedded into the binary) or compiling
1911 // user JS code at runtime.
1912 // * Builtin code runs in --jitless mode and thus must not call into on-heap
1913 // Code targets. Instead, we dispatch through the builtins entry table.
1914 // * Codegen at runtime does not have this restriction and we can use the
1915 // shorter, branchless instruction sequence. The assumption here is that
1916 // targets are usually generated code and not builtin Code objects.
1917
1918 if (options().isolate_independent_code) {
1919 DCHECK(root_array_available());
1920 Label if_code_is_off_heap, out;
1921
1922 UseScratchRegisterScope temps(this);
1923 Register scratch = temps.AcquireX();
1924
1925 DCHECK(!AreAliased(destination, scratch));
1926 DCHECK(!AreAliased(code_object, scratch));
1927
1928 // Check whether the Code object is an off-heap trampoline. If so, call its
1929 // (off-heap) entry point directly without going through the (on-heap)
1930 // trampoline. Otherwise, just call the Code object as always.
1931
1932 Ldrsw(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
1933 Tst(scratch, Operand(Code::IsOffHeapTrampoline::kMask));
1934 B(ne, &if_code_is_off_heap);
1935
1936 // Not an off-heap trampoline object, the entry point is at
1937 // Code::raw_instruction_start().
1938 Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
1939 B(&out);
1940
1941 // An off-heap trampoline, the entry point is loaded from the builtin entry
1942 // table.
1943 bind(&if_code_is_off_heap);
1944 Ldrsw(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
1945 Lsl(destination, scratch, kSystemPointerSizeLog2);
1946 Add(destination, destination, kRootRegister);
1947 Ldr(destination,
1948 MemOperand(destination, IsolateData::builtin_entry_table_offset()));
1949
1950 bind(&out);
1951 } else {
1952 Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
1953 }
1954 }
1955
CallCodeObject(Register code_object)1956 void TurboAssembler::CallCodeObject(Register code_object) {
1957 LoadCodeObjectEntry(code_object, code_object);
1958 Call(code_object);
1959 }
1960
JumpCodeObject(Register code_object)1961 void TurboAssembler::JumpCodeObject(Register code_object) {
1962 LoadCodeObjectEntry(code_object, code_object);
1963
1964 UseScratchRegisterScope temps(this);
1965 if (code_object != x17) {
1966 temps.Exclude(x17);
1967 Mov(x17, code_object);
1968 }
1969 Jump(x17);
1970 }
1971
StoreReturnAddressAndCall(Register target)1972 void TurboAssembler::StoreReturnAddressAndCall(Register target) {
1973 // This generates the final instruction sequence for calls to C functions
1974 // once an exit frame has been constructed.
1975 //
1976 // Note that this assumes the caller code (i.e. the Code object currently
1977 // being generated) is immovable or that the callee function cannot trigger
1978 // GC, since the callee function will return to it.
1979
1980 UseScratchRegisterScope temps(this);
1981 temps.Exclude(x16, x17);
1982
1983 Label return_location;
1984 Adr(x17, &return_location);
1985 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1986 Add(x16, sp, kSystemPointerSize);
1987 Pacib1716();
1988 #endif
1989 Poke(x17, 0);
1990
1991 if (emit_debug_code()) {
1992 // Verify that the slot below fp[kSPOffset]-8 points to the signed return
1993 // location.
1994 Ldr(x16, MemOperand(fp, ExitFrameConstants::kSPOffset));
1995 Ldr(x16, MemOperand(x16, -static_cast<int64_t>(kXRegSize)));
1996 Cmp(x16, x17);
1997 Check(eq, AbortReason::kReturnAddressNotFoundInFrame);
1998 }
1999
2000 Blr(target);
2001 Bind(&return_location);
2002 }
2003
IndirectCall(Address target,RelocInfo::Mode rmode)2004 void TurboAssembler::IndirectCall(Address target, RelocInfo::Mode rmode) {
2005 UseScratchRegisterScope temps(this);
2006 Register temp = temps.AcquireX();
2007 Mov(temp, Immediate(target, rmode));
2008 Blr(temp);
2009 }
2010
IsNearCallOffset(int64_t offset)2011 bool TurboAssembler::IsNearCallOffset(int64_t offset) {
2012 return is_int26(offset);
2013 }
2014
CallForDeoptimization(Builtins::Name target,int deopt_id,Label * exit,DeoptimizeKind kind,Label * jump_deoptimization_entry_label)2015 void TurboAssembler::CallForDeoptimization(
2016 Builtins::Name target, int deopt_id, Label* exit, DeoptimizeKind kind,
2017 Label* jump_deoptimization_entry_label) {
2018 BlockPoolsScope scope(this);
2019 bl(jump_deoptimization_entry_label);
2020 DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
2021 (kind == DeoptimizeKind::kLazy)
2022 ? Deoptimizer::kLazyDeoptExitSize
2023 : Deoptimizer::kNonLazyDeoptExitSize);
2024 USE(exit, kind);
2025 }
2026
PrepareForTailCall(Register callee_args_count,Register caller_args_count,Register scratch0,Register scratch1)2027 void TurboAssembler::PrepareForTailCall(Register callee_args_count,
2028 Register caller_args_count,
2029 Register scratch0, Register scratch1) {
2030 DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
2031
2032 // Calculate the end of destination area where we will put the arguments
2033 // after we drop current frame. We add kSystemPointerSize to count the
2034 // receiver argument which is not included into formal parameters count.
2035 Register dst_reg = scratch0;
2036 Add(dst_reg, fp, Operand(caller_args_count, LSL, kSystemPointerSizeLog2));
2037 Add(dst_reg, dst_reg,
2038 StandardFrameConstants::kCallerSPOffset + kSystemPointerSize);
2039 // Round dst_reg up to a multiple of 16 bytes, so that we overwrite any
2040 // potential padding.
2041 Add(dst_reg, dst_reg, 15);
2042 Bic(dst_reg, dst_reg, 15);
2043
2044 Register src_reg = caller_args_count;
2045 // Calculate the end of source area. +kSystemPointerSize is for the receiver.
2046 Add(src_reg, sp, Operand(callee_args_count, LSL, kSystemPointerSizeLog2));
2047 Add(src_reg, src_reg, kSystemPointerSize);
2048
2049 // Round src_reg up to a multiple of 16 bytes, so we include any potential
2050 // padding in the copy.
2051 Add(src_reg, src_reg, 15);
2052 Bic(src_reg, src_reg, 15);
2053
2054 if (FLAG_debug_code) {
2055 Cmp(src_reg, dst_reg);
2056 Check(lo, AbortReason::kStackAccessBelowStackPointer);
2057 }
2058
2059 // Restore caller's frame pointer and return address now as they will be
2060 // overwritten by the copying loop.
2061 RestoreFPAndLR();
2062
2063 // Now copy callee arguments to the caller frame going backwards to avoid
2064 // callee arguments corruption (source and destination areas could overlap).
2065
2066 // Both src_reg and dst_reg are pointing to the word after the one to copy,
2067 // so they must be pre-decremented in the loop.
2068 Register tmp_reg = scratch1;
2069 Label loop, entry;
2070 B(&entry);
2071 bind(&loop);
2072 Ldr(tmp_reg, MemOperand(src_reg, -kSystemPointerSize, PreIndex));
2073 Str(tmp_reg, MemOperand(dst_reg, -kSystemPointerSize, PreIndex));
2074 bind(&entry);
2075 Cmp(sp, src_reg);
2076 B(ne, &loop);
2077
2078 // Leave current frame.
2079 Mov(sp, dst_reg);
2080 }
2081
LoadStackLimit(Register destination,StackLimitKind kind)2082 void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
2083 DCHECK(root_array_available());
2084 Isolate* isolate = this->isolate();
2085 ExternalReference limit =
2086 kind == StackLimitKind::kRealStackLimit
2087 ? ExternalReference::address_of_real_jslimit(isolate)
2088 : ExternalReference::address_of_jslimit(isolate);
2089 DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
2090
2091 intptr_t offset =
2092 TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
2093 Ldr(destination, MemOperand(kRootRegister, offset));
2094 }
2095
StackOverflowCheck(Register num_args,Label * stack_overflow)2096 void MacroAssembler::StackOverflowCheck(Register num_args,
2097 Label* stack_overflow) {
2098 UseScratchRegisterScope temps(this);
2099 Register scratch = temps.AcquireX();
2100
2101 // Check the stack for overflow.
2102 // We are not trying to catch interruptions (e.g. debug break and
2103 // preemption) here, so the "real stack limit" is checked.
2104
2105 LoadStackLimit(scratch, StackLimitKind::kRealStackLimit);
2106 // Make scratch the space we have left. The stack might already be overflowed
2107 // here which will cause scratch to become negative.
2108 Sub(scratch, sp, scratch);
2109 // Check if the arguments will overflow the stack.
2110 Cmp(scratch, Operand(num_args, LSL, kSystemPointerSizeLog2));
2111 B(le, stack_overflow);
2112 }
2113
InvokePrologue(Register formal_parameter_count,Register actual_argument_count,Label * done,InvokeFlag flag)2114 void MacroAssembler::InvokePrologue(Register formal_parameter_count,
2115 Register actual_argument_count, Label* done,
2116 InvokeFlag flag) {
2117 // x0: actual arguments count.
2118 // x1: function (passed through to callee).
2119 // x2: expected arguments count.
2120 // x3: new target
2121 Label regular_invoke;
2122 DCHECK_EQ(actual_argument_count, x0);
2123 DCHECK_EQ(formal_parameter_count, x2);
2124
2125 #ifdef V8_NO_ARGUMENTS_ADAPTOR
2126 // If the formal parameter count is equal to the adaptor sentinel, no need
2127 // to push undefined value as arguments.
2128 Cmp(formal_parameter_count, Operand(kDontAdaptArgumentsSentinel));
2129 B(eq, ®ular_invoke);
2130
2131 // If overapplication or if the actual argument count is equal to the
2132 // formal parameter count, no need to push extra undefined values.
2133 Register extra_argument_count = x2;
2134 Subs(extra_argument_count, formal_parameter_count, actual_argument_count);
2135 B(le, ®ular_invoke);
2136
2137 // The stack pointer in arm64 needs to be 16-byte aligned. We might need to
2138 // (1) add an extra padding or (2) remove (re-use) the extra padding already
2139 // in the stack. Let {slots_to_copy} be the number of slots (arguments) to
2140 // move up in the stack and let {slots_to_claim} be the number of extra stack
2141 // slots to claim.
2142 Label even_extra_count, skip_move;
2143 Register slots_to_copy = x4;
2144 Register slots_to_claim = x5;
2145
2146 Add(slots_to_copy, actual_argument_count, 1); // Copy with receiver.
2147 Mov(slots_to_claim, extra_argument_count);
2148 Tbz(extra_argument_count, 0, &even_extra_count);
2149
2150 // Calculate {slots_to_claim} when {extra_argument_count} is odd.
2151 // If {actual_argument_count} is even, we need one extra padding slot
2152 // {slots_to_claim = extra_argument_count + 1}.
2153 // If {actual_argument_count} is odd, we know that the
2154 // original arguments will have a padding slot that we can reuse
2155 // {slots_to_claim = extra_argument_count - 1}.
2156 {
2157 Register scratch = x11;
2158 Add(slots_to_claim, extra_argument_count, 1);
2159 And(scratch, actual_argument_count, 1);
2160 Eor(scratch, scratch, 1);
2161 Sub(slots_to_claim, slots_to_claim, Operand(scratch, LSL, 1));
2162 }
2163
2164 Bind(&even_extra_count);
2165 Cbz(slots_to_claim, &skip_move);
2166
2167 Label stack_overflow;
2168 StackOverflowCheck(slots_to_claim, &stack_overflow);
2169 Claim(slots_to_claim);
2170
2171 // Move the arguments already in the stack including the receiver.
2172 {
2173 Register src = x6;
2174 Register dst = x7;
2175 SlotAddress(src, slots_to_claim);
2176 SlotAddress(dst, 0);
2177 CopyDoubleWords(dst, src, slots_to_copy);
2178 }
2179
2180 Bind(&skip_move);
2181 Register actual_argument_with_receiver = x4;
2182 Register pointer_next_value = x5;
2183 Add(actual_argument_with_receiver, actual_argument_count,
2184 1); // {slots_to_copy} was scratched.
2185
2186 // Copy extra arguments as undefined values.
2187 {
2188 Label loop;
2189 Register undefined_value = x6;
2190 Register count = x7;
2191 LoadRoot(undefined_value, RootIndex::kUndefinedValue);
2192 SlotAddress(pointer_next_value, actual_argument_with_receiver);
2193 Mov(count, extra_argument_count);
2194 Bind(&loop);
2195 Str(undefined_value,
2196 MemOperand(pointer_next_value, kSystemPointerSize, PostIndex));
2197 Subs(count, count, 1);
2198 Cbnz(count, &loop);
2199 }
2200
2201 // Set padding if needed.
2202 {
2203 Label skip;
2204 Register total_args_slots = x4;
2205 Add(total_args_slots, actual_argument_with_receiver, extra_argument_count);
2206 Tbz(total_args_slots, 0, &skip);
2207 Str(padreg, MemOperand(pointer_next_value));
2208 Bind(&skip);
2209 }
2210 B(®ular_invoke);
2211
2212 bind(&stack_overflow);
2213 {
2214 FrameScope frame(this,
2215 has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
2216 CallRuntime(Runtime::kThrowStackOverflow);
2217 Unreachable();
2218 }
2219 #else
2220 // Check whether the expected and actual arguments count match. The registers
2221 // are set up according to contract with ArgumentsAdaptorTrampoline.ct.
2222 // If actual == expected perform a regular invocation.
2223 Cmp(formal_parameter_count, actual_argument_count);
2224 B(eq, ®ular_invoke);
2225
2226 // The argument counts mismatch, generate a call to the argument adaptor.
2227 Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
2228 if (flag == CALL_FUNCTION) {
2229 Call(adaptor);
2230 // If the arg counts don't match, no extra code is emitted by
2231 // MAsm::InvokeFunctionCode and we can just fall through.
2232 B(done);
2233 } else {
2234 Jump(adaptor, RelocInfo::CODE_TARGET);
2235 }
2236 #endif
2237
2238 Bind(®ular_invoke);
2239 }
2240
CallDebugOnFunctionCall(Register fun,Register new_target,Register expected_parameter_count,Register actual_parameter_count)2241 void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
2242 Register expected_parameter_count,
2243 Register actual_parameter_count) {
2244 // Load receiver to pass it later to DebugOnFunctionCall hook.
2245 Peek(x4, ReceiverOperand(actual_parameter_count));
2246 FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
2247
2248 if (!new_target.is_valid()) new_target = padreg;
2249
2250 // Save values on stack.
2251 SmiTag(expected_parameter_count);
2252 SmiTag(actual_parameter_count);
2253 Push(expected_parameter_count, actual_parameter_count, new_target, fun);
2254 Push(fun, x4);
2255 CallRuntime(Runtime::kDebugOnFunctionCall);
2256
2257 // Restore values from stack.
2258 Pop(fun, new_target, actual_parameter_count, expected_parameter_count);
2259 SmiUntag(actual_parameter_count);
2260 SmiUntag(expected_parameter_count);
2261 }
2262
InvokeFunctionCode(Register function,Register new_target,Register expected_parameter_count,Register actual_parameter_count,InvokeFlag flag)2263 void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
2264 Register expected_parameter_count,
2265 Register actual_parameter_count,
2266 InvokeFlag flag) {
2267 // You can't call a function without a valid frame.
2268 DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
2269 DCHECK_EQ(function, x1);
2270 DCHECK_IMPLIES(new_target.is_valid(), new_target == x3);
2271
2272 // On function call, call into the debugger if necessary.
2273 Label debug_hook, continue_after_hook;
2274 {
2275 Mov(x4, ExternalReference::debug_hook_on_function_call_address(isolate()));
2276 Ldrsb(x4, MemOperand(x4));
2277 Cbnz(x4, &debug_hook);
2278 }
2279 bind(&continue_after_hook);
2280
2281 // Clear the new.target register if not given.
2282 if (!new_target.is_valid()) {
2283 LoadRoot(x3, RootIndex::kUndefinedValue);
2284 }
2285
2286 Label done;
2287 InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
2288
2289 // If actual != expected, InvokePrologue will have handled the call through
2290 // the argument adaptor mechanism.
2291 // The called function expects the call kind in x5.
2292 // We call indirectly through the code field in the function to
2293 // allow recompilation to take effect without changing any of the
2294 // call sites.
2295 Register code = kJavaScriptCallCodeStartRegister;
2296 LoadTaggedPointerField(code,
2297 FieldMemOperand(function, JSFunction::kCodeOffset));
2298 if (flag == CALL_FUNCTION) {
2299 CallCodeObject(code);
2300 } else {
2301 DCHECK(flag == JUMP_FUNCTION);
2302 JumpCodeObject(code);
2303 }
2304 B(&done);
2305
2306 // Deferred debug hook.
2307 bind(&debug_hook);
2308 CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
2309 actual_parameter_count);
2310 B(&continue_after_hook);
2311
2312 // Continue here if InvokePrologue does handle the invocation due to
2313 // mismatched parameter counts.
2314 Bind(&done);
2315 }
2316
ReceiverOperand(Register arg_count)2317 Operand MacroAssembler::ReceiverOperand(Register arg_count) {
2318 return Operand(0);
2319 }
2320
InvokeFunctionWithNewTarget(Register function,Register new_target,Register actual_parameter_count,InvokeFlag flag)2321 void MacroAssembler::InvokeFunctionWithNewTarget(
2322 Register function, Register new_target, Register actual_parameter_count,
2323 InvokeFlag flag) {
2324 // You can't call a function without a valid frame.
2325 DCHECK(flag == JUMP_FUNCTION || has_frame());
2326
2327 // Contract with called JS functions requires that function is passed in x1.
2328 // (See FullCodeGenerator::Generate().)
2329 DCHECK_EQ(function, x1);
2330
2331 Register expected_parameter_count = x2;
2332
2333 LoadTaggedPointerField(cp,
2334 FieldMemOperand(function, JSFunction::kContextOffset));
2335 // The number of arguments is stored as an int32_t, and -1 is a marker
2336 // (kDontAdaptArgumentsSentinel), so we need sign
2337 // extension to correctly handle it.
2338 LoadTaggedPointerField(
2339 expected_parameter_count,
2340 FieldMemOperand(function, JSFunction::kSharedFunctionInfoOffset));
2341 Ldrh(expected_parameter_count,
2342 FieldMemOperand(expected_parameter_count,
2343 SharedFunctionInfo::kFormalParameterCountOffset));
2344
2345 InvokeFunctionCode(function, new_target, expected_parameter_count,
2346 actual_parameter_count, flag);
2347 }
2348
InvokeFunction(Register function,Register expected_parameter_count,Register actual_parameter_count,InvokeFlag flag)2349 void MacroAssembler::InvokeFunction(Register function,
2350 Register expected_parameter_count,
2351 Register actual_parameter_count,
2352 InvokeFlag flag) {
2353 // You can't call a function without a valid frame.
2354 DCHECK(flag == JUMP_FUNCTION || has_frame());
2355
2356 // Contract with called JS functions requires that function is passed in x1.
2357 // (See FullCodeGenerator::Generate().)
2358 DCHECK_EQ(function, x1);
2359
2360 // Set up the context.
2361 LoadTaggedPointerField(cp,
2362 FieldMemOperand(function, JSFunction::kContextOffset));
2363
2364 InvokeFunctionCode(function, no_reg, expected_parameter_count,
2365 actual_parameter_count, flag);
2366 }
2367
TryConvertDoubleToInt64(Register result,DoubleRegister double_input,Label * done)2368 void TurboAssembler::TryConvertDoubleToInt64(Register result,
2369 DoubleRegister double_input,
2370 Label* done) {
2371 // Try to convert with an FPU convert instruction. It's trivial to compute
2372 // the modulo operation on an integer register so we convert to a 64-bit
2373 // integer.
2374 //
2375 // Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF)
2376 // when the double is out of range. NaNs and infinities will be converted to 0
2377 // (as ECMA-262 requires).
2378 Fcvtzs(result.X(), double_input);
2379
2380 // The values INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF) are not
2381 // representable using a double, so if the result is one of those then we know
2382 // that saturation occurred, and we need to manually handle the conversion.
2383 //
2384 // It is easy to detect INT64_MIN and INT64_MAX because adding or subtracting
2385 // 1 will cause signed overflow.
2386 Cmp(result.X(), 1);
2387 Ccmp(result.X(), -1, VFlag, vc);
2388
2389 B(vc, done);
2390 }
2391
TruncateDoubleToI(Isolate * isolate,Zone * zone,Register result,DoubleRegister double_input,StubCallMode stub_mode,LinkRegisterStatus lr_status)2392 void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
2393 Register result,
2394 DoubleRegister double_input,
2395 StubCallMode stub_mode,
2396 LinkRegisterStatus lr_status) {
2397 if (CpuFeatures::IsSupported(JSCVT)) {
2398 Fjcvtzs(result.W(), double_input);
2399 return;
2400 }
2401
2402 Label done;
2403
2404 // Try to convert the double to an int64. If successful, the bottom 32 bits
2405 // contain our truncated int32 result.
2406 TryConvertDoubleToInt64(result, double_input, &done);
2407
2408 // If we fell through then inline version didn't succeed - call stub instead.
2409 if (lr_status == kLRHasNotBeenSaved) {
2410 Push<TurboAssembler::kSignLR>(lr, double_input);
2411 } else {
2412 Push<TurboAssembler::kDontStoreLR>(xzr, double_input);
2413 }
2414
2415 // DoubleToI preserves any registers it needs to clobber.
2416 if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
2417 Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
2418 } else if (options().inline_offheap_trampolines) {
2419 CallBuiltin(Builtins::kDoubleToI);
2420 } else {
2421 Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
2422 }
2423 Ldr(result, MemOperand(sp, 0));
2424
2425 DCHECK_EQ(xzr.SizeInBytes(), double_input.SizeInBytes());
2426
2427 if (lr_status == kLRHasNotBeenSaved) {
2428 // Pop into xzr here to drop the double input on the stack:
2429 Pop<TurboAssembler::kAuthLR>(xzr, lr);
2430 } else {
2431 Drop(2);
2432 }
2433
2434 Bind(&done);
2435 // Keep our invariant that the upper 32 bits are zero.
2436 Uxtw(result.W(), result.W());
2437 }
2438
Prologue()2439 void TurboAssembler::Prologue() {
2440 Push<TurboAssembler::kSignLR>(lr, fp);
2441 mov(fp, sp);
2442 STATIC_ASSERT(kExtraSlotClaimedByPrologue == 1);
2443 Push(cp, kJSFunctionRegister, kJavaScriptCallArgCountRegister, padreg);
2444 }
2445
EnterFrame(StackFrame::Type type)2446 void TurboAssembler::EnterFrame(StackFrame::Type type) {
2447 UseScratchRegisterScope temps(this);
2448
2449 if (type == StackFrame::INTERNAL || type == StackFrame::WASM_DEBUG_BREAK) {
2450 Register type_reg = temps.AcquireX();
2451 Mov(type_reg, StackFrame::TypeToMarker(type));
2452 Push<TurboAssembler::kSignLR>(lr, fp, type_reg, padreg);
2453 const int kFrameSize =
2454 TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize;
2455 Add(fp, sp, kFrameSize);
2456 // sp[3] : lr
2457 // sp[2] : fp
2458 // sp[1] : type
2459 // sp[0] : for alignment
2460 } else if (type == StackFrame::WASM ||
2461 type == StackFrame::WASM_COMPILE_LAZY ||
2462 type == StackFrame::WASM_EXIT) {
2463 Register type_reg = temps.AcquireX();
2464 Mov(type_reg, StackFrame::TypeToMarker(type));
2465 Push<TurboAssembler::kSignLR>(lr, fp);
2466 Mov(fp, sp);
2467 Push(type_reg, padreg);
2468 // sp[3] : lr
2469 // sp[2] : fp
2470 // sp[1] : type
2471 // sp[0] : for alignment
2472 } else {
2473 DCHECK_EQ(type, StackFrame::CONSTRUCT);
2474 Register type_reg = temps.AcquireX();
2475 Mov(type_reg, StackFrame::TypeToMarker(type));
2476
2477 // Users of this frame type push a context pointer after the type field,
2478 // so do it here to keep the stack pointer aligned.
2479 Push<TurboAssembler::kSignLR>(lr, fp, type_reg, cp);
2480
2481 // The context pointer isn't part of the fixed frame, so add an extra slot
2482 // to account for it.
2483 Add(fp, sp,
2484 TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize);
2485 // sp[3] : lr
2486 // sp[2] : fp
2487 // sp[1] : type
2488 // sp[0] : cp
2489 }
2490 }
2491
LeaveFrame(StackFrame::Type type)2492 void TurboAssembler::LeaveFrame(StackFrame::Type type) {
2493 // Drop the execution stack down to the frame pointer and restore
2494 // the caller frame pointer and return address.
2495 Mov(sp, fp);
2496 Pop<TurboAssembler::kAuthLR>(fp, lr);
2497 }
2498
ExitFramePreserveFPRegs()2499 void MacroAssembler::ExitFramePreserveFPRegs() {
2500 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
2501 PushCPURegList(kCallerSavedV);
2502 }
2503
ExitFrameRestoreFPRegs()2504 void MacroAssembler::ExitFrameRestoreFPRegs() {
2505 // Read the registers from the stack without popping them. The stack pointer
2506 // will be reset as part of the unwinding process.
2507 CPURegList saved_fp_regs = kCallerSavedV;
2508 DCHECK_EQ(saved_fp_regs.Count() % 2, 0);
2509
2510 int offset = ExitFrameConstants::kLastExitFrameField;
2511 while (!saved_fp_regs.IsEmpty()) {
2512 const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
2513 const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
2514 offset -= 2 * kDRegSize;
2515 Ldp(dst1, dst0, MemOperand(fp, offset));
2516 }
2517 }
2518
EnterExitFrame(bool save_doubles,const Register & scratch,int extra_space,StackFrame::Type frame_type)2519 void MacroAssembler::EnterExitFrame(bool save_doubles, const Register& scratch,
2520 int extra_space,
2521 StackFrame::Type frame_type) {
2522 DCHECK(frame_type == StackFrame::EXIT ||
2523 frame_type == StackFrame::BUILTIN_EXIT);
2524
2525 // Set up the new stack frame.
2526 Push<TurboAssembler::kSignLR>(lr, fp);
2527 Mov(fp, sp);
2528 Mov(scratch, StackFrame::TypeToMarker(frame_type));
2529 Push(scratch, xzr);
2530 // fp[8]: CallerPC (lr)
2531 // fp -> fp[0]: CallerFP (old fp)
2532 // fp[-8]: STUB marker
2533 // sp -> fp[-16]: Space reserved for SPOffset.
2534 STATIC_ASSERT((2 * kSystemPointerSize) ==
2535 ExitFrameConstants::kCallerSPOffset);
2536 STATIC_ASSERT((1 * kSystemPointerSize) ==
2537 ExitFrameConstants::kCallerPCOffset);
2538 STATIC_ASSERT((0 * kSystemPointerSize) ==
2539 ExitFrameConstants::kCallerFPOffset);
2540 STATIC_ASSERT((-2 * kSystemPointerSize) == ExitFrameConstants::kSPOffset);
2541
2542 // Save the frame pointer and context pointer in the top frame.
2543 Mov(scratch,
2544 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2545 Str(fp, MemOperand(scratch));
2546 Mov(scratch,
2547 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2548 Str(cp, MemOperand(scratch));
2549
2550 STATIC_ASSERT((-2 * kSystemPointerSize) ==
2551 ExitFrameConstants::kLastExitFrameField);
2552 if (save_doubles) {
2553 ExitFramePreserveFPRegs();
2554 }
2555
2556 // Round the number of space we need to claim to a multiple of two.
2557 int slots_to_claim = RoundUp(extra_space + 1, 2);
2558
2559 // Reserve space for the return address and for user requested memory.
2560 // We do this before aligning to make sure that we end up correctly
2561 // aligned with the minimum of wasted space.
2562 Claim(slots_to_claim, kXRegSize);
2563 // fp[8]: CallerPC (lr)
2564 // fp -> fp[0]: CallerFP (old fp)
2565 // fp[-8]: STUB marker
2566 // fp[-16]: Space reserved for SPOffset.
2567 // fp[-16 - fp_size]: Saved doubles (if save_doubles is true).
2568 // sp[8]: Extra space reserved for caller (if extra_space != 0).
2569 // sp -> sp[0]: Space reserved for the return address.
2570
2571 // ExitFrame::GetStateForFramePointer expects to find the return address at
2572 // the memory address immediately below the pointer stored in SPOffset.
2573 // It is not safe to derive much else from SPOffset, because the size of the
2574 // padding can vary.
2575 Add(scratch, sp, kXRegSize);
2576 Str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
2577 }
2578
2579 // Leave the current exit frame.
LeaveExitFrame(bool restore_doubles,const Register & scratch,const Register & scratch2)2580 void MacroAssembler::LeaveExitFrame(bool restore_doubles,
2581 const Register& scratch,
2582 const Register& scratch2) {
2583 if (restore_doubles) {
2584 ExitFrameRestoreFPRegs();
2585 }
2586
2587 // Restore the context pointer from the top frame.
2588 Mov(scratch,
2589 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2590 Ldr(cp, MemOperand(scratch));
2591
2592 if (emit_debug_code()) {
2593 // Also emit debug code to clear the cp in the top frame.
2594 Mov(scratch2, Operand(Context::kInvalidContext));
2595 Mov(scratch, ExternalReference::Create(IsolateAddressId::kContextAddress,
2596 isolate()));
2597 Str(scratch2, MemOperand(scratch));
2598 }
2599 // Clear the frame pointer from the top frame.
2600 Mov(scratch,
2601 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2602 Str(xzr, MemOperand(scratch));
2603
2604 // Pop the exit frame.
2605 // fp[8]: CallerPC (lr)
2606 // fp -> fp[0]: CallerFP (old fp)
2607 // fp[...]: The rest of the frame.
2608 Mov(sp, fp);
2609 Pop<TurboAssembler::kAuthLR>(fp, lr);
2610 }
2611
LoadGlobalProxy(Register dst)2612 void MacroAssembler::LoadGlobalProxy(Register dst) {
2613 LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
2614 }
2615
LoadWeakValue(Register out,Register in,Label * target_if_cleared)2616 void MacroAssembler::LoadWeakValue(Register out, Register in,
2617 Label* target_if_cleared) {
2618 CompareAndBranch(in.W(), Operand(kClearedWeakHeapObjectLower32), eq,
2619 target_if_cleared);
2620
2621 and_(out, in, Operand(~kWeakHeapObjectMask));
2622 }
2623
IncrementCounter(StatsCounter * counter,int value,Register scratch1,Register scratch2)2624 void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
2625 Register scratch1, Register scratch2) {
2626 DCHECK_NE(value, 0);
2627 if (FLAG_native_code_counters && counter->Enabled()) {
2628 // This operation has to be exactly 32-bit wide in case the external
2629 // reference table redirects the counter to a uint32_t dummy_stats_counter_
2630 // field.
2631 Mov(scratch2, ExternalReference::Create(counter));
2632 Ldr(scratch1.W(), MemOperand(scratch2));
2633 Add(scratch1.W(), scratch1.W(), value);
2634 Str(scratch1.W(), MemOperand(scratch2));
2635 }
2636 }
2637
DecrementCounter(StatsCounter * counter,int value,Register scratch1,Register scratch2)2638 void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
2639 Register scratch1, Register scratch2) {
2640 IncrementCounter(counter, -value, scratch1, scratch2);
2641 }
2642
MaybeDropFrames()2643 void MacroAssembler::MaybeDropFrames() {
2644 // Check whether we need to drop frames to restart a function on the stack.
2645 Mov(x1, ExternalReference::debug_restart_fp_address(isolate()));
2646 Ldr(x1, MemOperand(x1));
2647 Tst(x1, x1);
2648 Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
2649 ne);
2650 }
2651
JumpIfObjectType(Register object,Register map,Register type_reg,InstanceType type,Label * if_cond_pass,Condition cond)2652 void MacroAssembler::JumpIfObjectType(Register object, Register map,
2653 Register type_reg, InstanceType type,
2654 Label* if_cond_pass, Condition cond) {
2655 CompareObjectType(object, map, type_reg, type);
2656 B(cond, if_cond_pass);
2657 }
2658
2659 // Sets condition flags based on comparison, and returns type in type_reg.
CompareObjectType(Register object,Register map,Register type_reg,InstanceType type)2660 void MacroAssembler::CompareObjectType(Register object, Register map,
2661 Register type_reg, InstanceType type) {
2662 LoadMap(map, object);
2663 CompareInstanceType(map, type_reg, type);
2664 }
2665
LoadMap(Register dst,Register object)2666 void MacroAssembler::LoadMap(Register dst, Register object) {
2667 LoadTaggedPointerField(dst, FieldMemOperand(object, HeapObject::kMapOffset));
2668 }
2669
2670 // Sets condition flags based on comparison, and returns type in type_reg.
CompareInstanceType(Register map,Register type_reg,InstanceType type)2671 void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
2672 InstanceType type) {
2673 Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
2674 Cmp(type_reg, type);
2675 }
2676
LoadElementsKindFromMap(Register result,Register map)2677 void MacroAssembler::LoadElementsKindFromMap(Register result, Register map) {
2678 // Load the map's "bit field 2".
2679 Ldrb(result, FieldMemOperand(map, Map::kBitField2Offset));
2680 // Retrieve elements_kind from bit field 2.
2681 DecodeField<Map::Bits2::ElementsKindBits>(result);
2682 }
2683
CompareRoot(const Register & obj,RootIndex index)2684 void MacroAssembler::CompareRoot(const Register& obj, RootIndex index) {
2685 UseScratchRegisterScope temps(this);
2686 Register temp = temps.AcquireX();
2687 DCHECK(!AreAliased(obj, temp));
2688 LoadRoot(temp, index);
2689 CmpTagged(obj, temp);
2690 }
2691
JumpIfRoot(const Register & obj,RootIndex index,Label * if_equal)2692 void MacroAssembler::JumpIfRoot(const Register& obj, RootIndex index,
2693 Label* if_equal) {
2694 CompareRoot(obj, index);
2695 B(eq, if_equal);
2696 }
2697
JumpIfNotRoot(const Register & obj,RootIndex index,Label * if_not_equal)2698 void MacroAssembler::JumpIfNotRoot(const Register& obj, RootIndex index,
2699 Label* if_not_equal) {
2700 CompareRoot(obj, index);
2701 B(ne, if_not_equal);
2702 }
2703
JumpIfIsInRange(const Register & value,unsigned lower_limit,unsigned higher_limit,Label * on_in_range)2704 void MacroAssembler::JumpIfIsInRange(const Register& value,
2705 unsigned lower_limit,
2706 unsigned higher_limit,
2707 Label* on_in_range) {
2708 if (lower_limit != 0) {
2709 UseScratchRegisterScope temps(this);
2710 Register scratch = temps.AcquireW();
2711 Sub(scratch, value, Operand(lower_limit));
2712 CompareAndBranch(scratch, Operand(higher_limit - lower_limit), ls,
2713 on_in_range);
2714 } else {
2715 CompareAndBranch(value, Operand(higher_limit - lower_limit), ls,
2716 on_in_range);
2717 }
2718 }
2719
LoadTaggedPointerField(const Register & destination,const MemOperand & field_operand)2720 void TurboAssembler::LoadTaggedPointerField(const Register& destination,
2721 const MemOperand& field_operand) {
2722 if (COMPRESS_POINTERS_BOOL) {
2723 DecompressTaggedPointer(destination, field_operand);
2724 } else {
2725 Ldr(destination, field_operand);
2726 }
2727 }
2728
LoadAnyTaggedField(const Register & destination,const MemOperand & field_operand)2729 void TurboAssembler::LoadAnyTaggedField(const Register& destination,
2730 const MemOperand& field_operand) {
2731 if (COMPRESS_POINTERS_BOOL) {
2732 DecompressAnyTagged(destination, field_operand);
2733 } else {
2734 Ldr(destination, field_operand);
2735 }
2736 }
2737
SmiUntagField(Register dst,const MemOperand & src)2738 void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src) {
2739 SmiUntag(dst, src);
2740 }
2741
StoreTaggedField(const Register & value,const MemOperand & dst_field_operand)2742 void TurboAssembler::StoreTaggedField(const Register& value,
2743 const MemOperand& dst_field_operand) {
2744 if (COMPRESS_POINTERS_BOOL) {
2745 Str(value.W(), dst_field_operand);
2746 } else {
2747 Str(value, dst_field_operand);
2748 }
2749 }
2750
DecompressTaggedSigned(const Register & destination,const MemOperand & field_operand)2751 void TurboAssembler::DecompressTaggedSigned(const Register& destination,
2752 const MemOperand& field_operand) {
2753 RecordComment("[ DecompressTaggedSigned");
2754 Ldr(destination.W(), field_operand);
2755 if (FLAG_debug_code) {
2756 // Corrupt the top 32 bits. Made up of 16 fixed bits and 16 pc offset bits.
2757 Add(destination, destination,
2758 ((kDebugZapValue << 16) | (pc_offset() & 0xffff)) << 32);
2759 }
2760 RecordComment("]");
2761 }
2762
DecompressTaggedPointer(const Register & destination,const MemOperand & field_operand)2763 void TurboAssembler::DecompressTaggedPointer(const Register& destination,
2764 const MemOperand& field_operand) {
2765 RecordComment("[ DecompressTaggedPointer");
2766 Ldr(destination.W(), field_operand);
2767 Add(destination, kRootRegister, destination);
2768 RecordComment("]");
2769 }
2770
DecompressTaggedPointer(const Register & destination,const Register & source)2771 void TurboAssembler::DecompressTaggedPointer(const Register& destination,
2772 const Register& source) {
2773 RecordComment("[ DecompressTaggedPointer");
2774 Add(destination, kRootRegister, Operand(source, UXTW));
2775 RecordComment("]");
2776 }
2777
DecompressAnyTagged(const Register & destination,const MemOperand & field_operand)2778 void TurboAssembler::DecompressAnyTagged(const Register& destination,
2779 const MemOperand& field_operand) {
2780 RecordComment("[ DecompressAnyTagged");
2781 Ldr(destination.W(), field_operand);
2782 Add(destination, kRootRegister, destination);
2783 RecordComment("]");
2784 }
2785
CheckPageFlag(const Register & object,int mask,Condition cc,Label * condition_met)2786 void TurboAssembler::CheckPageFlag(const Register& object, int mask,
2787 Condition cc, Label* condition_met) {
2788 UseScratchRegisterScope temps(this);
2789 Register scratch = temps.AcquireX();
2790 And(scratch, object, ~kPageAlignmentMask);
2791 Ldr(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
2792 if (cc == eq) {
2793 TestAndBranchIfAnySet(scratch, mask, condition_met);
2794 } else {
2795 DCHECK_EQ(cc, ne);
2796 TestAndBranchIfAllClear(scratch, mask, condition_met);
2797 }
2798 }
2799
RecordWriteField(Register object,int offset,Register value,LinkRegisterStatus lr_status,SaveFPRegsMode save_fp,RememberedSetAction remembered_set_action,SmiCheck smi_check)2800 void MacroAssembler::RecordWriteField(Register object, int offset,
2801 Register value,
2802 LinkRegisterStatus lr_status,
2803 SaveFPRegsMode save_fp,
2804 RememberedSetAction remembered_set_action,
2805 SmiCheck smi_check) {
2806 // First, check if a write barrier is even needed. The tests below
2807 // catch stores of Smis.
2808 Label done;
2809
2810 // Skip the barrier if writing a smi.
2811 if (smi_check == INLINE_SMI_CHECK) {
2812 JumpIfSmi(value, &done);
2813 }
2814
2815 // Although the object register is tagged, the offset is relative to the start
2816 // of the object, so offset must be a multiple of kTaggedSize.
2817 DCHECK(IsAligned(offset, kTaggedSize));
2818
2819 if (emit_debug_code()) {
2820 Label ok;
2821 UseScratchRegisterScope temps(this);
2822 Register scratch = temps.AcquireX();
2823 Add(scratch, object, offset - kHeapObjectTag);
2824 Tst(scratch, kTaggedSize - 1);
2825 B(eq, &ok);
2826 Abort(AbortReason::kUnalignedCellInWriteBarrier);
2827 Bind(&ok);
2828 }
2829
2830 RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,
2831 save_fp, remembered_set_action, OMIT_SMI_CHECK);
2832
2833 Bind(&done);
2834 }
2835
SaveRegisters(RegList registers)2836 void TurboAssembler::SaveRegisters(RegList registers) {
2837 DCHECK_GT(NumRegs(registers), 0);
2838 CPURegList regs(CPURegister::kRegister, kXRegSizeInBits, registers);
2839 // If we were saving LR, we might need to sign it.
2840 DCHECK(!regs.IncludesAliasOf(lr));
2841 regs.Align();
2842 PushCPURegList(regs);
2843 }
2844
RestoreRegisters(RegList registers)2845 void TurboAssembler::RestoreRegisters(RegList registers) {
2846 DCHECK_GT(NumRegs(registers), 0);
2847 CPURegList regs(CPURegister::kRegister, kXRegSizeInBits, registers);
2848 // If we were saving LR, we might need to sign it.
2849 DCHECK(!regs.IncludesAliasOf(lr));
2850 regs.Align();
2851 PopCPURegList(regs);
2852 }
2853
CallEphemeronKeyBarrier(Register object,Operand offset,SaveFPRegsMode fp_mode)2854 void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,
2855 SaveFPRegsMode fp_mode) {
2856 EphemeronKeyBarrierDescriptor descriptor;
2857 RegList registers = descriptor.allocatable_registers();
2858
2859 SaveRegisters(registers);
2860
2861 Register object_parameter(
2862 descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
2863 Register slot_parameter(descriptor.GetRegisterParameter(
2864 EphemeronKeyBarrierDescriptor::kSlotAddress));
2865 Register fp_mode_parameter(
2866 descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
2867
2868 MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
2869
2870 Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
2871 Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
2872 RelocInfo::CODE_TARGET);
2873 RestoreRegisters(registers);
2874 }
2875
CallRecordWriteStub(Register object,Operand offset,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode)2876 void TurboAssembler::CallRecordWriteStub(
2877 Register object, Operand offset, RememberedSetAction remembered_set_action,
2878 SaveFPRegsMode fp_mode) {
2879 CallRecordWriteStub(
2880 object, offset, remembered_set_action, fp_mode,
2881 isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
2882 kNullAddress);
2883 }
2884
CallRecordWriteStub(Register object,Operand offset,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode,Address wasm_target)2885 void TurboAssembler::CallRecordWriteStub(
2886 Register object, Operand offset, RememberedSetAction remembered_set_action,
2887 SaveFPRegsMode fp_mode, Address wasm_target) {
2888 CallRecordWriteStub(object, offset, remembered_set_action, fp_mode,
2889 Handle<Code>::null(), wasm_target);
2890 }
2891
CallRecordWriteStub(Register object,Operand offset,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode,Handle<Code> code_target,Address wasm_target)2892 void TurboAssembler::CallRecordWriteStub(
2893 Register object, Operand offset, RememberedSetAction remembered_set_action,
2894 SaveFPRegsMode fp_mode, Handle<Code> code_target, Address wasm_target) {
2895 DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
2896 // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
2897 // i.e. always emit remember set and save FP registers in RecordWriteStub. If
2898 // large performance regression is observed, we should use these values to
2899 // avoid unnecessary work.
2900
2901 RecordWriteDescriptor descriptor;
2902 RegList registers = descriptor.allocatable_registers();
2903
2904 SaveRegisters(registers);
2905
2906 Register object_parameter(
2907 descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
2908 Register slot_parameter(
2909 descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
2910 Register remembered_set_parameter(
2911 descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
2912 Register fp_mode_parameter(
2913 descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
2914
2915 MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
2916
2917 Mov(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
2918 Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
2919 if (code_target.is_null()) {
2920 Call(wasm_target, RelocInfo::WASM_STUB_CALL);
2921 } else {
2922 Call(code_target, RelocInfo::CODE_TARGET);
2923 }
2924
2925 RestoreRegisters(registers);
2926 }
2927
MoveObjectAndSlot(Register dst_object,Register dst_slot,Register object,Operand offset)2928 void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,
2929 Register object, Operand offset) {
2930 DCHECK_NE(dst_object, dst_slot);
2931 // If `offset` is a register, it cannot overlap with `object`.
2932 DCHECK_IMPLIES(!offset.IsImmediate(), offset.reg() != object);
2933
2934 // If the slot register does not overlap with the object register, we can
2935 // overwrite it.
2936 if (dst_slot != object) {
2937 Add(dst_slot, object, offset);
2938 Mov(dst_object, object);
2939 return;
2940 }
2941
2942 DCHECK_EQ(dst_slot, object);
2943
2944 // If the destination object register does not overlap with the offset
2945 // register, we can overwrite it.
2946 if (offset.IsImmediate() || (offset.reg() != dst_object)) {
2947 Mov(dst_object, dst_slot);
2948 Add(dst_slot, dst_slot, offset);
2949 return;
2950 }
2951
2952 DCHECK_EQ(dst_object, offset.reg());
2953
2954 // We only have `dst_slot` and `dst_object` left as distinct registers so we
2955 // have to swap them. We write this as a add+sub sequence to avoid using a
2956 // scratch register.
2957 Add(dst_slot, dst_slot, dst_object);
2958 Sub(dst_object, dst_slot, dst_object);
2959 }
2960
2961 // If lr_status is kLRHasBeenSaved, lr will be clobbered.
2962 //
2963 // The register 'object' contains a heap object pointer. The heap object tag is
2964 // shifted away.
RecordWrite(Register object,Operand offset,Register value,LinkRegisterStatus lr_status,SaveFPRegsMode fp_mode,RememberedSetAction remembered_set_action,SmiCheck smi_check)2965 void MacroAssembler::RecordWrite(Register object, Operand offset,
2966 Register value, LinkRegisterStatus lr_status,
2967 SaveFPRegsMode fp_mode,
2968 RememberedSetAction remembered_set_action,
2969 SmiCheck smi_check) {
2970 ASM_LOCATION_IN_ASSEMBLER("MacroAssembler::RecordWrite");
2971 DCHECK(!AreAliased(object, value));
2972
2973 if (emit_debug_code()) {
2974 UseScratchRegisterScope temps(this);
2975 Register temp = temps.AcquireX();
2976
2977 Add(temp, object, offset);
2978 LoadTaggedPointerField(temp, MemOperand(temp));
2979 Cmp(temp, value);
2980 Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
2981 }
2982
2983 if ((remembered_set_action == OMIT_REMEMBERED_SET &&
2984 !FLAG_incremental_marking) ||
2985 FLAG_disable_write_barriers) {
2986 return;
2987 }
2988
2989 // First, check if a write barrier is even needed. The tests below
2990 // catch stores of smis and stores into the young generation.
2991 Label done;
2992
2993 if (smi_check == INLINE_SMI_CHECK) {
2994 DCHECK_EQ(0, kSmiTag);
2995 JumpIfSmi(value, &done);
2996 }
2997 CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, ne,
2998 &done);
2999
3000 CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, ne,
3001 &done);
3002
3003 // Record the actual write.
3004 if (lr_status == kLRHasNotBeenSaved) {
3005 Push<TurboAssembler::kSignLR>(padreg, lr);
3006 }
3007 CallRecordWriteStub(object, offset, remembered_set_action, fp_mode);
3008 if (lr_status == kLRHasNotBeenSaved) {
3009 Pop<TurboAssembler::kAuthLR>(lr, padreg);
3010 }
3011
3012 Bind(&done);
3013 }
3014
Assert(Condition cond,AbortReason reason)3015 void TurboAssembler::Assert(Condition cond, AbortReason reason) {
3016 if (emit_debug_code()) {
3017 Check(cond, reason);
3018 }
3019 }
3020
AssertUnreachable(AbortReason reason)3021 void TurboAssembler::AssertUnreachable(AbortReason reason) {
3022 if (emit_debug_code()) Abort(reason);
3023 }
3024
Check(Condition cond,AbortReason reason)3025 void TurboAssembler::Check(Condition cond, AbortReason reason) {
3026 Label ok;
3027 B(cond, &ok);
3028 Abort(reason);
3029 // Will not return here.
3030 Bind(&ok);
3031 }
3032
Trap()3033 void TurboAssembler::Trap() { Brk(0); }
DebugBreak()3034 void TurboAssembler::DebugBreak() { Debug("DebugBreak", 0, BREAK); }
3035
Abort(AbortReason reason)3036 void TurboAssembler::Abort(AbortReason reason) {
3037 #ifdef DEBUG
3038 RecordComment("Abort message: ");
3039 RecordComment(GetAbortReason(reason));
3040 #endif
3041
3042 // Avoid emitting call to builtin if requested.
3043 if (trap_on_abort()) {
3044 Brk(0);
3045 return;
3046 }
3047
3048 // We need some scratch registers for the MacroAssembler, so make sure we have
3049 // some. This is safe here because Abort never returns.
3050 RegList old_tmp_list = TmpList()->list();
3051 TmpList()->Combine(MacroAssembler::DefaultTmpList());
3052
3053 if (should_abort_hard()) {
3054 // We don't care if we constructed a frame. Just pretend we did.
3055 FrameScope assume_frame(this, StackFrame::NONE);
3056 Mov(w0, static_cast<int>(reason));
3057 Call(ExternalReference::abort_with_reason());
3058 return;
3059 }
3060
3061 // Avoid infinite recursion; Push contains some assertions that use Abort.
3062 HardAbortScope hard_aborts(this);
3063
3064 Mov(x1, Smi::FromInt(static_cast<int>(reason)));
3065
3066 if (!has_frame_) {
3067 // We don't actually want to generate a pile of code for this, so just
3068 // claim there is a stack frame, without generating one.
3069 FrameScope scope(this, StackFrame::NONE);
3070 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3071 } else {
3072 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3073 }
3074
3075 TmpList()->set_list(old_tmp_list);
3076 }
3077
LoadNativeContextSlot(int index,Register dst)3078 void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
3079 LoadMap(dst, cp);
3080 LoadTaggedPointerField(
3081 dst, FieldMemOperand(
3082 dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
3083 LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
3084 }
3085
3086 // This is the main Printf implementation. All other Printf variants call
3087 // PrintfNoPreserve after setting up one or more PreserveRegisterScopes.
PrintfNoPreserve(const char * format,const CPURegister & arg0,const CPURegister & arg1,const CPURegister & arg2,const CPURegister & arg3)3088 void TurboAssembler::PrintfNoPreserve(const char* format,
3089 const CPURegister& arg0,
3090 const CPURegister& arg1,
3091 const CPURegister& arg2,
3092 const CPURegister& arg3) {
3093 // We cannot handle a caller-saved stack pointer. It doesn't make much sense
3094 // in most cases anyway, so this restriction shouldn't be too serious.
3095 DCHECK(!kCallerSaved.IncludesAliasOf(sp));
3096
3097 // The provided arguments, and their proper procedure-call standard registers.
3098 CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
3099 CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};
3100
3101 int arg_count = kPrintfMaxArgCount;
3102
3103 // The PCS varargs registers for printf. Note that x0 is used for the printf
3104 // format string.
3105 static const CPURegList kPCSVarargs =
3106 CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
3107 static const CPURegList kPCSVarargsFP =
3108 CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
3109
3110 // We can use caller-saved registers as scratch values, except for the
3111 // arguments and the PCS registers where they might need to go.
3112 CPURegList tmp_list = kCallerSaved;
3113 tmp_list.Remove(x0); // Used to pass the format string.
3114 tmp_list.Remove(kPCSVarargs);
3115 tmp_list.Remove(arg0, arg1, arg2, arg3);
3116
3117 CPURegList fp_tmp_list = kCallerSavedV;
3118 fp_tmp_list.Remove(kPCSVarargsFP);
3119 fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
3120
3121 // Override the TurboAssembler's scratch register list. The lists will be
3122 // reset automatically at the end of the UseScratchRegisterScope.
3123 UseScratchRegisterScope temps(this);
3124 TmpList()->set_list(tmp_list.list());
3125 FPTmpList()->set_list(fp_tmp_list.list());
3126
3127 // Copies of the printf vararg registers that we can pop from.
3128 CPURegList pcs_varargs = kPCSVarargs;
3129 #ifndef V8_OS_WIN
3130 CPURegList pcs_varargs_fp = kPCSVarargsFP;
3131 #endif
3132
3133 // Place the arguments. There are lots of clever tricks and optimizations we
3134 // could use here, but Printf is a debug tool so instead we just try to keep
3135 // it simple: Move each input that isn't already in the right place to a
3136 // scratch register, then move everything back.
3137 for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
3138 // Work out the proper PCS register for this argument.
3139 if (args[i].IsRegister()) {
3140 pcs[i] = pcs_varargs.PopLowestIndex().X();
3141 // We might only need a W register here. We need to know the size of the
3142 // argument so we can properly encode it for the simulator call.
3143 if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
3144 } else if (args[i].IsVRegister()) {
3145 // In C, floats are always cast to doubles for varargs calls.
3146 #ifdef V8_OS_WIN
3147 // In case of variadic functions SIMD and Floating-point registers
3148 // aren't used. The general x0-x7 should be used instead.
3149 // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions
3150 pcs[i] = pcs_varargs.PopLowestIndex().X();
3151 #else
3152 pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
3153 #endif
3154 } else {
3155 DCHECK(args[i].IsNone());
3156 arg_count = i;
3157 break;
3158 }
3159
3160 // If the argument is already in the right place, leave it where it is.
3161 if (args[i].Aliases(pcs[i])) continue;
3162
3163 // Otherwise, if the argument is in a PCS argument register, allocate an
3164 // appropriate scratch register and then move it out of the way.
3165 if (kPCSVarargs.IncludesAliasOf(args[i]) ||
3166 kPCSVarargsFP.IncludesAliasOf(args[i])) {
3167 if (args[i].IsRegister()) {
3168 Register old_arg = args[i].Reg();
3169 Register new_arg = temps.AcquireSameSizeAs(old_arg);
3170 Mov(new_arg, old_arg);
3171 args[i] = new_arg;
3172 } else {
3173 VRegister old_arg = args[i].VReg();
3174 VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
3175 Fmov(new_arg, old_arg);
3176 args[i] = new_arg;
3177 }
3178 }
3179 }
3180
3181 // Do a second pass to move values into their final positions and perform any
3182 // conversions that may be required.
3183 for (int i = 0; i < arg_count; i++) {
3184 #ifdef V8_OS_WIN
3185 if (args[i].IsVRegister()) {
3186 if (pcs[i].SizeInBytes() != args[i].SizeInBytes()) {
3187 // If the argument is half- or single-precision
3188 // converts to double-precision before that is
3189 // moved into the one of X scratch register.
3190 VRegister temp0 = temps.AcquireD();
3191 Fcvt(temp0.VReg(), args[i].VReg());
3192 Fmov(pcs[i].Reg(), temp0);
3193 } else {
3194 Fmov(pcs[i].Reg(), args[i].VReg());
3195 }
3196 } else {
3197 Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
3198 }
3199 #else
3200 DCHECK(pcs[i].type() == args[i].type());
3201 if (pcs[i].IsRegister()) {
3202 Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
3203 } else {
3204 DCHECK(pcs[i].IsVRegister());
3205 if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
3206 Fmov(pcs[i].VReg(), args[i].VReg());
3207 } else {
3208 Fcvt(pcs[i].VReg(), args[i].VReg());
3209 }
3210 }
3211 #endif
3212 }
3213
3214 // Load the format string into x0, as per the procedure-call standard.
3215 //
3216 // To make the code as portable as possible, the format string is encoded
3217 // directly in the instruction stream. It might be cleaner to encode it in a
3218 // literal pool, but since Printf is usually used for debugging, it is
3219 // beneficial for it to be minimally dependent on other features.
3220 Label format_address;
3221 Adr(x0, &format_address);
3222
3223 // Emit the format string directly in the instruction stream.
3224 {
3225 BlockPoolsScope scope(this);
3226 Label after_data;
3227 B(&after_data);
3228 Bind(&format_address);
3229 EmitStringData(format);
3230 Unreachable();
3231 Bind(&after_data);
3232 }
3233
3234 CallPrintf(arg_count, pcs);
3235 }
3236
CallPrintf(int arg_count,const CPURegister * args)3237 void TurboAssembler::CallPrintf(int arg_count, const CPURegister* args) {
3238 // A call to printf needs special handling for the simulator, since the system
3239 // printf function will use a different instruction set and the procedure-call
3240 // standard will not be compatible.
3241 if (options().enable_simulator_code) {
3242 InstructionAccurateScope scope(this, kPrintfLength / kInstrSize);
3243 hlt(kImmExceptionIsPrintf);
3244 dc32(arg_count); // kPrintfArgCountOffset
3245
3246 // Determine the argument pattern.
3247 uint32_t arg_pattern_list = 0;
3248 for (int i = 0; i < arg_count; i++) {
3249 uint32_t arg_pattern;
3250 if (args[i].IsRegister()) {
3251 arg_pattern = args[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
3252 } else {
3253 DCHECK(args[i].Is64Bits());
3254 arg_pattern = kPrintfArgD;
3255 }
3256 DCHECK(arg_pattern < (1 << kPrintfArgPatternBits));
3257 arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
3258 }
3259 dc32(arg_pattern_list); // kPrintfArgPatternListOffset
3260 return;
3261 }
3262
3263 Call(ExternalReference::printf_function());
3264 }
3265
Printf(const char * format,CPURegister arg0,CPURegister arg1,CPURegister arg2,CPURegister arg3)3266 void TurboAssembler::Printf(const char* format, CPURegister arg0,
3267 CPURegister arg1, CPURegister arg2,
3268 CPURegister arg3) {
3269 // Printf is expected to preserve all registers, so make sure that none are
3270 // available as scratch registers until we've preserved them.
3271 RegList old_tmp_list = TmpList()->list();
3272 RegList old_fp_tmp_list = FPTmpList()->list();
3273 TmpList()->set_list(0);
3274 FPTmpList()->set_list(0);
3275
3276 CPURegList saved_registers = kCallerSaved;
3277 saved_registers.Align();
3278
3279 // Preserve all caller-saved registers as well as NZCV.
3280 // PushCPURegList asserts that the size of each list is a multiple of 16
3281 // bytes.
3282 PushCPURegList<kDontStoreLR>(saved_registers);
3283 PushCPURegList(kCallerSavedV);
3284
3285 // We can use caller-saved registers as scratch values (except for argN).
3286 CPURegList tmp_list = saved_registers;
3287 CPURegList fp_tmp_list = kCallerSavedV;
3288 tmp_list.Remove(arg0, arg1, arg2, arg3);
3289 fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
3290 TmpList()->set_list(tmp_list.list());
3291 FPTmpList()->set_list(fp_tmp_list.list());
3292
3293 {
3294 UseScratchRegisterScope temps(this);
3295 // If any of the arguments are the current stack pointer, allocate a new
3296 // register for them, and adjust the value to compensate for pushing the
3297 // caller-saved registers.
3298 bool arg0_sp = arg0.is_valid() && sp.Aliases(arg0);
3299 bool arg1_sp = arg1.is_valid() && sp.Aliases(arg1);
3300 bool arg2_sp = arg2.is_valid() && sp.Aliases(arg2);
3301 bool arg3_sp = arg3.is_valid() && sp.Aliases(arg3);
3302 if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
3303 // Allocate a register to hold the original stack pointer value, to pass
3304 // to PrintfNoPreserve as an argument.
3305 Register arg_sp = temps.AcquireX();
3306 Add(arg_sp, sp,
3307 saved_registers.TotalSizeInBytes() +
3308 kCallerSavedV.TotalSizeInBytes());
3309 if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
3310 if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
3311 if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
3312 if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
3313 }
3314
3315 // Preserve NZCV.
3316 {
3317 UseScratchRegisterScope temps(this);
3318 Register tmp = temps.AcquireX();
3319 Mrs(tmp, NZCV);
3320 Push(tmp, xzr);
3321 }
3322
3323 PrintfNoPreserve(format, arg0, arg1, arg2, arg3);
3324
3325 // Restore NZCV.
3326 {
3327 UseScratchRegisterScope temps(this);
3328 Register tmp = temps.AcquireX();
3329 Pop(xzr, tmp);
3330 Msr(NZCV, tmp);
3331 }
3332 }
3333
3334 PopCPURegList(kCallerSavedV);
3335 PopCPURegList<kDontLoadLR>(saved_registers);
3336
3337 TmpList()->set_list(old_tmp_list);
3338 FPTmpList()->set_list(old_fp_tmp_list);
3339 }
3340
~UseScratchRegisterScope()3341 UseScratchRegisterScope::~UseScratchRegisterScope() {
3342 available_->set_list(old_available_);
3343 availablefp_->set_list(old_availablefp_);
3344 }
3345
AcquireSameSizeAs(const Register & reg)3346 Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
3347 int code = AcquireNextAvailable(available_).code();
3348 return Register::Create(code, reg.SizeInBits());
3349 }
3350
AcquireSameSizeAs(const VRegister & reg)3351 VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
3352 int code = AcquireNextAvailable(availablefp_).code();
3353 return VRegister::Create(code, reg.SizeInBits());
3354 }
3355
AcquireNextAvailable(CPURegList * available)3356 CPURegister UseScratchRegisterScope::AcquireNextAvailable(
3357 CPURegList* available) {
3358 CHECK(!available->IsEmpty());
3359 CPURegister result = available->PopLowestIndex();
3360 DCHECK(!AreAliased(result, xzr, sp));
3361 return result;
3362 }
3363
ComputeCodeStartAddress(const Register & rd)3364 void TurboAssembler::ComputeCodeStartAddress(const Register& rd) {
3365 // We can use adr to load a pc relative location.
3366 adr(rd, -pc_offset());
3367 }
3368
ResetSpeculationPoisonRegister()3369 void TurboAssembler::ResetSpeculationPoisonRegister() {
3370 Mov(kSpeculationPoisonRegister, -1);
3371 }
3372
RestoreFPAndLR()3373 void TurboAssembler::RestoreFPAndLR() {
3374 static_assert(StandardFrameConstants::kCallerFPOffset + kSystemPointerSize ==
3375 StandardFrameConstants::kCallerPCOffset,
3376 "Offsets must be consecutive for ldp!");
3377 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
3378 // Make sure we can use x16 and x17.
3379 UseScratchRegisterScope temps(this);
3380 temps.Exclude(x16, x17);
3381 // We can load the return address directly into x17.
3382 Add(x16, fp, StandardFrameConstants::kCallerSPOffset);
3383 Ldp(fp, x17, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
3384 Autib1716();
3385 Mov(lr, x17);
3386 #else
3387 Ldp(fp, lr, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
3388 #endif
3389 }
3390
StoreReturnAddressInWasmExitFrame(Label * return_location)3391 void TurboAssembler::StoreReturnAddressInWasmExitFrame(Label* return_location) {
3392 UseScratchRegisterScope temps(this);
3393 temps.Exclude(x16, x17);
3394 Adr(x17, return_location);
3395 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
3396 Add(x16, fp, WasmExitFrameConstants::kCallingPCOffset + kSystemPointerSize);
3397 Pacib1716();
3398 #endif
3399 Str(x17, MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
3400 }
3401
3402 } // namespace internal
3403 } // namespace v8
3404
3405 #endif // V8_TARGET_ARCH_ARM64
3406