1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #if V8_TARGET_ARCH_ARM64
6
7 #include "src/assembler.h"
8 #include "src/base/bits.h"
9 #include "src/base/division-by-constant.h"
10 #include "src/bootstrapper.h"
11 #include "src/builtins/constants-table-builder.h"
12 #include "src/callable.h"
13 #include "src/code-factory.h"
14 #include "src/code-stubs.h"
15 #include "src/debug/debug.h"
16 #include "src/external-reference-table.h"
17 #include "src/frame-constants.h"
18 #include "src/frames-inl.h"
19 #include "src/heap/heap-inl.h"
20 #include "src/instruction-stream.h"
21 #include "src/register-configuration.h"
22 #include "src/runtime/runtime.h"
23 #include "src/snapshot/serializer-common.h"
24
25 #include "src/arm64/macro-assembler-arm64-inl.h"
26 #include "src/arm64/macro-assembler-arm64.h" // Cannot be the first include
27
28 namespace v8 {
29 namespace internal {
30
MacroAssembler(Isolate * isolate,byte * buffer,unsigned buffer_size,CodeObjectRequired create_code_object)31 MacroAssembler::MacroAssembler(Isolate* isolate, byte* buffer,
32 unsigned buffer_size,
33 CodeObjectRequired create_code_object)
34 : TurboAssembler(isolate, buffer, buffer_size, create_code_object) {
35 if (create_code_object == CodeObjectRequired::kYes) {
36 // Unlike TurboAssembler, which can be used off the main thread and may not
37 // allocate, macro assembler creates its own copy of the self-reference
38 // marker in order to disambiguate between self-references during nested
39 // code generation (e.g.: codegen of the current object triggers stub
40 // compilation through CodeStub::GetCode()).
41 code_object_ = Handle<HeapObject>::New(
42 *isolate->factory()->NewSelfReferenceMarker(), isolate);
43 }
44 }
45
DefaultTmpList()46 CPURegList TurboAssembler::DefaultTmpList() { return CPURegList(ip0, ip1); }
47
DefaultFPTmpList()48 CPURegList TurboAssembler::DefaultFPTmpList() {
49 return CPURegList(fp_scratch1, fp_scratch2);
50 }
51
TurboAssembler(Isolate * isolate,void * buffer,int buffer_size,CodeObjectRequired create_code_object)52 TurboAssembler::TurboAssembler(Isolate* isolate, void* buffer, int buffer_size,
53 CodeObjectRequired create_code_object)
54 : Assembler(isolate, buffer, buffer_size),
55 isolate_(isolate),
56 #if DEBUG
57 allow_macro_instructions_(true),
58 #endif
59 tmp_list_(DefaultTmpList()),
60 fptmp_list_(DefaultFPTmpList()),
61 use_real_aborts_(true) {
62 if (create_code_object == CodeObjectRequired::kYes) {
63 code_object_ = Handle<HeapObject>::New(
64 isolate->heap()->self_reference_marker(), isolate);
65 }
66 }
67
RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,Register exclusion) const68 int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
69 Register exclusion) const {
70 int bytes = 0;
71 auto list = kCallerSaved;
72 DCHECK_EQ(list.Count() % 2, 0);
73 // We only allow one exclusion register, so if the list is of even length
74 // before exclusions, it must still be afterwards, to maintain alignment.
75 // Therefore, we can ignore the exclusion register in the computation.
76 // However, we leave it in the argument list to mirror the prototype for
77 // Push/PopCallerSaved().
78 USE(exclusion);
79 bytes += list.Count() * kXRegSizeInBits / 8;
80
81 if (fp_mode == kSaveFPRegs) {
82 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
83 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
84 }
85 return bytes;
86 }
87
PushCallerSaved(SaveFPRegsMode fp_mode,Register exclusion)88 int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
89 Register exclusion) {
90 int bytes = 0;
91 auto list = kCallerSaved;
92 DCHECK_EQ(list.Count() % 2, 0);
93 if (!exclusion.Is(no_reg)) {
94 // Replace the excluded register with padding to maintain alignment.
95 list.Remove(exclusion);
96 list.Combine(padreg);
97 }
98 PushCPURegList(list);
99 bytes += list.Count() * kXRegSizeInBits / 8;
100
101 if (fp_mode == kSaveFPRegs) {
102 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
103 PushCPURegList(kCallerSavedV);
104 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
105 }
106 return bytes;
107 }
108
PopCallerSaved(SaveFPRegsMode fp_mode,Register exclusion)109 int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion) {
110 int bytes = 0;
111 if (fp_mode == kSaveFPRegs) {
112 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
113 PopCPURegList(kCallerSavedV);
114 bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
115 }
116
117 auto list = kCallerSaved;
118 DCHECK_EQ(list.Count() % 2, 0);
119 if (!exclusion.Is(no_reg)) {
120 // Replace the excluded register with padding to maintain alignment.
121 list.Remove(exclusion);
122 list.Combine(padreg);
123 }
124 PopCPURegList(list);
125 bytes += list.Count() * kXRegSizeInBits / 8;
126
127 return bytes;
128 }
129
LogicalMacro(const Register & rd,const Register & rn,const Operand & operand,LogicalOp op)130 void TurboAssembler::LogicalMacro(const Register& rd, const Register& rn,
131 const Operand& operand, LogicalOp op) {
132 UseScratchRegisterScope temps(this);
133
134 if (operand.NeedsRelocation(this)) {
135 Register temp = temps.AcquireX();
136 Ldr(temp, operand.immediate());
137 Logical(rd, rn, temp, op);
138
139 } else if (operand.IsImmediate()) {
140 int64_t immediate = operand.ImmediateValue();
141 unsigned reg_size = rd.SizeInBits();
142
143 // If the operation is NOT, invert the operation and immediate.
144 if ((op & NOT) == NOT) {
145 op = static_cast<LogicalOp>(op & ~NOT);
146 immediate = ~immediate;
147 }
148
149 // Ignore the top 32 bits of an immediate if we're moving to a W register.
150 if (rd.Is32Bits()) {
151 // Check that the top 32 bits are consistent.
152 DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
153 ((immediate >> kWRegSizeInBits) == -1));
154 immediate &= kWRegMask;
155 }
156
157 DCHECK(rd.Is64Bits() || is_uint32(immediate));
158
159 // Special cases for all set or all clear immediates.
160 if (immediate == 0) {
161 switch (op) {
162 case AND:
163 Mov(rd, 0);
164 return;
165 case ORR: // Fall through.
166 case EOR:
167 Mov(rd, rn);
168 return;
169 case ANDS: // Fall through.
170 case BICS:
171 break;
172 default:
173 UNREACHABLE();
174 }
175 } else if ((rd.Is64Bits() && (immediate == -1L)) ||
176 (rd.Is32Bits() && (immediate == 0xFFFFFFFFL))) {
177 switch (op) {
178 case AND:
179 Mov(rd, rn);
180 return;
181 case ORR:
182 Mov(rd, immediate);
183 return;
184 case EOR:
185 Mvn(rd, rn);
186 return;
187 case ANDS: // Fall through.
188 case BICS:
189 break;
190 default:
191 UNREACHABLE();
192 }
193 }
194
195 unsigned n, imm_s, imm_r;
196 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
197 // Immediate can be encoded in the instruction.
198 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
199 } else {
200 // Immediate can't be encoded: synthesize using move immediate.
201 Register temp = temps.AcquireSameSizeAs(rn);
202
203 // If the left-hand input is the stack pointer, we can't pre-shift the
204 // immediate, as the encoding won't allow the subsequent post shift.
205 PreShiftImmMode mode = rn.Is(sp) ? kNoShift : kAnyShift;
206 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);
207
208 if (rd.IsSP()) {
209 // If rd is the stack pointer we cannot use it as the destination
210 // register so we use the temp register as an intermediate again.
211 Logical(temp, rn, imm_operand, op);
212 Mov(sp, temp);
213 } else {
214 Logical(rd, rn, imm_operand, op);
215 }
216 }
217
218 } else if (operand.IsExtendedRegister()) {
219 DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
220 // Add/sub extended supports shift <= 4. We want to support exactly the
221 // same modes here.
222 DCHECK_LE(operand.shift_amount(), 4);
223 DCHECK(operand.reg().Is64Bits() ||
224 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
225 Register temp = temps.AcquireSameSizeAs(rn);
226 EmitExtendShift(temp, operand.reg(), operand.extend(),
227 operand.shift_amount());
228 Logical(rd, rn, temp, op);
229
230 } else {
231 // The operand can be encoded in the instruction.
232 DCHECK(operand.IsShiftedRegister());
233 Logical(rd, rn, operand, op);
234 }
235 }
236
Mov(const Register & rd,uint64_t imm)237 void TurboAssembler::Mov(const Register& rd, uint64_t imm) {
238 DCHECK(allow_macro_instructions());
239 DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
240 DCHECK(!rd.IsZero());
241
242 // TODO(all) extend to support more immediates.
243 //
244 // Immediates on Aarch64 can be produced using an initial value, and zero to
245 // three move keep operations.
246 //
247 // Initial values can be generated with:
248 // 1. 64-bit move zero (movz).
249 // 2. 32-bit move inverted (movn).
250 // 3. 64-bit move inverted.
251 // 4. 32-bit orr immediate.
252 // 5. 64-bit orr immediate.
253 // Move-keep may then be used to modify each of the 16-bit half-words.
254 //
255 // The code below supports all five initial value generators, and
256 // applying move-keep operations to move-zero and move-inverted initial
257 // values.
258
259 // Try to move the immediate in one instruction, and if that fails, switch to
260 // using multiple instructions.
261 if (!TryOneInstrMoveImmediate(rd, imm)) {
262 unsigned reg_size = rd.SizeInBits();
263
264 // Generic immediate case. Imm will be represented by
265 // [imm3, imm2, imm1, imm0], where each imm is 16 bits.
266 // A move-zero or move-inverted is generated for the first non-zero or
267 // non-0xFFFF immX, and a move-keep for subsequent non-zero immX.
268
269 uint64_t ignored_halfword = 0;
270 bool invert_move = false;
271 // If the number of 0xFFFF halfwords is greater than the number of 0x0000
272 // halfwords, it's more efficient to use move-inverted.
273 if (CountClearHalfWords(~imm, reg_size) >
274 CountClearHalfWords(imm, reg_size)) {
275 ignored_halfword = 0xFFFFL;
276 invert_move = true;
277 }
278
279 // Mov instructions can't move immediate values into the stack pointer, so
280 // set up a temporary register, if needed.
281 UseScratchRegisterScope temps(this);
282 Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
283
284 // Iterate through the halfwords. Use movn/movz for the first non-ignored
285 // halfword, and movk for subsequent halfwords.
286 DCHECK_EQ(reg_size % 16, 0);
287 bool first_mov_done = false;
288 for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
289 uint64_t imm16 = (imm >> (16 * i)) & 0xFFFFL;
290 if (imm16 != ignored_halfword) {
291 if (!first_mov_done) {
292 if (invert_move) {
293 movn(temp, (~imm16) & 0xFFFFL, 16 * i);
294 } else {
295 movz(temp, imm16, 16 * i);
296 }
297 first_mov_done = true;
298 } else {
299 // Construct a wider constant.
300 movk(temp, imm16, 16 * i);
301 }
302 }
303 }
304 DCHECK(first_mov_done);
305
306 // Move the temporary if the original destination register was the stack
307 // pointer.
308 if (rd.IsSP()) {
309 mov(rd, temp);
310 }
311 }
312 }
313
Mov(const Register & rd,const Operand & operand,DiscardMoveMode discard_mode)314 void TurboAssembler::Mov(const Register& rd, const Operand& operand,
315 DiscardMoveMode discard_mode) {
316 DCHECK(allow_macro_instructions());
317 DCHECK(!rd.IsZero());
318
319 // Provide a swap register for instructions that need to write into the
320 // system stack pointer (and can't do this inherently).
321 UseScratchRegisterScope temps(this);
322 Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
323
324 if (operand.NeedsRelocation(this)) {
325 Ldr(dst, operand);
326 } else if (operand.IsImmediate()) {
327 // Call the macro assembler for generic immediates.
328 Mov(dst, operand.ImmediateValue());
329
330 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
331 // Emit a shift instruction if moving a shifted register. This operation
332 // could also be achieved using an orr instruction (like orn used by Mvn),
333 // but using a shift instruction makes the disassembly clearer.
334 EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
335
336 } else if (operand.IsExtendedRegister()) {
337 // Emit an extend instruction if moving an extended register. This handles
338 // extend with post-shift operations, too.
339 EmitExtendShift(dst, operand.reg(), operand.extend(),
340 operand.shift_amount());
341
342 } else {
343 // Otherwise, emit a register move only if the registers are distinct, or
344 // if they are not X registers.
345 //
346 // Note that mov(w0, w0) is not a no-op because it clears the top word of
347 // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
348 // registers is not required to clear the top word of the X register. In
349 // this case, the instruction is discarded.
350 //
351 // If sp is an operand, add #0 is emitted, otherwise, orr #0.
352 if (!rd.Is(operand.reg()) || (rd.Is32Bits() &&
353 (discard_mode == kDontDiscardForSameWReg))) {
354 Assembler::mov(rd, operand.reg());
355 }
356 // This case can handle writes into the system stack pointer directly.
357 dst = rd;
358 }
359
360 // Copy the result to the system stack pointer.
361 if (!dst.Is(rd)) {
362 DCHECK(rd.IsSP());
363 Assembler::mov(rd, dst);
364 }
365 }
366
Mov(const Register & rd,ExternalReference reference)367 void TurboAssembler::Mov(const Register& rd, ExternalReference reference) {
368 #ifdef V8_EMBEDDED_BUILTINS
369 if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
370 LookupExternalReference(rd, reference);
371 return;
372 }
373 #endif // V8_EMBEDDED_BUILTINS
374 Mov(rd, Operand(reference));
375 }
376
Movi16bitHelper(const VRegister & vd,uint64_t imm)377 void TurboAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
378 DCHECK(is_uint16(imm));
379 int byte1 = (imm & 0xFF);
380 int byte2 = ((imm >> 8) & 0xFF);
381 if (byte1 == byte2) {
382 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
383 } else if (byte1 == 0) {
384 movi(vd, byte2, LSL, 8);
385 } else if (byte2 == 0) {
386 movi(vd, byte1);
387 } else if (byte1 == 0xFF) {
388 mvni(vd, ~byte2 & 0xFF, LSL, 8);
389 } else if (byte2 == 0xFF) {
390 mvni(vd, ~byte1 & 0xFF);
391 } else {
392 UseScratchRegisterScope temps(this);
393 Register temp = temps.AcquireW();
394 movz(temp, imm);
395 dup(vd, temp);
396 }
397 }
398
Movi32bitHelper(const VRegister & vd,uint64_t imm)399 void TurboAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
400 DCHECK(is_uint32(imm));
401
402 uint8_t bytes[sizeof(imm)];
403 memcpy(bytes, &imm, sizeof(imm));
404
405 // All bytes are either 0x00 or 0xFF.
406 {
407 bool all0orff = true;
408 for (int i = 0; i < 4; ++i) {
409 if ((bytes[i] != 0) && (bytes[i] != 0xFF)) {
410 all0orff = false;
411 break;
412 }
413 }
414
415 if (all0orff == true) {
416 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
417 return;
418 }
419 }
420
421 // Of the 4 bytes, only one byte is non-zero.
422 for (int i = 0; i < 4; i++) {
423 if ((imm & (0xFF << (i * 8))) == imm) {
424 movi(vd, bytes[i], LSL, i * 8);
425 return;
426 }
427 }
428
429 // Of the 4 bytes, only one byte is not 0xFF.
430 for (int i = 0; i < 4; i++) {
431 uint32_t mask = ~(0xFF << (i * 8));
432 if ((imm & mask) == mask) {
433 mvni(vd, ~bytes[i] & 0xFF, LSL, i * 8);
434 return;
435 }
436 }
437
438 // Immediate is of the form 0x00MMFFFF.
439 if ((imm & 0xFF00FFFF) == 0x0000FFFF) {
440 movi(vd, bytes[2], MSL, 16);
441 return;
442 }
443
444 // Immediate is of the form 0x0000MMFF.
445 if ((imm & 0xFFFF00FF) == 0x000000FF) {
446 movi(vd, bytes[1], MSL, 8);
447 return;
448 }
449
450 // Immediate is of the form 0xFFMM0000.
451 if ((imm & 0xFF00FFFF) == 0xFF000000) {
452 mvni(vd, ~bytes[2] & 0xFF, MSL, 16);
453 return;
454 }
455 // Immediate is of the form 0xFFFFMM00.
456 if ((imm & 0xFFFF00FF) == 0xFFFF0000) {
457 mvni(vd, ~bytes[1] & 0xFF, MSL, 8);
458 return;
459 }
460
461 // Top and bottom 16-bits are equal.
462 if (((imm >> 16) & 0xFFFF) == (imm & 0xFFFF)) {
463 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xFFFF);
464 return;
465 }
466
467 // Default case.
468 {
469 UseScratchRegisterScope temps(this);
470 Register temp = temps.AcquireW();
471 Mov(temp, imm);
472 dup(vd, temp);
473 }
474 }
475
Movi64bitHelper(const VRegister & vd,uint64_t imm)476 void TurboAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
477 // All bytes are either 0x00 or 0xFF.
478 {
479 bool all0orff = true;
480 for (int i = 0; i < 8; ++i) {
481 int byteval = (imm >> (i * 8)) & 0xFF;
482 if (byteval != 0 && byteval != 0xFF) {
483 all0orff = false;
484 break;
485 }
486 }
487 if (all0orff == true) {
488 movi(vd, imm);
489 return;
490 }
491 }
492
493 // Top and bottom 32-bits are equal.
494 if (((imm >> 32) & 0xFFFFFFFF) == (imm & 0xFFFFFFFF)) {
495 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xFFFFFFFF);
496 return;
497 }
498
499 // Default case.
500 {
501 UseScratchRegisterScope temps(this);
502 Register temp = temps.AcquireX();
503 Mov(temp, imm);
504 if (vd.Is1D()) {
505 mov(vd.D(), 0, temp);
506 } else {
507 dup(vd.V2D(), temp);
508 }
509 }
510 }
511
Movi(const VRegister & vd,uint64_t imm,Shift shift,int shift_amount)512 void TurboAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
513 int shift_amount) {
514 DCHECK(allow_macro_instructions());
515 if (shift_amount != 0 || shift != LSL) {
516 movi(vd, imm, shift, shift_amount);
517 } else if (vd.Is8B() || vd.Is16B()) {
518 // 8-bit immediate.
519 DCHECK(is_uint8(imm));
520 movi(vd, imm);
521 } else if (vd.Is4H() || vd.Is8H()) {
522 // 16-bit immediate.
523 Movi16bitHelper(vd, imm);
524 } else if (vd.Is2S() || vd.Is4S()) {
525 // 32-bit immediate.
526 Movi32bitHelper(vd, imm);
527 } else {
528 // 64-bit immediate.
529 Movi64bitHelper(vd, imm);
530 }
531 }
532
Movi(const VRegister & vd,uint64_t hi,uint64_t lo)533 void TurboAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
534 // TODO(all): Move 128-bit values in a more efficient way.
535 DCHECK(vd.Is128Bits());
536 UseScratchRegisterScope temps(this);
537 Movi(vd.V2D(), lo);
538 Register temp = temps.AcquireX();
539 Mov(temp, hi);
540 Ins(vd.V2D(), 1, temp);
541 }
542
Mvn(const Register & rd,const Operand & operand)543 void TurboAssembler::Mvn(const Register& rd, const Operand& operand) {
544 DCHECK(allow_macro_instructions());
545
546 if (operand.NeedsRelocation(this)) {
547 Ldr(rd, operand.immediate());
548 mvn(rd, rd);
549
550 } else if (operand.IsImmediate()) {
551 // Call the macro assembler for generic immediates.
552 Mov(rd, ~operand.ImmediateValue());
553
554 } else if (operand.IsExtendedRegister()) {
555 // Emit two instructions for the extend case. This differs from Mov, as
556 // the extend and invert can't be achieved in one instruction.
557 EmitExtendShift(rd, operand.reg(), operand.extend(),
558 operand.shift_amount());
559 mvn(rd, rd);
560
561 } else {
562 mvn(rd, operand);
563 }
564 }
565
CountClearHalfWords(uint64_t imm,unsigned reg_size)566 unsigned TurboAssembler::CountClearHalfWords(uint64_t imm, unsigned reg_size) {
567 DCHECK_EQ(reg_size % 8, 0);
568 int count = 0;
569 for (unsigned i = 0; i < (reg_size / 16); i++) {
570 if ((imm & 0xFFFF) == 0) {
571 count++;
572 }
573 imm >>= 16;
574 }
575 return count;
576 }
577
578
579 // The movz instruction can generate immediates containing an arbitrary 16-bit
580 // half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
IsImmMovz(uint64_t imm,unsigned reg_size)581 bool TurboAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
582 DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
583 return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
584 }
585
586 // The movn instruction can generate immediates containing an arbitrary 16-bit
587 // half-word, with remaining bits set, eg. 0xFFFF1234, 0xFFFF1234FFFFFFFF.
IsImmMovn(uint64_t imm,unsigned reg_size)588 bool TurboAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
589 return IsImmMovz(~imm, reg_size);
590 }
591
ConditionalCompareMacro(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)592 void TurboAssembler::ConditionalCompareMacro(const Register& rn,
593 const Operand& operand,
594 StatusFlags nzcv, Condition cond,
595 ConditionalCompareOp op) {
596 DCHECK((cond != al) && (cond != nv));
597 if (operand.NeedsRelocation(this)) {
598 UseScratchRegisterScope temps(this);
599 Register temp = temps.AcquireX();
600 Ldr(temp, operand.immediate());
601 ConditionalCompareMacro(rn, temp, nzcv, cond, op);
602
603 } else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
604 (operand.IsImmediate() &&
605 IsImmConditionalCompare(operand.ImmediateValue()))) {
606 // The immediate can be encoded in the instruction, or the operand is an
607 // unshifted register: call the assembler.
608 ConditionalCompare(rn, operand, nzcv, cond, op);
609
610 } else {
611 // The operand isn't directly supported by the instruction: perform the
612 // operation on a temporary register.
613 UseScratchRegisterScope temps(this);
614 Register temp = temps.AcquireSameSizeAs(rn);
615 Mov(temp, operand);
616 ConditionalCompare(rn, temp, nzcv, cond, op);
617 }
618 }
619
Csel(const Register & rd,const Register & rn,const Operand & operand,Condition cond)620 void TurboAssembler::Csel(const Register& rd, const Register& rn,
621 const Operand& operand, Condition cond) {
622 DCHECK(allow_macro_instructions());
623 DCHECK(!rd.IsZero());
624 DCHECK((cond != al) && (cond != nv));
625 if (operand.IsImmediate()) {
626 // Immediate argument. Handle special cases of 0, 1 and -1 using zero
627 // register.
628 int64_t imm = operand.ImmediateValue();
629 Register zr = AppropriateZeroRegFor(rn);
630 if (imm == 0) {
631 csel(rd, rn, zr, cond);
632 } else if (imm == 1) {
633 csinc(rd, rn, zr, cond);
634 } else if (imm == -1) {
635 csinv(rd, rn, zr, cond);
636 } else {
637 UseScratchRegisterScope temps(this);
638 Register temp = temps.AcquireSameSizeAs(rn);
639 Mov(temp, imm);
640 csel(rd, rn, temp, cond);
641 }
642 } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
643 // Unshifted register argument.
644 csel(rd, rn, operand.reg(), cond);
645 } else {
646 // All other arguments.
647 UseScratchRegisterScope temps(this);
648 Register temp = temps.AcquireSameSizeAs(rn);
649 Mov(temp, operand);
650 csel(rd, rn, temp, cond);
651 }
652 }
653
TryOneInstrMoveImmediate(const Register & dst,int64_t imm)654 bool TurboAssembler::TryOneInstrMoveImmediate(const Register& dst,
655 int64_t imm) {
656 unsigned n, imm_s, imm_r;
657 int reg_size = dst.SizeInBits();
658 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
659 // Immediate can be represented in a move zero instruction. Movz can't write
660 // to the stack pointer.
661 movz(dst, imm);
662 return true;
663 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
664 // Immediate can be represented in a move not instruction. Movn can't write
665 // to the stack pointer.
666 movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
667 return true;
668 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
669 // Immediate can be represented in a logical orr instruction.
670 LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
671 return true;
672 }
673 return false;
674 }
675
MoveImmediateForShiftedOp(const Register & dst,int64_t imm,PreShiftImmMode mode)676 Operand TurboAssembler::MoveImmediateForShiftedOp(const Register& dst,
677 int64_t imm,
678 PreShiftImmMode mode) {
679 int reg_size = dst.SizeInBits();
680 // Encode the immediate in a single move instruction, if possible.
681 if (TryOneInstrMoveImmediate(dst, imm)) {
682 // The move was successful; nothing to do here.
683 } else {
684 // Pre-shift the immediate to the least-significant bits of the register.
685 int shift_low = CountTrailingZeros(imm, reg_size);
686 if (mode == kLimitShiftForSP) {
687 // When applied to the stack pointer, the subsequent arithmetic operation
688 // can use the extend form to shift left by a maximum of four bits. Right
689 // shifts are not allowed, so we filter them out later before the new
690 // immediate is tested.
691 shift_low = std::min(shift_low, 4);
692 }
693 int64_t imm_low = imm >> shift_low;
694
695 // Pre-shift the immediate to the most-significant bits of the register. We
696 // insert set bits in the least-significant bits, as this creates a
697 // different immediate that may be encodable using movn or orr-immediate.
698 // If this new immediate is encodable, the set bits will be eliminated by
699 // the post shift on the following instruction.
700 int shift_high = CountLeadingZeros(imm, reg_size);
701 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
702
703 if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
704 // The new immediate has been moved into the destination's low bits:
705 // return a new leftward-shifting operand.
706 return Operand(dst, LSL, shift_low);
707 } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
708 // The new immediate has been moved into the destination's high bits:
709 // return a new rightward-shifting operand.
710 return Operand(dst, LSR, shift_high);
711 } else {
712 // Use the generic move operation to set up the immediate.
713 Mov(dst, imm);
714 }
715 }
716 return Operand(dst);
717 }
718
AddSubMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)719 void TurboAssembler::AddSubMacro(const Register& rd, const Register& rn,
720 const Operand& operand, FlagsUpdate S,
721 AddSubOp op) {
722 if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() &&
723 !operand.NeedsRelocation(this) && (S == LeaveFlags)) {
724 // The instruction would be a nop. Avoid generating useless code.
725 return;
726 }
727
728 if (operand.NeedsRelocation(this)) {
729 UseScratchRegisterScope temps(this);
730 Register temp = temps.AcquireX();
731 Ldr(temp, operand.immediate());
732 AddSubMacro(rd, rn, temp, S, op);
733 } else if ((operand.IsImmediate() &&
734 !IsImmAddSub(operand.ImmediateValue())) ||
735 (rn.IsZero() && !operand.IsShiftedRegister()) ||
736 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
737 UseScratchRegisterScope temps(this);
738 Register temp = temps.AcquireSameSizeAs(rn);
739 if (operand.IsImmediate()) {
740 PreShiftImmMode mode = kAnyShift;
741
742 // If the destination or source register is the stack pointer, we can
743 // only pre-shift the immediate right by values supported in the add/sub
744 // extend encoding.
745 if (rd.Is(sp)) {
746 // If the destination is SP and flags will be set, we can't pre-shift
747 // the immediate at all.
748 mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
749 } else if (rn.Is(sp)) {
750 mode = kLimitShiftForSP;
751 }
752
753 Operand imm_operand =
754 MoveImmediateForShiftedOp(temp, operand.ImmediateValue(), mode);
755 AddSub(rd, rn, imm_operand, S, op);
756 } else {
757 Mov(temp, operand);
758 AddSub(rd, rn, temp, S, op);
759 }
760 } else {
761 AddSub(rd, rn, operand, S, op);
762 }
763 }
764
AddSubWithCarryMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)765 void TurboAssembler::AddSubWithCarryMacro(const Register& rd,
766 const Register& rn,
767 const Operand& operand, FlagsUpdate S,
768 AddSubWithCarryOp op) {
769 DCHECK(rd.SizeInBits() == rn.SizeInBits());
770 UseScratchRegisterScope temps(this);
771
772 if (operand.NeedsRelocation(this)) {
773 Register temp = temps.AcquireX();
774 Ldr(temp, operand.immediate());
775 AddSubWithCarryMacro(rd, rn, temp, S, op);
776
777 } else if (operand.IsImmediate() ||
778 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
779 // Add/sub with carry (immediate or ROR shifted register.)
780 Register temp = temps.AcquireSameSizeAs(rn);
781 Mov(temp, operand);
782 AddSubWithCarry(rd, rn, temp, S, op);
783
784 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
785 // Add/sub with carry (shifted register).
786 DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
787 DCHECK(operand.shift() != ROR);
788 DCHECK(is_uintn(operand.shift_amount(),
789 rd.SizeInBits() == kXRegSizeInBits ? kXRegSizeInBitsLog2
790 : kWRegSizeInBitsLog2));
791 Register temp = temps.AcquireSameSizeAs(rn);
792 EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
793 AddSubWithCarry(rd, rn, temp, S, op);
794
795 } else if (operand.IsExtendedRegister()) {
796 // Add/sub with carry (extended register).
797 DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
798 // Add/sub extended supports a shift <= 4. We want to support exactly the
799 // same modes.
800 DCHECK_LE(operand.shift_amount(), 4);
801 DCHECK(operand.reg().Is64Bits() ||
802 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
803 Register temp = temps.AcquireSameSizeAs(rn);
804 EmitExtendShift(temp, operand.reg(), operand.extend(),
805 operand.shift_amount());
806 AddSubWithCarry(rd, rn, temp, S, op);
807
808 } else {
809 // The addressing mode is directly supported by the instruction.
810 AddSubWithCarry(rd, rn, operand, S, op);
811 }
812 }
813
LoadStoreMacro(const CPURegister & rt,const MemOperand & addr,LoadStoreOp op)814 void TurboAssembler::LoadStoreMacro(const CPURegister& rt,
815 const MemOperand& addr, LoadStoreOp op) {
816 int64_t offset = addr.offset();
817 unsigned size = CalcLSDataSize(op);
818
819 // Check if an immediate offset fits in the immediate field of the
820 // appropriate instruction. If not, emit two instructions to perform
821 // the operation.
822 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
823 !IsImmLSUnscaled(offset)) {
824 // Immediate offset that can't be encoded using unsigned or unscaled
825 // addressing modes.
826 UseScratchRegisterScope temps(this);
827 Register temp = temps.AcquireSameSizeAs(addr.base());
828 Mov(temp, addr.offset());
829 LoadStore(rt, MemOperand(addr.base(), temp), op);
830 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
831 // Post-index beyond unscaled addressing range.
832 LoadStore(rt, MemOperand(addr.base()), op);
833 add(addr.base(), addr.base(), offset);
834 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
835 // Pre-index beyond unscaled addressing range.
836 add(addr.base(), addr.base(), offset);
837 LoadStore(rt, MemOperand(addr.base()), op);
838 } else {
839 // Encodable in one load/store instruction.
840 LoadStore(rt, addr, op);
841 }
842 }
843
LoadStorePairMacro(const CPURegister & rt,const CPURegister & rt2,const MemOperand & addr,LoadStorePairOp op)844 void TurboAssembler::LoadStorePairMacro(const CPURegister& rt,
845 const CPURegister& rt2,
846 const MemOperand& addr,
847 LoadStorePairOp op) {
848 // TODO(all): Should we support register offset for load-store-pair?
849 DCHECK(!addr.IsRegisterOffset());
850
851 int64_t offset = addr.offset();
852 unsigned size = CalcLSPairDataSize(op);
853
854 // Check if the offset fits in the immediate field of the appropriate
855 // instruction. If not, emit two instructions to perform the operation.
856 if (IsImmLSPair(offset, size)) {
857 // Encodable in one load/store pair instruction.
858 LoadStorePair(rt, rt2, addr, op);
859 } else {
860 Register base = addr.base();
861 if (addr.IsImmediateOffset()) {
862 UseScratchRegisterScope temps(this);
863 Register temp = temps.AcquireSameSizeAs(base);
864 Add(temp, base, offset);
865 LoadStorePair(rt, rt2, MemOperand(temp), op);
866 } else if (addr.IsPostIndex()) {
867 LoadStorePair(rt, rt2, MemOperand(base), op);
868 Add(base, base, offset);
869 } else {
870 DCHECK(addr.IsPreIndex());
871 Add(base, base, offset);
872 LoadStorePair(rt, rt2, MemOperand(base), op);
873 }
874 }
875 }
876
NeedExtraInstructionsOrRegisterBranch(Label * label,ImmBranchType b_type)877 bool TurboAssembler::NeedExtraInstructionsOrRegisterBranch(
878 Label* label, ImmBranchType b_type) {
879 bool need_longer_range = false;
880 // There are two situations in which we care about the offset being out of
881 // range:
882 // - The label is bound but too far away.
883 // - The label is not bound but linked, and the previous branch
884 // instruction in the chain is too far away.
885 if (label->is_bound() || label->is_linked()) {
886 need_longer_range =
887 !Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
888 }
889 if (!need_longer_range && !label->is_bound()) {
890 int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
891 unresolved_branches_.insert(
892 std::pair<int, FarBranchInfo>(max_reachable_pc,
893 FarBranchInfo(pc_offset(), label)));
894 // Also maintain the next pool check.
895 next_veneer_pool_check_ =
896 Min(next_veneer_pool_check_,
897 max_reachable_pc - kVeneerDistanceCheckMargin);
898 }
899 return need_longer_range;
900 }
901
Adr(const Register & rd,Label * label,AdrHint hint)902 void TurboAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
903 DCHECK(allow_macro_instructions());
904 DCHECK(!rd.IsZero());
905
906 if (hint == kAdrNear) {
907 adr(rd, label);
908 return;
909 }
910
911 DCHECK_EQ(hint, kAdrFar);
912 if (label->is_bound()) {
913 int label_offset = label->pos() - pc_offset();
914 if (Instruction::IsValidPCRelOffset(label_offset)) {
915 adr(rd, label);
916 } else {
917 DCHECK_LE(label_offset, 0);
918 int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
919 adr(rd, min_adr_offset);
920 Add(rd, rd, label_offset - min_adr_offset);
921 }
922 } else {
923 UseScratchRegisterScope temps(this);
924 Register scratch = temps.AcquireX();
925
926 InstructionAccurateScope scope(
927 this, PatchingAssembler::kAdrFarPatchableNInstrs);
928 adr(rd, label);
929 for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
930 nop(ADR_FAR_NOP);
931 }
932 movz(scratch, 0);
933 }
934 }
935
B(Label * label,BranchType type,Register reg,int bit)936 void TurboAssembler::B(Label* label, BranchType type, Register reg, int bit) {
937 DCHECK((reg.Is(NoReg) || type >= kBranchTypeFirstUsingReg) &&
938 (bit == -1 || type >= kBranchTypeFirstUsingBit));
939 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
940 B(static_cast<Condition>(type), label);
941 } else {
942 switch (type) {
943 case always: B(label); break;
944 case never: break;
945 case reg_zero: Cbz(reg, label); break;
946 case reg_not_zero: Cbnz(reg, label); break;
947 case reg_bit_clear: Tbz(reg, bit, label); break;
948 case reg_bit_set: Tbnz(reg, bit, label); break;
949 default:
950 UNREACHABLE();
951 }
952 }
953 }
954
B(Label * label,Condition cond)955 void TurboAssembler::B(Label* label, Condition cond) {
956 DCHECK(allow_macro_instructions());
957 DCHECK((cond != al) && (cond != nv));
958
959 Label done;
960 bool need_extra_instructions =
961 NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
962
963 if (need_extra_instructions) {
964 b(&done, NegateCondition(cond));
965 B(label);
966 } else {
967 b(label, cond);
968 }
969 bind(&done);
970 }
971
Tbnz(const Register & rt,unsigned bit_pos,Label * label)972 void TurboAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
973 DCHECK(allow_macro_instructions());
974
975 Label done;
976 bool need_extra_instructions =
977 NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
978
979 if (need_extra_instructions) {
980 tbz(rt, bit_pos, &done);
981 B(label);
982 } else {
983 tbnz(rt, bit_pos, label);
984 }
985 bind(&done);
986 }
987
Tbz(const Register & rt,unsigned bit_pos,Label * label)988 void TurboAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
989 DCHECK(allow_macro_instructions());
990
991 Label done;
992 bool need_extra_instructions =
993 NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
994
995 if (need_extra_instructions) {
996 tbnz(rt, bit_pos, &done);
997 B(label);
998 } else {
999 tbz(rt, bit_pos, label);
1000 }
1001 bind(&done);
1002 }
1003
Cbnz(const Register & rt,Label * label)1004 void TurboAssembler::Cbnz(const Register& rt, Label* label) {
1005 DCHECK(allow_macro_instructions());
1006
1007 Label done;
1008 bool need_extra_instructions =
1009 NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1010
1011 if (need_extra_instructions) {
1012 cbz(rt, &done);
1013 B(label);
1014 } else {
1015 cbnz(rt, label);
1016 }
1017 bind(&done);
1018 }
1019
Cbz(const Register & rt,Label * label)1020 void TurboAssembler::Cbz(const Register& rt, Label* label) {
1021 DCHECK(allow_macro_instructions());
1022
1023 Label done;
1024 bool need_extra_instructions =
1025 NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1026
1027 if (need_extra_instructions) {
1028 cbnz(rt, &done);
1029 B(label);
1030 } else {
1031 cbz(rt, label);
1032 }
1033 bind(&done);
1034 }
1035
1036
1037 // Pseudo-instructions.
1038
Abs(const Register & rd,const Register & rm,Label * is_not_representable,Label * is_representable)1039 void TurboAssembler::Abs(const Register& rd, const Register& rm,
1040 Label* is_not_representable, Label* is_representable) {
1041 DCHECK(allow_macro_instructions());
1042 DCHECK(AreSameSizeAndType(rd, rm));
1043
1044 Cmp(rm, 1);
1045 Cneg(rd, rm, lt);
1046
1047 // If the comparison sets the v flag, the input was the smallest value
1048 // representable by rm, and the mathematical result of abs(rm) is not
1049 // representable using two's complement.
1050 if ((is_not_representable != nullptr) && (is_representable != nullptr)) {
1051 B(is_not_representable, vs);
1052 B(is_representable);
1053 } else if (is_not_representable != nullptr) {
1054 B(is_not_representable, vs);
1055 } else if (is_representable != nullptr) {
1056 B(is_representable, vc);
1057 }
1058 }
1059
1060
1061 // Abstracted stack operations.
1062
Push(const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1063 void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1064 const CPURegister& src2, const CPURegister& src3) {
1065 DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
1066
1067 int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid();
1068 int size = src0.SizeInBytes();
1069
1070 PushPreamble(count, size);
1071 PushHelper(count, size, src0, src1, src2, src3);
1072 }
1073
Push(const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3,const CPURegister & src4,const CPURegister & src5,const CPURegister & src6,const CPURegister & src7)1074 void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1075 const CPURegister& src2, const CPURegister& src3,
1076 const CPURegister& src4, const CPURegister& src5,
1077 const CPURegister& src6, const CPURegister& src7) {
1078 DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
1079
1080 int count = 5 + src5.IsValid() + src6.IsValid() + src6.IsValid();
1081 int size = src0.SizeInBytes();
1082
1083 PushPreamble(count, size);
1084 PushHelper(4, size, src0, src1, src2, src3);
1085 PushHelper(count - 4, size, src4, src5, src6, src7);
1086 }
1087
Pop(const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1088 void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1089 const CPURegister& dst2, const CPURegister& dst3) {
1090 // It is not valid to pop into the same register more than once in one
1091 // instruction, not even into the zero register.
1092 DCHECK(!AreAliased(dst0, dst1, dst2, dst3));
1093 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1094 DCHECK(dst0.IsValid());
1095
1096 int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid();
1097 int size = dst0.SizeInBytes();
1098
1099 PopHelper(count, size, dst0, dst1, dst2, dst3);
1100 PopPostamble(count, size);
1101 }
1102
Pop(const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3,const CPURegister & dst4,const CPURegister & dst5,const CPURegister & dst6,const CPURegister & dst7)1103 void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1104 const CPURegister& dst2, const CPURegister& dst3,
1105 const CPURegister& dst4, const CPURegister& dst5,
1106 const CPURegister& dst6, const CPURegister& dst7) {
1107 // It is not valid to pop into the same register more than once in one
1108 // instruction, not even into the zero register.
1109 DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1110 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1111 DCHECK(dst0.IsValid());
1112
1113 int count = 5 + dst5.IsValid() + dst6.IsValid() + dst7.IsValid();
1114 int size = dst0.SizeInBytes();
1115
1116 PopHelper(4, size, dst0, dst1, dst2, dst3);
1117 PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
1118 PopPostamble(count, size);
1119 }
1120
Push(const Register & src0,const VRegister & src1)1121 void TurboAssembler::Push(const Register& src0, const VRegister& src1) {
1122 int size = src0.SizeInBytes() + src1.SizeInBytes();
1123
1124 PushPreamble(size);
1125 // Reserve room for src0 and push src1.
1126 str(src1, MemOperand(sp, -size, PreIndex));
1127 // Fill the gap with src0.
1128 str(src0, MemOperand(sp, src1.SizeInBytes()));
1129 }
1130
1131
PushQueued(PreambleDirective preamble_directive)1132 void MacroAssembler::PushPopQueue::PushQueued(
1133 PreambleDirective preamble_directive) {
1134 if (queued_.empty()) return;
1135
1136 if (preamble_directive == WITH_PREAMBLE) {
1137 masm_->PushPreamble(size_);
1138 }
1139
1140 size_t count = queued_.size();
1141 size_t index = 0;
1142 while (index < count) {
1143 // PushHelper can only handle registers with the same size and type, and it
1144 // can handle only four at a time. Batch them up accordingly.
1145 CPURegister batch[4] = {NoReg, NoReg, NoReg, NoReg};
1146 int batch_index = 0;
1147 do {
1148 batch[batch_index++] = queued_[index++];
1149 } while ((batch_index < 4) && (index < count) &&
1150 batch[0].IsSameSizeAndType(queued_[index]));
1151
1152 masm_->PushHelper(batch_index, batch[0].SizeInBytes(),
1153 batch[0], batch[1], batch[2], batch[3]);
1154 }
1155
1156 queued_.clear();
1157 }
1158
1159
PopQueued()1160 void MacroAssembler::PushPopQueue::PopQueued() {
1161 if (queued_.empty()) return;
1162
1163 size_t count = queued_.size();
1164 size_t index = 0;
1165 while (index < count) {
1166 // PopHelper can only handle registers with the same size and type, and it
1167 // can handle only four at a time. Batch them up accordingly.
1168 CPURegister batch[4] = {NoReg, NoReg, NoReg, NoReg};
1169 int batch_index = 0;
1170 do {
1171 batch[batch_index++] = queued_[index++];
1172 } while ((batch_index < 4) && (index < count) &&
1173 batch[0].IsSameSizeAndType(queued_[index]));
1174
1175 masm_->PopHelper(batch_index, batch[0].SizeInBytes(),
1176 batch[0], batch[1], batch[2], batch[3]);
1177 }
1178
1179 masm_->PopPostamble(size_);
1180 queued_.clear();
1181 }
1182
PushCPURegList(CPURegList registers)1183 void TurboAssembler::PushCPURegList(CPURegList registers) {
1184 int size = registers.RegisterSizeInBytes();
1185
1186 PushPreamble(registers.Count(), size);
1187 // Push up to four registers at a time.
1188 while (!registers.IsEmpty()) {
1189 int count_before = registers.Count();
1190 const CPURegister& src0 = registers.PopHighestIndex();
1191 const CPURegister& src1 = registers.PopHighestIndex();
1192 const CPURegister& src2 = registers.PopHighestIndex();
1193 const CPURegister& src3 = registers.PopHighestIndex();
1194 int count = count_before - registers.Count();
1195 PushHelper(count, size, src0, src1, src2, src3);
1196 }
1197 }
1198
PopCPURegList(CPURegList registers)1199 void TurboAssembler::PopCPURegList(CPURegList registers) {
1200 int size = registers.RegisterSizeInBytes();
1201
1202 // Pop up to four registers at a time.
1203 while (!registers.IsEmpty()) {
1204 int count_before = registers.Count();
1205 const CPURegister& dst0 = registers.PopLowestIndex();
1206 const CPURegister& dst1 = registers.PopLowestIndex();
1207 const CPURegister& dst2 = registers.PopLowestIndex();
1208 const CPURegister& dst3 = registers.PopLowestIndex();
1209 int count = count_before - registers.Count();
1210 PopHelper(count, size, dst0, dst1, dst2, dst3);
1211 }
1212 PopPostamble(registers.Count(), size);
1213 }
1214
PushMultipleTimes(CPURegister src,Register count)1215 void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
1216 PushPreamble(Operand(count, UXTW, WhichPowerOf2(src.SizeInBytes())));
1217
1218 UseScratchRegisterScope temps(this);
1219 Register temp = temps.AcquireSameSizeAs(count);
1220
1221 if (FLAG_optimize_for_size) {
1222 Label loop, done;
1223
1224 Subs(temp, count, 1);
1225 B(mi, &done);
1226
1227 // Push all registers individually, to save code size.
1228 Bind(&loop);
1229 Subs(temp, temp, 1);
1230 PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
1231 B(pl, &loop);
1232
1233 Bind(&done);
1234 } else {
1235 Label loop, leftover2, leftover1, done;
1236
1237 Subs(temp, count, 4);
1238 B(mi, &leftover2);
1239
1240 // Push groups of four first.
1241 Bind(&loop);
1242 Subs(temp, temp, 4);
1243 PushHelper(4, src.SizeInBytes(), src, src, src, src);
1244 B(pl, &loop);
1245
1246 // Push groups of two.
1247 Bind(&leftover2);
1248 Tbz(count, 1, &leftover1);
1249 PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
1250
1251 // Push the last one (if required).
1252 Bind(&leftover1);
1253 Tbz(count, 0, &done);
1254 PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
1255
1256 Bind(&done);
1257 }
1258 }
1259
PushHelper(int count,int size,const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1260 void TurboAssembler::PushHelper(int count, int size, const CPURegister& src0,
1261 const CPURegister& src1,
1262 const CPURegister& src2,
1263 const CPURegister& src3) {
1264 // Ensure that we don't unintentially modify scratch or debug registers.
1265 InstructionAccurateScope scope(this);
1266
1267 DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
1268 DCHECK(size == src0.SizeInBytes());
1269
1270 // When pushing multiple registers, the store order is chosen such that
1271 // Push(a, b) is equivalent to Push(a) followed by Push(b).
1272 switch (count) {
1273 case 1:
1274 DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
1275 str(src0, MemOperand(sp, -1 * size, PreIndex));
1276 break;
1277 case 2:
1278 DCHECK(src2.IsNone() && src3.IsNone());
1279 stp(src1, src0, MemOperand(sp, -2 * size, PreIndex));
1280 break;
1281 case 3:
1282 DCHECK(src3.IsNone());
1283 stp(src2, src1, MemOperand(sp, -3 * size, PreIndex));
1284 str(src0, MemOperand(sp, 2 * size));
1285 break;
1286 case 4:
1287 // Skip over 4 * size, then fill in the gap. This allows four W registers
1288 // to be pushed using sp, whilst maintaining 16-byte alignment for sp
1289 // at all times.
1290 stp(src3, src2, MemOperand(sp, -4 * size, PreIndex));
1291 stp(src1, src0, MemOperand(sp, 2 * size));
1292 break;
1293 default:
1294 UNREACHABLE();
1295 }
1296 }
1297
PopHelper(int count,int size,const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1298 void TurboAssembler::PopHelper(int count, int size, const CPURegister& dst0,
1299 const CPURegister& dst1, const CPURegister& dst2,
1300 const CPURegister& dst3) {
1301 // Ensure that we don't unintentially modify scratch or debug registers.
1302 InstructionAccurateScope scope(this);
1303
1304 DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1305 DCHECK(size == dst0.SizeInBytes());
1306
1307 // When popping multiple registers, the load order is chosen such that
1308 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
1309 switch (count) {
1310 case 1:
1311 DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
1312 ldr(dst0, MemOperand(sp, 1 * size, PostIndex));
1313 break;
1314 case 2:
1315 DCHECK(dst2.IsNone() && dst3.IsNone());
1316 ldp(dst0, dst1, MemOperand(sp, 2 * size, PostIndex));
1317 break;
1318 case 3:
1319 DCHECK(dst3.IsNone());
1320 ldr(dst2, MemOperand(sp, 2 * size));
1321 ldp(dst0, dst1, MemOperand(sp, 3 * size, PostIndex));
1322 break;
1323 case 4:
1324 // Load the higher addresses first, then load the lower addresses and
1325 // skip the whole block in the second instruction. This allows four W
1326 // registers to be popped using sp, whilst maintaining 16-byte alignment
1327 // for sp at all times.
1328 ldp(dst2, dst3, MemOperand(sp, 2 * size));
1329 ldp(dst0, dst1, MemOperand(sp, 4 * size, PostIndex));
1330 break;
1331 default:
1332 UNREACHABLE();
1333 }
1334 }
1335
PushPreamble(Operand total_size)1336 void TurboAssembler::PushPreamble(Operand total_size) {
1337 if (total_size.IsZero()) return;
1338
1339 // The stack pointer must be aligned to 16 bytes on entry, and the total
1340 // size of the specified registers must also be a multiple of 16 bytes.
1341 if (total_size.IsImmediate()) {
1342 DCHECK_EQ(total_size.ImmediateValue() % 16, 0);
1343 }
1344
1345 // Don't check access size for non-immediate sizes. It's difficult to do
1346 // well, and it will be caught by hardware (or the simulator) anyway.
1347 }
1348
PopPostamble(Operand total_size)1349 void TurboAssembler::PopPostamble(Operand total_size) {
1350 if (total_size.IsZero()) return;
1351
1352 // The stack pointer must be aligned to 16 bytes on entry, and the total
1353 // size of the specified registers must also be a multiple of 16 bytes.
1354 if (total_size.IsImmediate()) {
1355 DCHECK_EQ(total_size.ImmediateValue() % 16, 0);
1356 }
1357
1358 // Don't check access size for non-immediate sizes. It's difficult to do
1359 // well, and it will be caught by hardware (or the simulator) anyway.
1360 }
1361
PushPreamble(int count,int size)1362 void TurboAssembler::PushPreamble(int count, int size) {
1363 PushPreamble(count * size);
1364 }
PopPostamble(int count,int size)1365 void TurboAssembler::PopPostamble(int count, int size) {
1366 PopPostamble(count * size);
1367 }
1368
Poke(const CPURegister & src,const Operand & offset)1369 void TurboAssembler::Poke(const CPURegister& src, const Operand& offset) {
1370 if (offset.IsImmediate()) {
1371 DCHECK_GE(offset.ImmediateValue(), 0);
1372 } else if (emit_debug_code()) {
1373 Cmp(xzr, offset);
1374 Check(le, AbortReason::kStackAccessBelowStackPointer);
1375 }
1376
1377 Str(src, MemOperand(sp, offset));
1378 }
1379
Peek(const CPURegister & dst,const Operand & offset)1380 void TurboAssembler::Peek(const CPURegister& dst, const Operand& offset) {
1381 if (offset.IsImmediate()) {
1382 DCHECK_GE(offset.ImmediateValue(), 0);
1383 } else if (emit_debug_code()) {
1384 Cmp(xzr, offset);
1385 Check(le, AbortReason::kStackAccessBelowStackPointer);
1386 }
1387
1388 Ldr(dst, MemOperand(sp, offset));
1389 }
1390
PokePair(const CPURegister & src1,const CPURegister & src2,int offset)1391 void TurboAssembler::PokePair(const CPURegister& src1, const CPURegister& src2,
1392 int offset) {
1393 DCHECK(AreSameSizeAndType(src1, src2));
1394 DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
1395 Stp(src1, src2, MemOperand(sp, offset));
1396 }
1397
1398
PeekPair(const CPURegister & dst1,const CPURegister & dst2,int offset)1399 void MacroAssembler::PeekPair(const CPURegister& dst1,
1400 const CPURegister& dst2,
1401 int offset) {
1402 DCHECK(AreSameSizeAndType(dst1, dst2));
1403 DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
1404 Ldp(dst1, dst2, MemOperand(sp, offset));
1405 }
1406
1407
PushCalleeSavedRegisters()1408 void MacroAssembler::PushCalleeSavedRegisters() {
1409 // Ensure that the macro-assembler doesn't use any scratch registers.
1410 InstructionAccurateScope scope(this);
1411
1412 MemOperand tos(sp, -2 * static_cast<int>(kXRegSize), PreIndex);
1413
1414 stp(d14, d15, tos);
1415 stp(d12, d13, tos);
1416 stp(d10, d11, tos);
1417 stp(d8, d9, tos);
1418
1419 stp(x29, x30, tos);
1420 stp(x27, x28, tos);
1421 stp(x25, x26, tos);
1422 stp(x23, x24, tos);
1423 stp(x21, x22, tos);
1424 stp(x19, x20, tos);
1425 }
1426
1427
PopCalleeSavedRegisters()1428 void MacroAssembler::PopCalleeSavedRegisters() {
1429 // Ensure that the macro-assembler doesn't use any scratch registers.
1430 InstructionAccurateScope scope(this);
1431
1432 MemOperand tos(sp, 2 * kXRegSize, PostIndex);
1433
1434 ldp(x19, x20, tos);
1435 ldp(x21, x22, tos);
1436 ldp(x23, x24, tos);
1437 ldp(x25, x26, tos);
1438 ldp(x27, x28, tos);
1439 ldp(x29, x30, tos);
1440
1441 ldp(d8, d9, tos);
1442 ldp(d10, d11, tos);
1443 ldp(d12, d13, tos);
1444 ldp(d14, d15, tos);
1445 }
1446
AssertSpAligned()1447 void TurboAssembler::AssertSpAligned() {
1448 if (emit_debug_code() && use_real_aborts()) {
1449 // Arm64 requires the stack pointer to be 16-byte aligned prior to address
1450 // calculation.
1451 UseScratchRegisterScope scope(this);
1452 Register temp = scope.AcquireX();
1453 Mov(temp, sp);
1454 Tst(temp, 15);
1455 Check(eq, AbortReason::kUnexpectedStackPointer);
1456 }
1457 }
1458
CopySlots(int dst,Register src,Register slot_count)1459 void TurboAssembler::CopySlots(int dst, Register src, Register slot_count) {
1460 DCHECK(!src.IsZero());
1461 UseScratchRegisterScope scope(this);
1462 Register dst_reg = scope.AcquireX();
1463 SlotAddress(dst_reg, dst);
1464 SlotAddress(src, src);
1465 CopyDoubleWords(dst_reg, src, slot_count);
1466 }
1467
CopySlots(Register dst,Register src,Register slot_count)1468 void TurboAssembler::CopySlots(Register dst, Register src,
1469 Register slot_count) {
1470 DCHECK(!dst.IsZero() && !src.IsZero());
1471 SlotAddress(dst, dst);
1472 SlotAddress(src, src);
1473 CopyDoubleWords(dst, src, slot_count);
1474 }
1475
CopyDoubleWords(Register dst,Register src,Register count,CopyDoubleWordsMode mode)1476 void TurboAssembler::CopyDoubleWords(Register dst, Register src, Register count,
1477 CopyDoubleWordsMode mode) {
1478 DCHECK(!AreAliased(dst, src, count));
1479
1480 if (emit_debug_code()) {
1481 Register pointer1 = dst;
1482 Register pointer2 = src;
1483 if (mode == kSrcLessThanDst) {
1484 pointer1 = src;
1485 pointer2 = dst;
1486 }
1487 // Copy requires pointer1 < pointer2 || (pointer1 - pointer2) >= count.
1488 Label pointer1_below_pointer2;
1489 Subs(pointer1, pointer1, pointer2);
1490 B(lt, &pointer1_below_pointer2);
1491 Cmp(pointer1, count);
1492 Check(ge, AbortReason::kOffsetOutOfRange);
1493 Bind(&pointer1_below_pointer2);
1494 Add(pointer1, pointer1, pointer2);
1495 }
1496 static_assert(kPointerSize == kDRegSize,
1497 "pointers must be the same size as doubles");
1498
1499 int direction = (mode == kDstLessThanSrc) ? 1 : -1;
1500 UseScratchRegisterScope scope(this);
1501 VRegister temp0 = scope.AcquireD();
1502 VRegister temp1 = scope.AcquireD();
1503
1504 Label pairs, loop, done;
1505
1506 Tbz(count, 0, &pairs);
1507 Ldr(temp0, MemOperand(src, direction * kPointerSize, PostIndex));
1508 Sub(count, count, 1);
1509 Str(temp0, MemOperand(dst, direction * kPointerSize, PostIndex));
1510
1511 Bind(&pairs);
1512 if (mode == kSrcLessThanDst) {
1513 // Adjust pointers for post-index ldp/stp with negative offset:
1514 Sub(dst, dst, kPointerSize);
1515 Sub(src, src, kPointerSize);
1516 }
1517 Bind(&loop);
1518 Cbz(count, &done);
1519 Ldp(temp0, temp1, MemOperand(src, 2 * direction * kPointerSize, PostIndex));
1520 Sub(count, count, 2);
1521 Stp(temp0, temp1, MemOperand(dst, 2 * direction * kPointerSize, PostIndex));
1522 B(&loop);
1523
1524 // TODO(all): large copies may benefit from using temporary Q registers
1525 // to copy four double words per iteration.
1526
1527 Bind(&done);
1528 }
1529
SlotAddress(Register dst,int slot_offset)1530 void TurboAssembler::SlotAddress(Register dst, int slot_offset) {
1531 Add(dst, sp, slot_offset << kPointerSizeLog2);
1532 }
1533
SlotAddress(Register dst,Register slot_offset)1534 void TurboAssembler::SlotAddress(Register dst, Register slot_offset) {
1535 Add(dst, sp, Operand(slot_offset, LSL, kPointerSizeLog2));
1536 }
1537
AssertFPCRState(Register fpcr)1538 void TurboAssembler::AssertFPCRState(Register fpcr) {
1539 if (emit_debug_code()) {
1540 Label unexpected_mode, done;
1541 UseScratchRegisterScope temps(this);
1542 if (fpcr.IsNone()) {
1543 fpcr = temps.AcquireX();
1544 Mrs(fpcr, FPCR);
1545 }
1546
1547 // Settings left to their default values:
1548 // - Assert that flush-to-zero is not set.
1549 Tbnz(fpcr, FZ_offset, &unexpected_mode);
1550 // - Assert that the rounding mode is nearest-with-ties-to-even.
1551 STATIC_ASSERT(FPTieEven == 0);
1552 Tst(fpcr, RMode_mask);
1553 B(eq, &done);
1554
1555 Bind(&unexpected_mode);
1556 Abort(AbortReason::kUnexpectedFPCRMode);
1557
1558 Bind(&done);
1559 }
1560 }
1561
CanonicalizeNaN(const VRegister & dst,const VRegister & src)1562 void TurboAssembler::CanonicalizeNaN(const VRegister& dst,
1563 const VRegister& src) {
1564 AssertFPCRState();
1565
1566 // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
1567 // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
1568 // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
1569 Fsub(dst, src, fp_zero);
1570 }
1571
LoadRoot(CPURegister destination,Heap::RootListIndex index)1572 void TurboAssembler::LoadRoot(CPURegister destination,
1573 Heap::RootListIndex index) {
1574 // TODO(jbramley): Most root values are constants, and can be synthesized
1575 // without a load. Refer to the ARM back end for details.
1576 Ldr(destination, MemOperand(kRootRegister, index << kPointerSizeLog2));
1577 }
1578
1579
LoadObject(Register result,Handle<Object> object)1580 void MacroAssembler::LoadObject(Register result, Handle<Object> object) {
1581 AllowDeferredHandleDereference heap_object_check;
1582 if (object->IsHeapObject()) {
1583 Move(result, Handle<HeapObject>::cast(object));
1584 } else {
1585 Mov(result, Operand(Smi::cast(*object)));
1586 }
1587 }
1588
Move(Register dst,Register src)1589 void TurboAssembler::Move(Register dst, Register src) { Mov(dst, src); }
1590
Move(Register dst,Handle<HeapObject> x)1591 void TurboAssembler::Move(Register dst, Handle<HeapObject> x) {
1592 #ifdef V8_EMBEDDED_BUILTINS
1593 if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
1594 LookupConstant(dst, x);
1595 return;
1596 }
1597 #endif // V8_EMBEDDED_BUILTINS
1598 Mov(dst, x);
1599 }
1600
Move(Register dst,Smi * src)1601 void TurboAssembler::Move(Register dst, Smi* src) { Mov(dst, src); }
1602
Swap(Register lhs,Register rhs)1603 void TurboAssembler::Swap(Register lhs, Register rhs) {
1604 DCHECK(lhs.IsSameSizeAndType(rhs));
1605 DCHECK(!lhs.Is(rhs));
1606 UseScratchRegisterScope temps(this);
1607 Register temp = temps.AcquireX();
1608 Mov(temp, rhs);
1609 Mov(rhs, lhs);
1610 Mov(lhs, temp);
1611 }
1612
Swap(VRegister lhs,VRegister rhs)1613 void TurboAssembler::Swap(VRegister lhs, VRegister rhs) {
1614 DCHECK(lhs.IsSameSizeAndType(rhs));
1615 DCHECK(!lhs.Is(rhs));
1616 UseScratchRegisterScope temps(this);
1617 VRegister temp = VRegister::no_reg();
1618 if (lhs.IsS()) {
1619 temp = temps.AcquireS();
1620 } else if (lhs.IsD()) {
1621 temp = temps.AcquireD();
1622 } else {
1623 DCHECK(lhs.IsQ());
1624 temp = temps.AcquireQ();
1625 }
1626 Mov(temp, rhs);
1627 Mov(rhs, lhs);
1628 Mov(lhs, temp);
1629 }
1630
AssertSmi(Register object,AbortReason reason)1631 void TurboAssembler::AssertSmi(Register object, AbortReason reason) {
1632 if (emit_debug_code()) {
1633 STATIC_ASSERT(kSmiTag == 0);
1634 Tst(object, kSmiTagMask);
1635 Check(eq, reason);
1636 }
1637 }
1638
AssertNotSmi(Register object,AbortReason reason)1639 void MacroAssembler::AssertNotSmi(Register object, AbortReason reason) {
1640 if (emit_debug_code()) {
1641 STATIC_ASSERT(kSmiTag == 0);
1642 Tst(object, kSmiTagMask);
1643 Check(ne, reason);
1644 }
1645 }
1646
AssertFixedArray(Register object)1647 void MacroAssembler::AssertFixedArray(Register object) {
1648 if (emit_debug_code()) {
1649 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFixedArray);
1650
1651 UseScratchRegisterScope temps(this);
1652 Register temp = temps.AcquireX();
1653
1654 CompareObjectType(object, temp, temp, FIXED_ARRAY_TYPE);
1655 Check(eq, AbortReason::kOperandIsNotAFixedArray);
1656 }
1657 }
1658
AssertConstructor(Register object)1659 void MacroAssembler::AssertConstructor(Register object) {
1660 if (emit_debug_code()) {
1661 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAConstructor);
1662
1663 UseScratchRegisterScope temps(this);
1664 Register temp = temps.AcquireX();
1665
1666 Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
1667 Ldrb(temp, FieldMemOperand(temp, Map::kBitFieldOffset));
1668 Tst(temp, Operand(Map::IsConstructorBit::kMask));
1669
1670 Check(ne, AbortReason::kOperandIsNotAConstructor);
1671 }
1672 }
1673
AssertFunction(Register object)1674 void MacroAssembler::AssertFunction(Register object) {
1675 if (emit_debug_code()) {
1676 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);
1677
1678 UseScratchRegisterScope temps(this);
1679 Register temp = temps.AcquireX();
1680
1681 CompareObjectType(object, temp, temp, JS_FUNCTION_TYPE);
1682 Check(eq, AbortReason::kOperandIsNotAFunction);
1683 }
1684 }
1685
1686
AssertBoundFunction(Register object)1687 void MacroAssembler::AssertBoundFunction(Register object) {
1688 if (emit_debug_code()) {
1689 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotABoundFunction);
1690
1691 UseScratchRegisterScope temps(this);
1692 Register temp = temps.AcquireX();
1693
1694 CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
1695 Check(eq, AbortReason::kOperandIsNotABoundFunction);
1696 }
1697 }
1698
AssertGeneratorObject(Register object)1699 void MacroAssembler::AssertGeneratorObject(Register object) {
1700 if (!emit_debug_code()) return;
1701 AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
1702
1703 // Load map
1704 UseScratchRegisterScope temps(this);
1705 Register temp = temps.AcquireX();
1706 Ldr(temp, FieldMemOperand(object, HeapObject::kMapOffset));
1707
1708 Label do_check;
1709 // Load instance type and check if JSGeneratorObject
1710 CompareInstanceType(temp, temp, JS_GENERATOR_OBJECT_TYPE);
1711 B(eq, &do_check);
1712
1713 // Check if JSAsyncGeneratorObject
1714 Cmp(temp, JS_ASYNC_GENERATOR_OBJECT_TYPE);
1715
1716 bind(&do_check);
1717 // Restore generator object to register and perform assertion
1718 Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
1719 }
1720
AssertUndefinedOrAllocationSite(Register object)1721 void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
1722 if (emit_debug_code()) {
1723 UseScratchRegisterScope temps(this);
1724 Register scratch = temps.AcquireX();
1725 Label done_checking;
1726 AssertNotSmi(object);
1727 JumpIfRoot(object, Heap::kUndefinedValueRootIndex, &done_checking);
1728 Ldr(scratch, FieldMemOperand(object, HeapObject::kMapOffset));
1729 CompareRoot(scratch, Heap::kAllocationSiteMapRootIndex);
1730 Assert(eq, AbortReason::kExpectedUndefinedOrCell);
1731 Bind(&done_checking);
1732 }
1733 }
1734
AssertPositiveOrZero(Register value)1735 void TurboAssembler::AssertPositiveOrZero(Register value) {
1736 if (emit_debug_code()) {
1737 Label done;
1738 int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
1739 Tbz(value, sign_bit, &done);
1740 Abort(AbortReason::kUnexpectedNegativeValue);
1741 Bind(&done);
1742 }
1743 }
1744
CallStubDelayed(CodeStub * stub)1745 void TurboAssembler::CallStubDelayed(CodeStub* stub) {
1746 DCHECK(AllowThisStubCall(stub)); // Stub calls are not allowed in some stubs.
1747 BlockPoolsScope scope(this);
1748 #ifdef DEBUG
1749 Label start_call;
1750 Bind(&start_call);
1751 #endif
1752 Operand operand = Operand::EmbeddedCode(stub);
1753 near_call(operand.heap_object_request());
1754 #ifdef DEBUG
1755 AssertSizeOfCodeGeneratedSince(&start_call, kNearCallSize);
1756 #endif
1757 }
1758
CallStub(CodeStub * stub)1759 void MacroAssembler::CallStub(CodeStub* stub) {
1760 DCHECK(AllowThisStubCall(stub)); // Stub calls are not allowed in some stubs.
1761 Call(stub->GetCode(), RelocInfo::CODE_TARGET);
1762 }
1763
TailCallStub(CodeStub * stub)1764 void MacroAssembler::TailCallStub(CodeStub* stub) {
1765 Jump(stub->GetCode(), RelocInfo::CODE_TARGET);
1766 }
1767
CallRuntimeDelayed(Zone * zone,Runtime::FunctionId fid,SaveFPRegsMode save_doubles)1768 void TurboAssembler::CallRuntimeDelayed(Zone* zone, Runtime::FunctionId fid,
1769 SaveFPRegsMode save_doubles) {
1770 const Runtime::Function* f = Runtime::FunctionForId(fid);
1771 // TODO(1236192): Most runtime routines don't need the number of
1772 // arguments passed in because it is constant. At some point we
1773 // should remove this need and make the runtime routine entry code
1774 // smarter.
1775 Mov(x0, f->nargs);
1776 Mov(x1, ExternalReference::Create(f));
1777 Handle<Code> code =
1778 CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
1779 Call(code, RelocInfo::CODE_TARGET);
1780 }
1781
CallRuntime(const Runtime::Function * f,int num_arguments,SaveFPRegsMode save_doubles)1782 void MacroAssembler::CallRuntime(const Runtime::Function* f,
1783 int num_arguments,
1784 SaveFPRegsMode save_doubles) {
1785 // All arguments must be on the stack before this function is called.
1786 // x0 holds the return value after the call.
1787
1788 // Check that the number of arguments matches what the function expects.
1789 // If f->nargs is -1, the function can accept a variable number of arguments.
1790 CHECK(f->nargs < 0 || f->nargs == num_arguments);
1791
1792 // Place the necessary arguments.
1793 Mov(x0, num_arguments);
1794 Mov(x1, ExternalReference::Create(f));
1795
1796 Handle<Code> code =
1797 CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
1798 Call(code, RelocInfo::CODE_TARGET);
1799 }
1800
JumpToExternalReference(const ExternalReference & builtin,bool builtin_exit_frame)1801 void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
1802 bool builtin_exit_frame) {
1803 Mov(x1, builtin);
1804 Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
1805 kArgvOnStack, builtin_exit_frame);
1806 Jump(code, RelocInfo::CODE_TARGET);
1807 }
1808
JumpToInstructionStream(Address entry)1809 void MacroAssembler::JumpToInstructionStream(Address entry) {
1810 Mov(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1811 Br(kOffHeapTrampolineRegister);
1812 }
1813
TailCallRuntime(Runtime::FunctionId fid)1814 void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
1815 const Runtime::Function* function = Runtime::FunctionForId(fid);
1816 DCHECK_EQ(1, function->result_size);
1817 if (function->nargs >= 0) {
1818 // TODO(1236192): Most runtime routines don't need the number of
1819 // arguments passed in because it is constant. At some point we
1820 // should remove this need and make the runtime routine entry code
1821 // smarter.
1822 Mov(x0, function->nargs);
1823 }
1824 JumpToExternalReference(ExternalReference::Create(fid));
1825 }
1826
ActivationFrameAlignment()1827 int TurboAssembler::ActivationFrameAlignment() {
1828 #if V8_HOST_ARCH_ARM64
1829 // Running on the real platform. Use the alignment as mandated by the local
1830 // environment.
1831 // Note: This will break if we ever start generating snapshots on one ARM
1832 // platform for another ARM platform with a different alignment.
1833 return base::OS::ActivationFrameAlignment();
1834 #else // V8_HOST_ARCH_ARM64
1835 // If we are using the simulator then we should always align to the expected
1836 // alignment. As the simulator is used to generate snapshots we do not know
1837 // if the target platform will need alignment, so this is controlled from a
1838 // flag.
1839 return FLAG_sim_stack_alignment;
1840 #endif // V8_HOST_ARCH_ARM64
1841 }
1842
CallCFunction(ExternalReference function,int num_of_reg_args)1843 void TurboAssembler::CallCFunction(ExternalReference function,
1844 int num_of_reg_args) {
1845 CallCFunction(function, num_of_reg_args, 0);
1846 }
1847
CallCFunction(ExternalReference function,int num_of_reg_args,int num_of_double_args)1848 void TurboAssembler::CallCFunction(ExternalReference function,
1849 int num_of_reg_args,
1850 int num_of_double_args) {
1851 UseScratchRegisterScope temps(this);
1852 Register temp = temps.AcquireX();
1853 Mov(temp, function);
1854 CallCFunction(temp, num_of_reg_args, num_of_double_args);
1855 }
1856
1857 static const int kRegisterPassedArguments = 8;
1858
CallCFunction(Register function,int num_of_reg_args,int num_of_double_args)1859 void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
1860 int num_of_double_args) {
1861 DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
1862 DCHECK(has_frame());
1863
1864 // If we're passing doubles, we're limited to the following prototypes
1865 // (defined by ExternalReference::Type):
1866 // BUILTIN_COMPARE_CALL: int f(double, double)
1867 // BUILTIN_FP_FP_CALL: double f(double, double)
1868 // BUILTIN_FP_CALL: double f(double)
1869 // BUILTIN_FP_INT_CALL: double f(double, int)
1870 if (num_of_double_args > 0) {
1871 DCHECK_LE(num_of_reg_args, 1);
1872 DCHECK_LE(num_of_double_args + num_of_reg_args, 2);
1873 }
1874
1875 // Call directly. The function called cannot cause a GC, or allow preemption,
1876 // so the return address in the link register stays correct.
1877 Call(function);
1878
1879 if (num_of_reg_args > kRegisterPassedArguments) {
1880 // Drop the register passed arguments.
1881 int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
1882 Drop(claim_slots);
1883 }
1884 }
1885
1886 #ifdef V8_EMBEDDED_BUILTINS
LookupConstant(Register destination,Handle<Object> object)1887 void TurboAssembler::LookupConstant(Register destination,
1888 Handle<Object> object) {
1889 CHECK(isolate()->ShouldLoadConstantsFromRootList());
1890 CHECK(root_array_available_);
1891
1892 // Ensure the given object is in the builtins constants table and fetch its
1893 // index.
1894 BuiltinsConstantsTableBuilder* builder =
1895 isolate()->builtins_constants_table_builder();
1896 uint32_t index = builder->AddObject(object);
1897
1898 // TODO(jgruber): Load builtins from the builtins table.
1899 // TODO(jgruber): Ensure that code generation can recognize constant targets
1900 // in kArchCallCodeObject.
1901
1902 DCHECK(isolate()->heap()->RootCanBeTreatedAsConstant(
1903 Heap::kBuiltinsConstantsTableRootIndex));
1904
1905 LoadRoot(destination, Heap::kBuiltinsConstantsTableRootIndex);
1906 Ldr(destination, FieldMemOperand(destination, FixedArray::kHeaderSize +
1907 index * kPointerSize));
1908 }
1909
LookupExternalReference(Register destination,ExternalReference reference)1910 void TurboAssembler::LookupExternalReference(Register destination,
1911 ExternalReference reference) {
1912 CHECK(reference.address() !=
1913 ExternalReference::roots_array_start(isolate()).address());
1914 CHECK(isolate()->ShouldLoadConstantsFromRootList());
1915 CHECK(root_array_available_);
1916
1917 // Encode as an index into the external reference table stored on the isolate.
1918
1919 ExternalReferenceEncoder encoder(isolate());
1920 ExternalReferenceEncoder::Value v = encoder.Encode(reference.address());
1921 CHECK(!v.is_from_api());
1922 uint32_t index = v.index();
1923
1924 // Generate code to load from the external reference table.
1925
1926 int32_t roots_to_external_reference_offset =
1927 Heap::roots_to_external_reference_table_offset() +
1928 ExternalReferenceTable::OffsetOfEntry(index);
1929
1930 Ldr(destination,
1931 MemOperand(kRootRegister, roots_to_external_reference_offset));
1932 }
1933 #endif // V8_EMBEDDED_BUILTINS
1934
Jump(Register target,Condition cond)1935 void TurboAssembler::Jump(Register target, Condition cond) {
1936 if (cond == nv) return;
1937 Label done;
1938 if (cond != al) B(NegateCondition(cond), &done);
1939 Br(target);
1940 Bind(&done);
1941 }
1942
JumpHelper(int64_t offset,RelocInfo::Mode rmode,Condition cond)1943 void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
1944 Condition cond) {
1945 if (cond == nv) return;
1946 Label done;
1947 if (cond != al) B(NegateCondition(cond), &done);
1948 if (CanUseNearCallOrJump(rmode)) {
1949 DCHECK(IsNearCallOffset(offset));
1950 near_jump(static_cast<int>(offset), rmode);
1951 } else {
1952 UseScratchRegisterScope temps(this);
1953 Register temp = temps.AcquireX();
1954 uint64_t imm = reinterpret_cast<uint64_t>(pc_) + offset * kInstructionSize;
1955 Mov(temp, Immediate(imm, rmode));
1956 Br(temp);
1957 }
1958 Bind(&done);
1959 }
1960
1961 namespace {
1962
1963 // The calculated offset is either:
1964 // * the 'target' input unmodified if this is a WASM call, or
1965 // * the offset of the target from the current PC, in instructions, for any
1966 // other type of call.
CalculateTargetOffset(Address target,RelocInfo::Mode rmode,byte * pc)1967 static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
1968 byte* pc) {
1969 int64_t offset = static_cast<int64_t>(target);
1970 // The target of WebAssembly calls is still an index instead of an actual
1971 // address at this point, and needs to be encoded as-is.
1972 if (rmode != RelocInfo::WASM_CALL) {
1973 offset -= reinterpret_cast<int64_t>(pc);
1974 DCHECK_EQ(offset % kInstructionSize, 0);
1975 offset = offset / static_cast<int>(kInstructionSize);
1976 }
1977 return offset;
1978 }
1979 } // namespace
1980
Jump(Address target,RelocInfo::Mode rmode,Condition cond)1981 void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
1982 Condition cond) {
1983 JumpHelper(CalculateTargetOffset(target, rmode, pc_), rmode, cond);
1984 }
1985
Jump(Handle<Code> code,RelocInfo::Mode rmode,Condition cond)1986 void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
1987 Condition cond) {
1988 DCHECK(RelocInfo::IsCodeTarget(rmode));
1989 #ifdef V8_EMBEDDED_BUILTINS
1990 if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
1991 UseScratchRegisterScope temps(this);
1992 Register scratch = temps.AcquireX();
1993 LookupConstant(scratch, code);
1994 Add(scratch, scratch, Operand(Code::kHeaderSize - kHeapObjectTag));
1995 Jump(scratch, cond);
1996 return;
1997 }
1998 #endif // V8_EMBEDDED_BUILTINS
1999 if (CanUseNearCallOrJump(rmode)) {
2000 JumpHelper(static_cast<int64_t>(GetCodeTargetIndex(code)), rmode, cond);
2001 } else {
2002 Jump(code.address(), rmode, cond);
2003 }
2004 }
2005
Call(Register target)2006 void TurboAssembler::Call(Register target) {
2007 BlockPoolsScope scope(this);
2008 #ifdef DEBUG
2009 Label start_call;
2010 Bind(&start_call);
2011 #endif
2012
2013 Blr(target);
2014
2015 #ifdef DEBUG
2016 AssertSizeOfCodeGeneratedSince(&start_call, CallSize(target));
2017 #endif
2018 }
2019
2020 // TurboAssembler::CallSize is sensitive to changes in this function, as it
2021 // requires to know how many instructions are used to branch to the target.
Call(Address target,RelocInfo::Mode rmode)2022 void TurboAssembler::Call(Address target, RelocInfo::Mode rmode) {
2023 BlockPoolsScope scope(this);
2024 #ifdef DEBUG
2025 Label start_call;
2026 Bind(&start_call);
2027 #endif
2028
2029 if (CanUseNearCallOrJump(rmode)) {
2030 int64_t offset = CalculateTargetOffset(target, rmode, pc_);
2031 DCHECK(IsNearCallOffset(offset));
2032 near_call(static_cast<int>(offset), rmode);
2033 } else {
2034 IndirectCall(target, rmode);
2035 }
2036 #ifdef DEBUG
2037 AssertSizeOfCodeGeneratedSince(&start_call, CallSize(target, rmode));
2038 #endif
2039 }
2040
Call(Handle<Code> code,RelocInfo::Mode rmode)2041 void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
2042 BlockPoolsScope scope(this);
2043 #ifdef DEBUG
2044 Label start_call;
2045 Bind(&start_call);
2046 #endif
2047
2048 #ifdef V8_EMBEDDED_BUILTINS
2049 if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
2050 UseScratchRegisterScope temps(this);
2051 Register scratch = temps.AcquireX();
2052 LookupConstant(scratch, code);
2053 Add(scratch, scratch, Operand(Code::kHeaderSize - kHeapObjectTag));
2054 Call(scratch);
2055 return;
2056 }
2057 #endif // V8_EMBEDDED_BUILTINS
2058 if (CanUseNearCallOrJump(rmode)) {
2059 near_call(GetCodeTargetIndex(code), rmode);
2060 } else {
2061 IndirectCall(code.address(), rmode);
2062 }
2063
2064 #ifdef DEBUG
2065 // Check the size of the code generated.
2066 AssertSizeOfCodeGeneratedSince(&start_call, CallSize(code, rmode));
2067 #endif
2068 }
2069
Call(ExternalReference target)2070 void TurboAssembler::Call(ExternalReference target) {
2071 UseScratchRegisterScope temps(this);
2072 Register temp = temps.AcquireX();
2073 // Immediate is in charge of setting the relocation mode to
2074 // EXTERNAL_REFERENCE.
2075 Mov(temp, Immediate(target));
2076 Call(temp);
2077 }
2078
IndirectCall(Address target,RelocInfo::Mode rmode)2079 void TurboAssembler::IndirectCall(Address target, RelocInfo::Mode rmode) {
2080 UseScratchRegisterScope temps(this);
2081 Register temp = temps.AcquireX();
2082 Mov(temp, Immediate(target, rmode));
2083 Blr(temp);
2084 }
2085
IsNearCallOffset(int64_t offset)2086 bool TurboAssembler::IsNearCallOffset(int64_t offset) {
2087 return is_int26(offset);
2088 }
2089
CallForDeoptimization(Address target,RelocInfo::Mode rmode)2090 void TurboAssembler::CallForDeoptimization(Address target,
2091 RelocInfo::Mode rmode) {
2092 DCHECK_EQ(rmode, RelocInfo::RUNTIME_ENTRY);
2093
2094 BlockPoolsScope scope(this);
2095 #ifdef DEBUG
2096 Label start_call;
2097 Bind(&start_call);
2098 #endif
2099 UseScratchRegisterScope temps(this);
2100 Register temp = temps.AcquireX();
2101
2102 // Deoptimisation table entries require the call address to be in x16, in
2103 // order to compute the entry id.
2104 // TODO(all): Put the entry id back in the table now that we are using
2105 // a direct branch for the call and do not need to set up x16.
2106 DCHECK(temp.Is(x16));
2107 Mov(temp, Immediate(target, rmode));
2108
2109 int64_t offset = static_cast<int64_t>(target) -
2110 static_cast<int64_t>(isolate_data().code_range_start_);
2111 DCHECK_EQ(offset % kInstructionSize, 0);
2112 offset = offset / static_cast<int>(kInstructionSize);
2113 DCHECK(IsNearCallOffset(offset));
2114 near_call(static_cast<int>(offset), rmode);
2115
2116 #ifdef DEBUG
2117 AssertSizeOfCodeGeneratedSince(&start_call, kNearCallSize + kInstructionSize);
2118 #endif
2119 }
2120
CallSize(Register target)2121 int TurboAssembler::CallSize(Register target) {
2122 USE(target);
2123 return kInstructionSize;
2124 }
2125
CallSize(Address target,RelocInfo::Mode rmode)2126 int TurboAssembler::CallSize(Address target, RelocInfo::Mode rmode) {
2127 USE(target);
2128 return CanUseNearCallOrJump(rmode) ? kNearCallSize : kFarCallSize;
2129 }
2130
CallSize(Handle<Code> code,RelocInfo::Mode rmode)2131 int TurboAssembler::CallSize(Handle<Code> code, RelocInfo::Mode rmode) {
2132 USE(code);
2133 return CanUseNearCallOrJump(rmode) ? kNearCallSize : kFarCallSize;
2134 }
2135
TryRepresentDoubleAsInt(Register as_int,VRegister value,VRegister scratch_d,Label * on_successful_conversion,Label * on_failed_conversion)2136 void MacroAssembler::TryRepresentDoubleAsInt(Register as_int, VRegister value,
2137 VRegister scratch_d,
2138 Label* on_successful_conversion,
2139 Label* on_failed_conversion) {
2140 // Convert to an int and back again, then compare with the original value.
2141 Fcvtzs(as_int, value);
2142 Scvtf(scratch_d, as_int);
2143 Fcmp(value, scratch_d);
2144
2145 if (on_successful_conversion) {
2146 B(on_successful_conversion, eq);
2147 }
2148 if (on_failed_conversion) {
2149 B(on_failed_conversion, ne);
2150 }
2151 }
2152
PrepareForTailCall(const ParameterCount & callee_args_count,Register caller_args_count_reg,Register scratch0,Register scratch1)2153 void TurboAssembler::PrepareForTailCall(const ParameterCount& callee_args_count,
2154 Register caller_args_count_reg,
2155 Register scratch0, Register scratch1) {
2156 #if DEBUG
2157 if (callee_args_count.is_reg()) {
2158 DCHECK(!AreAliased(callee_args_count.reg(), caller_args_count_reg, scratch0,
2159 scratch1));
2160 } else {
2161 DCHECK(!AreAliased(caller_args_count_reg, scratch0, scratch1));
2162 }
2163 #endif
2164
2165 // Calculate the end of destination area where we will put the arguments
2166 // after we drop current frame. We add kPointerSize to count the receiver
2167 // argument which is not included into formal parameters count.
2168 Register dst_reg = scratch0;
2169 Add(dst_reg, fp, Operand(caller_args_count_reg, LSL, kPointerSizeLog2));
2170 Add(dst_reg, dst_reg, StandardFrameConstants::kCallerSPOffset + kPointerSize);
2171 // Round dst_reg up to a multiple of 16 bytes, so that we overwrite any
2172 // potential padding.
2173 Add(dst_reg, dst_reg, 15);
2174 Bic(dst_reg, dst_reg, 15);
2175
2176 Register src_reg = caller_args_count_reg;
2177 // Calculate the end of source area. +kPointerSize is for the receiver.
2178 if (callee_args_count.is_reg()) {
2179 Add(src_reg, sp, Operand(callee_args_count.reg(), LSL, kPointerSizeLog2));
2180 Add(src_reg, src_reg, kPointerSize);
2181 } else {
2182 Add(src_reg, sp, (callee_args_count.immediate() + 1) * kPointerSize);
2183 }
2184
2185 // Round src_reg up to a multiple of 16 bytes, so we include any potential
2186 // padding in the copy.
2187 Add(src_reg, src_reg, 15);
2188 Bic(src_reg, src_reg, 15);
2189
2190 if (FLAG_debug_code) {
2191 Cmp(src_reg, dst_reg);
2192 Check(lo, AbortReason::kStackAccessBelowStackPointer);
2193 }
2194
2195 // Restore caller's frame pointer and return address now as they will be
2196 // overwritten by the copying loop.
2197 Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
2198 Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
2199
2200 // Now copy callee arguments to the caller frame going backwards to avoid
2201 // callee arguments corruption (source and destination areas could overlap).
2202
2203 // Both src_reg and dst_reg are pointing to the word after the one to copy,
2204 // so they must be pre-decremented in the loop.
2205 Register tmp_reg = scratch1;
2206 Label loop, entry;
2207 B(&entry);
2208 bind(&loop);
2209 Ldr(tmp_reg, MemOperand(src_reg, -kPointerSize, PreIndex));
2210 Str(tmp_reg, MemOperand(dst_reg, -kPointerSize, PreIndex));
2211 bind(&entry);
2212 Cmp(sp, src_reg);
2213 B(ne, &loop);
2214
2215 // Leave current frame.
2216 Mov(sp, dst_reg);
2217 }
2218
InvokePrologue(const ParameterCount & expected,const ParameterCount & actual,Label * done,InvokeFlag flag,bool * definitely_mismatches)2219 void MacroAssembler::InvokePrologue(const ParameterCount& expected,
2220 const ParameterCount& actual, Label* done,
2221 InvokeFlag flag,
2222 bool* definitely_mismatches) {
2223 bool definitely_matches = false;
2224 *definitely_mismatches = false;
2225 Label regular_invoke;
2226
2227 // Check whether the expected and actual arguments count match. If not,
2228 // setup registers according to contract with ArgumentsAdaptorTrampoline:
2229 // x0: actual arguments count.
2230 // x1: function (passed through to callee).
2231 // x2: expected arguments count.
2232
2233 // The code below is made a lot easier because the calling code already sets
2234 // up actual and expected registers according to the contract if values are
2235 // passed in registers.
2236 DCHECK(actual.is_immediate() || actual.reg().is(x0));
2237 DCHECK(expected.is_immediate() || expected.reg().is(x2));
2238
2239 if (expected.is_immediate()) {
2240 DCHECK(actual.is_immediate());
2241 Mov(x0, actual.immediate());
2242 if (expected.immediate() == actual.immediate()) {
2243 definitely_matches = true;
2244
2245 } else {
2246 if (expected.immediate() ==
2247 SharedFunctionInfo::kDontAdaptArgumentsSentinel) {
2248 // Don't worry about adapting arguments for builtins that
2249 // don't want that done. Skip adaption code by making it look
2250 // like we have a match between expected and actual number of
2251 // arguments.
2252 definitely_matches = true;
2253 } else {
2254 *definitely_mismatches = true;
2255 // Set up x2 for the argument adaptor.
2256 Mov(x2, expected.immediate());
2257 }
2258 }
2259
2260 } else { // expected is a register.
2261 Operand actual_op = actual.is_immediate() ? Operand(actual.immediate())
2262 : Operand(actual.reg());
2263 Mov(x0, actual_op);
2264 // If actual == expected perform a regular invocation.
2265 Cmp(expected.reg(), actual_op);
2266 B(eq, ®ular_invoke);
2267 }
2268
2269 // If the argument counts may mismatch, generate a call to the argument
2270 // adaptor.
2271 if (!definitely_matches) {
2272 Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
2273 if (flag == CALL_FUNCTION) {
2274 Call(adaptor);
2275 if (!*definitely_mismatches) {
2276 // If the arg counts don't match, no extra code is emitted by
2277 // MAsm::InvokeFunctionCode and we can just fall through.
2278 B(done);
2279 }
2280 } else {
2281 Jump(adaptor, RelocInfo::CODE_TARGET);
2282 }
2283 }
2284 Bind(®ular_invoke);
2285 }
2286
CheckDebugHook(Register fun,Register new_target,const ParameterCount & expected,const ParameterCount & actual)2287 void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
2288 const ParameterCount& expected,
2289 const ParameterCount& actual) {
2290 Label skip_hook;
2291
2292 Mov(x4, ExternalReference::debug_hook_on_function_call_address(isolate()));
2293 Ldrsb(x4, MemOperand(x4));
2294 Cbz(x4, &skip_hook);
2295
2296 {
2297 // Load receiver to pass it later to DebugOnFunctionCall hook.
2298 Operand actual_op = actual.is_immediate() ? Operand(actual.immediate())
2299 : Operand(actual.reg());
2300 Mov(x4, actual_op);
2301 Ldr(x4, MemOperand(sp, x4, LSL, kPointerSizeLog2));
2302 FrameScope frame(this,
2303 has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
2304
2305 Register expected_reg = padreg;
2306 Register actual_reg = padreg;
2307 if (expected.is_reg()) expected_reg = expected.reg();
2308 if (actual.is_reg()) actual_reg = actual.reg();
2309 if (!new_target.is_valid()) new_target = padreg;
2310
2311 // Save values on stack.
2312 SmiTag(expected_reg);
2313 SmiTag(actual_reg);
2314 Push(expected_reg, actual_reg, new_target, fun);
2315 Push(fun, x4);
2316 CallRuntime(Runtime::kDebugOnFunctionCall);
2317
2318 // Restore values from stack.
2319 Pop(fun, new_target, actual_reg, expected_reg);
2320 SmiUntag(actual_reg);
2321 SmiUntag(expected_reg);
2322 }
2323 Bind(&skip_hook);
2324 }
2325
InvokeFunctionCode(Register function,Register new_target,const ParameterCount & expected,const ParameterCount & actual,InvokeFlag flag)2326 void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
2327 const ParameterCount& expected,
2328 const ParameterCount& actual,
2329 InvokeFlag flag) {
2330 // You can't call a function without a valid frame.
2331 DCHECK(flag == JUMP_FUNCTION || has_frame());
2332 DCHECK(function.is(x1));
2333 DCHECK_IMPLIES(new_target.is_valid(), new_target.is(x3));
2334
2335 // On function call, call into the debugger if necessary.
2336 CheckDebugHook(function, new_target, expected, actual);
2337
2338 // Clear the new.target register if not given.
2339 if (!new_target.is_valid()) {
2340 LoadRoot(x3, Heap::kUndefinedValueRootIndex);
2341 }
2342
2343 Label done;
2344 bool definitely_mismatches = false;
2345 InvokePrologue(expected, actual, &done, flag, &definitely_mismatches);
2346
2347 // If we are certain that actual != expected, then we know InvokePrologue will
2348 // have handled the call through the argument adaptor mechanism.
2349 // The called function expects the call kind in x5.
2350 if (!definitely_mismatches) {
2351 // We call indirectly through the code field in the function to
2352 // allow recompilation to take effect without changing any of the
2353 // call sites.
2354 Register code = kJavaScriptCallCodeStartRegister;
2355 Ldr(code, FieldMemOperand(function, JSFunction::kCodeOffset));
2356 Add(code, code, Operand(Code::kHeaderSize - kHeapObjectTag));
2357 if (flag == CALL_FUNCTION) {
2358 Call(code);
2359 } else {
2360 DCHECK(flag == JUMP_FUNCTION);
2361 Jump(code);
2362 }
2363 }
2364
2365 // Continue here if InvokePrologue does handle the invocation due to
2366 // mismatched parameter counts.
2367 Bind(&done);
2368 }
2369
InvokeFunction(Register function,Register new_target,const ParameterCount & actual,InvokeFlag flag)2370 void MacroAssembler::InvokeFunction(Register function, Register new_target,
2371 const ParameterCount& actual,
2372 InvokeFlag flag) {
2373 // You can't call a function without a valid frame.
2374 DCHECK(flag == JUMP_FUNCTION || has_frame());
2375
2376 // Contract with called JS functions requires that function is passed in x1.
2377 // (See FullCodeGenerator::Generate().)
2378 DCHECK(function.is(x1));
2379
2380 Register expected_reg = x2;
2381
2382 Ldr(cp, FieldMemOperand(function, JSFunction::kContextOffset));
2383 // The number of arguments is stored as an int32_t, and -1 is a marker
2384 // (SharedFunctionInfo::kDontAdaptArgumentsSentinel), so we need sign
2385 // extension to correctly handle it.
2386 Ldr(expected_reg, FieldMemOperand(function,
2387 JSFunction::kSharedFunctionInfoOffset));
2388 Ldrsw(expected_reg,
2389 FieldMemOperand(expected_reg,
2390 SharedFunctionInfo::kFormalParameterCountOffset));
2391
2392 ParameterCount expected(expected_reg);
2393 InvokeFunctionCode(function, new_target, expected, actual, flag);
2394 }
2395
InvokeFunction(Register function,const ParameterCount & expected,const ParameterCount & actual,InvokeFlag flag)2396 void MacroAssembler::InvokeFunction(Register function,
2397 const ParameterCount& expected,
2398 const ParameterCount& actual,
2399 InvokeFlag flag) {
2400 // You can't call a function without a valid frame.
2401 DCHECK(flag == JUMP_FUNCTION || has_frame());
2402
2403 // Contract with called JS functions requires that function is passed in x1.
2404 // (See FullCodeGenerator::Generate().)
2405 DCHECK(function.Is(x1));
2406
2407 // Set up the context.
2408 Ldr(cp, FieldMemOperand(function, JSFunction::kContextOffset));
2409
2410 InvokeFunctionCode(function, no_reg, expected, actual, flag);
2411 }
2412
TryConvertDoubleToInt64(Register result,DoubleRegister double_input,Label * done)2413 void TurboAssembler::TryConvertDoubleToInt64(Register result,
2414 DoubleRegister double_input,
2415 Label* done) {
2416 // Try to convert with an FPU convert instruction. It's trivial to compute
2417 // the modulo operation on an integer register so we convert to a 64-bit
2418 // integer.
2419 //
2420 // Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF)
2421 // when the double is out of range. NaNs and infinities will be converted to 0
2422 // (as ECMA-262 requires).
2423 Fcvtzs(result.X(), double_input);
2424
2425 // The values INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF) are not
2426 // representable using a double, so if the result is one of those then we know
2427 // that saturation occurred, and we need to manually handle the conversion.
2428 //
2429 // It is easy to detect INT64_MIN and INT64_MAX because adding or subtracting
2430 // 1 will cause signed overflow.
2431 Cmp(result.X(), 1);
2432 Ccmp(result.X(), -1, VFlag, vc);
2433
2434 B(vc, done);
2435 }
2436
TruncateDoubleToI(Isolate * isolate,Zone * zone,Register result,DoubleRegister double_input)2437 void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
2438 Register result,
2439 DoubleRegister double_input) {
2440 Label done;
2441
2442 // Try to convert the double to an int64. If successful, the bottom 32 bits
2443 // contain our truncated int32 result.
2444 TryConvertDoubleToInt64(result, double_input, &done);
2445
2446 // If we fell through then inline version didn't succeed - call stub instead.
2447 Push(lr, double_input);
2448
2449 // DoubleToI preserves any registers it needs to clobber.
2450 Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
2451 Ldr(result, MemOperand(sp, 0));
2452
2453 DCHECK_EQ(xzr.SizeInBytes(), double_input.SizeInBytes());
2454 Pop(xzr, lr); // xzr to drop the double input on the stack.
2455
2456 Bind(&done);
2457 // Keep our invariant that the upper 32 bits are zero.
2458 Uxtw(result.W(), result.W());
2459 }
2460
Prologue()2461 void TurboAssembler::Prologue() {
2462 Push(lr, fp, cp, x1);
2463 Add(fp, sp, StandardFrameConstants::kFixedFrameSizeFromFp);
2464 }
2465
EnterFrame(StackFrame::Type type)2466 void TurboAssembler::EnterFrame(StackFrame::Type type) {
2467 UseScratchRegisterScope temps(this);
2468
2469 if (type == StackFrame::INTERNAL) {
2470 Register code_reg = temps.AcquireX();
2471 Move(code_reg, CodeObject());
2472 Register type_reg = temps.AcquireX();
2473 Mov(type_reg, StackFrame::TypeToMarker(type));
2474 Push(lr, fp, type_reg, code_reg);
2475 Add(fp, sp, InternalFrameConstants::kFixedFrameSizeFromFp);
2476 // sp[4] : lr
2477 // sp[3] : fp
2478 // sp[1] : type
2479 // sp[0] : [code object]
2480 } else if (type == StackFrame::WASM_COMPILED) {
2481 Register type_reg = temps.AcquireX();
2482 Mov(type_reg, StackFrame::TypeToMarker(type));
2483 Push(lr, fp);
2484 Mov(fp, sp);
2485 Push(type_reg, padreg);
2486 // sp[3] : lr
2487 // sp[2] : fp
2488 // sp[1] : type
2489 // sp[0] : for alignment
2490 } else {
2491 DCHECK_EQ(type, StackFrame::CONSTRUCT);
2492 Register type_reg = temps.AcquireX();
2493 Mov(type_reg, StackFrame::TypeToMarker(type));
2494
2495 // Users of this frame type push a context pointer after the type field,
2496 // so do it here to keep the stack pointer aligned.
2497 Push(lr, fp, type_reg, cp);
2498
2499 // The context pointer isn't part of the fixed frame, so add an extra slot
2500 // to account for it.
2501 Add(fp, sp, TypedFrameConstants::kFixedFrameSizeFromFp + kPointerSize);
2502 // sp[3] : lr
2503 // sp[2] : fp
2504 // sp[1] : type
2505 // sp[0] : cp
2506 }
2507 }
2508
LeaveFrame(StackFrame::Type type)2509 void TurboAssembler::LeaveFrame(StackFrame::Type type) {
2510 if (type == StackFrame::WASM_COMPILED) {
2511 Mov(sp, fp);
2512 Pop(fp, lr);
2513 } else {
2514 // Drop the execution stack down to the frame pointer and restore
2515 // the caller frame pointer and return address.
2516 Mov(sp, fp);
2517 Pop(fp, lr);
2518 }
2519 }
2520
2521
ExitFramePreserveFPRegs()2522 void MacroAssembler::ExitFramePreserveFPRegs() {
2523 DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
2524 PushCPURegList(kCallerSavedV);
2525 }
2526
2527
ExitFrameRestoreFPRegs()2528 void MacroAssembler::ExitFrameRestoreFPRegs() {
2529 // Read the registers from the stack without popping them. The stack pointer
2530 // will be reset as part of the unwinding process.
2531 CPURegList saved_fp_regs = kCallerSavedV;
2532 DCHECK_EQ(saved_fp_regs.Count() % 2, 0);
2533
2534 int offset = ExitFrameConstants::kLastExitFrameField;
2535 while (!saved_fp_regs.IsEmpty()) {
2536 const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
2537 const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
2538 offset -= 2 * kDRegSize;
2539 Ldp(dst1, dst0, MemOperand(fp, offset));
2540 }
2541 }
2542
EnterExitFrame(bool save_doubles,const Register & scratch,int extra_space,StackFrame::Type frame_type)2543 void MacroAssembler::EnterExitFrame(bool save_doubles, const Register& scratch,
2544 int extra_space,
2545 StackFrame::Type frame_type) {
2546 DCHECK(frame_type == StackFrame::EXIT ||
2547 frame_type == StackFrame::BUILTIN_EXIT);
2548
2549 // Set up the new stack frame.
2550 Push(lr, fp);
2551 Mov(fp, sp);
2552 Mov(scratch, StackFrame::TypeToMarker(frame_type));
2553 Push(scratch, xzr);
2554 Move(scratch, CodeObject());
2555 Push(scratch, padreg);
2556 // fp[8]: CallerPC (lr)
2557 // fp -> fp[0]: CallerFP (old fp)
2558 // fp[-8]: STUB marker
2559 // fp[-16]: Space reserved for SPOffset.
2560 // fp[-24]: CodeObject()
2561 // sp -> fp[-32]: padding
2562 STATIC_ASSERT((2 * kPointerSize) == ExitFrameConstants::kCallerSPOffset);
2563 STATIC_ASSERT((1 * kPointerSize) == ExitFrameConstants::kCallerPCOffset);
2564 STATIC_ASSERT((0 * kPointerSize) == ExitFrameConstants::kCallerFPOffset);
2565 STATIC_ASSERT((-2 * kPointerSize) == ExitFrameConstants::kSPOffset);
2566 STATIC_ASSERT((-3 * kPointerSize) == ExitFrameConstants::kCodeOffset);
2567 STATIC_ASSERT((-4 * kPointerSize) == ExitFrameConstants::kPaddingOffset);
2568
2569 // Save the frame pointer and context pointer in the top frame.
2570 Mov(scratch,
2571 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2572 Str(fp, MemOperand(scratch));
2573 Mov(scratch,
2574 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2575 Str(cp, MemOperand(scratch));
2576
2577 STATIC_ASSERT((-4 * kPointerSize) == ExitFrameConstants::kLastExitFrameField);
2578 if (save_doubles) {
2579 ExitFramePreserveFPRegs();
2580 }
2581
2582 // Round the number of space we need to claim to a multiple of two.
2583 int slots_to_claim = RoundUp(extra_space + 1, 2);
2584
2585 // Reserve space for the return address and for user requested memory.
2586 // We do this before aligning to make sure that we end up correctly
2587 // aligned with the minimum of wasted space.
2588 Claim(slots_to_claim, kXRegSize);
2589 // fp[8]: CallerPC (lr)
2590 // fp -> fp[0]: CallerFP (old fp)
2591 // fp[-8]: STUB marker
2592 // fp[-16]: Space reserved for SPOffset.
2593 // fp[-24]: CodeObject()
2594 // fp[-24 - fp_size]: Saved doubles (if save_doubles is true).
2595 // sp[8]: Extra space reserved for caller (if extra_space != 0).
2596 // sp -> sp[0]: Space reserved for the return address.
2597
2598 // ExitFrame::GetStateForFramePointer expects to find the return address at
2599 // the memory address immediately below the pointer stored in SPOffset.
2600 // It is not safe to derive much else from SPOffset, because the size of the
2601 // padding can vary.
2602 Add(scratch, sp, kXRegSize);
2603 Str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
2604 }
2605
2606
2607 // Leave the current exit frame.
LeaveExitFrame(bool restore_doubles,const Register & scratch,const Register & scratch2)2608 void MacroAssembler::LeaveExitFrame(bool restore_doubles,
2609 const Register& scratch,
2610 const Register& scratch2) {
2611 if (restore_doubles) {
2612 ExitFrameRestoreFPRegs();
2613 }
2614
2615 // Restore the context pointer from the top frame.
2616 Mov(scratch,
2617 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2618 Ldr(cp, MemOperand(scratch));
2619
2620 if (emit_debug_code()) {
2621 // Also emit debug code to clear the cp in the top frame.
2622 Mov(scratch2, Operand(Context::kInvalidContext));
2623 Mov(scratch, ExternalReference::Create(IsolateAddressId::kContextAddress,
2624 isolate()));
2625 Str(scratch2, MemOperand(scratch));
2626 }
2627 // Clear the frame pointer from the top frame.
2628 Mov(scratch,
2629 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2630 Str(xzr, MemOperand(scratch));
2631
2632 // Pop the exit frame.
2633 // fp[8]: CallerPC (lr)
2634 // fp -> fp[0]: CallerFP (old fp)
2635 // fp[...]: The rest of the frame.
2636 Mov(sp, fp);
2637 Pop(fp, lr);
2638 }
2639
LoadWeakValue(Register out,Register in,Label * target_if_cleared)2640 void MacroAssembler::LoadWeakValue(Register out, Register in,
2641 Label* target_if_cleared) {
2642 CompareAndBranch(in, Operand(kClearedWeakHeapObject), eq, target_if_cleared);
2643
2644 and_(out, in, Operand(~kWeakHeapObjectMask));
2645 }
2646
IncrementCounter(StatsCounter * counter,int value,Register scratch1,Register scratch2)2647 void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
2648 Register scratch1, Register scratch2) {
2649 DCHECK_NE(value, 0);
2650 if (FLAG_native_code_counters && counter->Enabled()) {
2651 Mov(scratch2, ExternalReference::Create(counter));
2652 Ldr(scratch1.W(), MemOperand(scratch2));
2653 Add(scratch1.W(), scratch1.W(), value);
2654 Str(scratch1.W(), MemOperand(scratch2));
2655 }
2656 }
2657
2658
DecrementCounter(StatsCounter * counter,int value,Register scratch1,Register scratch2)2659 void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
2660 Register scratch1, Register scratch2) {
2661 IncrementCounter(counter, -value, scratch1, scratch2);
2662 }
2663
MaybeDropFrames()2664 void MacroAssembler::MaybeDropFrames() {
2665 // Check whether we need to drop frames to restart a function on the stack.
2666 Mov(x1, ExternalReference::debug_restart_fp_address(isolate()));
2667 Ldr(x1, MemOperand(x1));
2668 Tst(x1, x1);
2669 Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
2670 ne);
2671 }
2672
JumpIfObjectType(Register object,Register map,Register type_reg,InstanceType type,Label * if_cond_pass,Condition cond)2673 void MacroAssembler::JumpIfObjectType(Register object,
2674 Register map,
2675 Register type_reg,
2676 InstanceType type,
2677 Label* if_cond_pass,
2678 Condition cond) {
2679 CompareObjectType(object, map, type_reg, type);
2680 B(cond, if_cond_pass);
2681 }
2682
2683
2684 // Sets condition flags based on comparison, and returns type in type_reg.
CompareObjectType(Register object,Register map,Register type_reg,InstanceType type)2685 void MacroAssembler::CompareObjectType(Register object,
2686 Register map,
2687 Register type_reg,
2688 InstanceType type) {
2689 Ldr(map, FieldMemOperand(object, HeapObject::kMapOffset));
2690 CompareInstanceType(map, type_reg, type);
2691 }
2692
2693
2694 // Sets condition flags based on comparison, and returns type in type_reg.
CompareInstanceType(Register map,Register type_reg,InstanceType type)2695 void MacroAssembler::CompareInstanceType(Register map,
2696 Register type_reg,
2697 InstanceType type) {
2698 Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
2699 Cmp(type_reg, type);
2700 }
2701
2702
LoadElementsKindFromMap(Register result,Register map)2703 void MacroAssembler::LoadElementsKindFromMap(Register result, Register map) {
2704 // Load the map's "bit field 2".
2705 Ldrb(result, FieldMemOperand(map, Map::kBitField2Offset));
2706 // Retrieve elements_kind from bit field 2.
2707 DecodeField<Map::ElementsKindBits>(result);
2708 }
2709
CompareRoot(const Register & obj,Heap::RootListIndex index)2710 void MacroAssembler::CompareRoot(const Register& obj,
2711 Heap::RootListIndex index) {
2712 UseScratchRegisterScope temps(this);
2713 Register temp = temps.AcquireX();
2714 DCHECK(!AreAliased(obj, temp));
2715 LoadRoot(temp, index);
2716 Cmp(obj, temp);
2717 }
2718
2719
JumpIfRoot(const Register & obj,Heap::RootListIndex index,Label * if_equal)2720 void MacroAssembler::JumpIfRoot(const Register& obj,
2721 Heap::RootListIndex index,
2722 Label* if_equal) {
2723 CompareRoot(obj, index);
2724 B(eq, if_equal);
2725 }
2726
2727
JumpIfNotRoot(const Register & obj,Heap::RootListIndex index,Label * if_not_equal)2728 void MacroAssembler::JumpIfNotRoot(const Register& obj,
2729 Heap::RootListIndex index,
2730 Label* if_not_equal) {
2731 CompareRoot(obj, index);
2732 B(ne, if_not_equal);
2733 }
2734
2735
CompareAndSplit(const Register & lhs,const Operand & rhs,Condition cond,Label * if_true,Label * if_false,Label * fall_through)2736 void MacroAssembler::CompareAndSplit(const Register& lhs,
2737 const Operand& rhs,
2738 Condition cond,
2739 Label* if_true,
2740 Label* if_false,
2741 Label* fall_through) {
2742 if ((if_true == if_false) && (if_false == fall_through)) {
2743 // Fall through.
2744 } else if (if_true == if_false) {
2745 B(if_true);
2746 } else if (if_false == fall_through) {
2747 CompareAndBranch(lhs, rhs, cond, if_true);
2748 } else if (if_true == fall_through) {
2749 CompareAndBranch(lhs, rhs, NegateCondition(cond), if_false);
2750 } else {
2751 CompareAndBranch(lhs, rhs, cond, if_true);
2752 B(if_false);
2753 }
2754 }
2755
2756
TestAndSplit(const Register & reg,uint64_t bit_pattern,Label * if_all_clear,Label * if_any_set,Label * fall_through)2757 void MacroAssembler::TestAndSplit(const Register& reg,
2758 uint64_t bit_pattern,
2759 Label* if_all_clear,
2760 Label* if_any_set,
2761 Label* fall_through) {
2762 if ((if_all_clear == if_any_set) && (if_any_set == fall_through)) {
2763 // Fall through.
2764 } else if (if_all_clear == if_any_set) {
2765 B(if_all_clear);
2766 } else if (if_all_clear == fall_through) {
2767 TestAndBranchIfAnySet(reg, bit_pattern, if_any_set);
2768 } else if (if_any_set == fall_through) {
2769 TestAndBranchIfAllClear(reg, bit_pattern, if_all_clear);
2770 } else {
2771 TestAndBranchIfAnySet(reg, bit_pattern, if_any_set);
2772 B(if_all_clear);
2773 }
2774 }
2775
AllowThisStubCall(CodeStub * stub)2776 bool TurboAssembler::AllowThisStubCall(CodeStub* stub) {
2777 return has_frame() || !stub->SometimesSetsUpAFrame();
2778 }
2779
PopSafepointRegisters()2780 void MacroAssembler::PopSafepointRegisters() {
2781 const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
2782 DCHECK_GE(num_unsaved, 0);
2783 DCHECK_EQ(num_unsaved % 2, 0);
2784 DCHECK_EQ(kSafepointSavedRegisters % 2, 0);
2785 PopXRegList(kSafepointSavedRegisters);
2786 Drop(num_unsaved);
2787 }
2788
2789
PushSafepointRegisters()2790 void MacroAssembler::PushSafepointRegisters() {
2791 // Safepoints expect a block of kNumSafepointRegisters values on the stack, so
2792 // adjust the stack for unsaved registers.
2793 const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
2794 DCHECK_GE(num_unsaved, 0);
2795 DCHECK_EQ(num_unsaved % 2, 0);
2796 DCHECK_EQ(kSafepointSavedRegisters % 2, 0);
2797 Claim(num_unsaved);
2798 PushXRegList(kSafepointSavedRegisters);
2799 }
2800
SafepointRegisterStackIndex(int reg_code)2801 int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
2802 // Make sure the safepoint registers list is what we expect.
2803 DCHECK_EQ(CPURegList::GetSafepointSavedRegisters().list(), 0x6FFCFFFF);
2804
2805 // Safepoint registers are stored contiguously on the stack, but not all the
2806 // registers are saved. The following registers are excluded:
2807 // - x16 and x17 (ip0 and ip1) because they shouldn't be preserved outside of
2808 // the macro assembler.
2809 // - x31 (sp) because the system stack pointer doesn't need to be included
2810 // in safepoint registers.
2811 //
2812 // This function implements the mapping of register code to index into the
2813 // safepoint register slots.
2814 if ((reg_code >= 0) && (reg_code <= 15)) {
2815 return reg_code;
2816 } else if ((reg_code >= 18) && (reg_code <= 30)) {
2817 // Skip ip0 and ip1.
2818 return reg_code - 2;
2819 } else {
2820 // This register has no safepoint register slot.
2821 UNREACHABLE();
2822 }
2823 }
2824
CheckPageFlag(const Register & object,const Register & scratch,int mask,Condition cc,Label * condition_met)2825 void MacroAssembler::CheckPageFlag(const Register& object,
2826 const Register& scratch, int mask,
2827 Condition cc, Label* condition_met) {
2828 And(scratch, object, ~Page::kPageAlignmentMask);
2829 Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
2830 if (cc == eq) {
2831 TestAndBranchIfAnySet(scratch, mask, condition_met);
2832 } else {
2833 TestAndBranchIfAllClear(scratch, mask, condition_met);
2834 }
2835 }
2836
CheckPageFlagSet(const Register & object,const Register & scratch,int mask,Label * if_any_set)2837 void TurboAssembler::CheckPageFlagSet(const Register& object,
2838 const Register& scratch, int mask,
2839 Label* if_any_set) {
2840 And(scratch, object, ~Page::kPageAlignmentMask);
2841 Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
2842 TestAndBranchIfAnySet(scratch, mask, if_any_set);
2843 }
2844
CheckPageFlagClear(const Register & object,const Register & scratch,int mask,Label * if_all_clear)2845 void TurboAssembler::CheckPageFlagClear(const Register& object,
2846 const Register& scratch, int mask,
2847 Label* if_all_clear) {
2848 And(scratch, object, ~Page::kPageAlignmentMask);
2849 Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
2850 TestAndBranchIfAllClear(scratch, mask, if_all_clear);
2851 }
2852
RecordWriteField(Register object,int offset,Register value,Register scratch,LinkRegisterStatus lr_status,SaveFPRegsMode save_fp,RememberedSetAction remembered_set_action,SmiCheck smi_check)2853 void MacroAssembler::RecordWriteField(Register object, int offset,
2854 Register value, Register scratch,
2855 LinkRegisterStatus lr_status,
2856 SaveFPRegsMode save_fp,
2857 RememberedSetAction remembered_set_action,
2858 SmiCheck smi_check) {
2859 // First, check if a write barrier is even needed. The tests below
2860 // catch stores of Smis.
2861 Label done;
2862
2863 // Skip the barrier if writing a smi.
2864 if (smi_check == INLINE_SMI_CHECK) {
2865 JumpIfSmi(value, &done);
2866 }
2867
2868 // Although the object register is tagged, the offset is relative to the start
2869 // of the object, so offset must be a multiple of kPointerSize.
2870 DCHECK(IsAligned(offset, kPointerSize));
2871
2872 Add(scratch, object, offset - kHeapObjectTag);
2873 if (emit_debug_code()) {
2874 Label ok;
2875 Tst(scratch, kPointerSize - 1);
2876 B(eq, &ok);
2877 Abort(AbortReason::kUnalignedCellInWriteBarrier);
2878 Bind(&ok);
2879 }
2880
2881 RecordWrite(object, scratch, value, lr_status, save_fp, remembered_set_action,
2882 OMIT_SMI_CHECK);
2883
2884 Bind(&done);
2885
2886 // Clobber clobbered input registers when running with the debug-code flag
2887 // turned on to provoke errors.
2888 if (emit_debug_code()) {
2889 Mov(value, Operand(bit_cast<int64_t>(kZapValue + 4)));
2890 Mov(scratch, Operand(bit_cast<int64_t>(kZapValue + 8)));
2891 }
2892 }
2893
SaveRegisters(RegList registers)2894 void TurboAssembler::SaveRegisters(RegList registers) {
2895 DCHECK_GT(NumRegs(registers), 0);
2896 CPURegList regs(lr);
2897 for (int i = 0; i < Register::kNumRegisters; ++i) {
2898 if ((registers >> i) & 1u) {
2899 regs.Combine(Register::XRegFromCode(i));
2900 }
2901 }
2902
2903 PushCPURegList(regs);
2904 }
2905
RestoreRegisters(RegList registers)2906 void TurboAssembler::RestoreRegisters(RegList registers) {
2907 DCHECK_GT(NumRegs(registers), 0);
2908 CPURegList regs(lr);
2909 for (int i = 0; i < Register::kNumRegisters; ++i) {
2910 if ((registers >> i) & 1u) {
2911 regs.Combine(Register::XRegFromCode(i));
2912 }
2913 }
2914
2915 PopCPURegList(regs);
2916 }
2917
CallRecordWriteStub(Register object,Register address,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode)2918 void TurboAssembler::CallRecordWriteStub(
2919 Register object, Register address,
2920 RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
2921 // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
2922 // i.e. always emit remember set and save FP registers in RecordWriteStub. If
2923 // large performance regression is observed, we should use these values to
2924 // avoid unnecessary work.
2925
2926 Callable const callable =
2927 Builtins::CallableFor(isolate(), Builtins::kRecordWrite);
2928 RegList registers = callable.descriptor().allocatable_registers();
2929
2930 SaveRegisters(registers);
2931
2932 Register object_parameter(callable.descriptor().GetRegisterParameter(
2933 RecordWriteDescriptor::kObject));
2934 Register slot_parameter(
2935 callable.descriptor().GetRegisterParameter(RecordWriteDescriptor::kSlot));
2936 Register isolate_parameter(callable.descriptor().GetRegisterParameter(
2937 RecordWriteDescriptor::kIsolate));
2938 Register remembered_set_parameter(callable.descriptor().GetRegisterParameter(
2939 RecordWriteDescriptor::kRememberedSet));
2940 Register fp_mode_parameter(callable.descriptor().GetRegisterParameter(
2941 RecordWriteDescriptor::kFPMode));
2942
2943 Push(object, address);
2944
2945 Pop(slot_parameter, object_parameter);
2946
2947 Mov(isolate_parameter, ExternalReference::isolate_address(isolate()));
2948 Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
2949 Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
2950 Call(callable.code(), RelocInfo::CODE_TARGET);
2951
2952 RestoreRegisters(registers);
2953 }
2954
2955 // Will clobber: object, address, value.
2956 // If lr_status is kLRHasBeenSaved, lr will also be clobbered.
2957 //
2958 // The register 'object' contains a heap object pointer. The heap object tag is
2959 // shifted away.
RecordWrite(Register object,Register address,Register value,LinkRegisterStatus lr_status,SaveFPRegsMode fp_mode,RememberedSetAction remembered_set_action,SmiCheck smi_check)2960 void MacroAssembler::RecordWrite(Register object, Register address,
2961 Register value, LinkRegisterStatus lr_status,
2962 SaveFPRegsMode fp_mode,
2963 RememberedSetAction remembered_set_action,
2964 SmiCheck smi_check) {
2965 ASM_LOCATION_IN_ASSEMBLER("MacroAssembler::RecordWrite");
2966 DCHECK(!AreAliased(object, value));
2967
2968 if (emit_debug_code()) {
2969 UseScratchRegisterScope temps(this);
2970 Register temp = temps.AcquireX();
2971
2972 Ldr(temp, MemOperand(address));
2973 Cmp(temp, value);
2974 Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
2975 }
2976
2977 // First, check if a write barrier is even needed. The tests below
2978 // catch stores of smis and stores into the young generation.
2979 Label done;
2980
2981 if (smi_check == INLINE_SMI_CHECK) {
2982 DCHECK_EQ(0, kSmiTag);
2983 JumpIfSmi(value, &done);
2984 }
2985
2986 CheckPageFlagClear(value,
2987 value, // Used as scratch.
2988 MemoryChunk::kPointersToHereAreInterestingMask, &done);
2989 CheckPageFlagClear(object,
2990 value, // Used as scratch.
2991 MemoryChunk::kPointersFromHereAreInterestingMask,
2992 &done);
2993
2994 // Record the actual write.
2995 if (lr_status == kLRHasNotBeenSaved) {
2996 Push(padreg, lr);
2997 }
2998 CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
2999 if (lr_status == kLRHasNotBeenSaved) {
3000 Pop(lr, padreg);
3001 }
3002
3003 Bind(&done);
3004
3005 // Count number of write barriers in generated code.
3006 isolate()->counters()->write_barriers_static()->Increment();
3007 IncrementCounter(isolate()->counters()->write_barriers_dynamic(), 1, address,
3008 value);
3009
3010 // Clobber clobbered registers when running with the debug-code flag
3011 // turned on to provoke errors.
3012 if (emit_debug_code()) {
3013 Mov(address, Operand(bit_cast<int64_t>(kZapValue + 12)));
3014 Mov(value, Operand(bit_cast<int64_t>(kZapValue + 16)));
3015 }
3016 }
3017
Assert(Condition cond,AbortReason reason)3018 void TurboAssembler::Assert(Condition cond, AbortReason reason) {
3019 if (emit_debug_code()) {
3020 Check(cond, reason);
3021 }
3022 }
3023
AssertUnreachable(AbortReason reason)3024 void TurboAssembler::AssertUnreachable(AbortReason reason) {
3025 if (emit_debug_code()) Abort(reason);
3026 }
3027
AssertRegisterIsRoot(Register reg,Heap::RootListIndex index,AbortReason reason)3028 void MacroAssembler::AssertRegisterIsRoot(Register reg,
3029 Heap::RootListIndex index,
3030 AbortReason reason) {
3031 if (emit_debug_code()) {
3032 CompareRoot(reg, index);
3033 Check(eq, reason);
3034 }
3035 }
3036
Check(Condition cond,AbortReason reason)3037 void TurboAssembler::Check(Condition cond, AbortReason reason) {
3038 Label ok;
3039 B(cond, &ok);
3040 Abort(reason);
3041 // Will not return here.
3042 Bind(&ok);
3043 }
3044
Abort(AbortReason reason)3045 void TurboAssembler::Abort(AbortReason reason) {
3046 #ifdef DEBUG
3047 RecordComment("Abort message: ");
3048 RecordComment(GetAbortReason(reason));
3049
3050 if (FLAG_trap_on_abort) {
3051 Brk(0);
3052 return;
3053 }
3054 #endif
3055
3056 // We need some scratch registers for the MacroAssembler, so make sure we have
3057 // some. This is safe here because Abort never returns.
3058 RegList old_tmp_list = TmpList()->list();
3059 TmpList()->Combine(MacroAssembler::DefaultTmpList());
3060
3061 if (use_real_aborts()) {
3062 // Avoid infinite recursion; Push contains some assertions that use Abort.
3063 NoUseRealAbortsScope no_real_aborts(this);
3064
3065 Move(x1, Smi::FromInt(static_cast<int>(reason)));
3066
3067 if (!has_frame_) {
3068 // We don't actually want to generate a pile of code for this, so just
3069 // claim there is a stack frame, without generating one.
3070 FrameScope scope(this, StackFrame::NONE);
3071 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3072 } else {
3073 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3074 }
3075 } else {
3076 // Load the string to pass to Printf.
3077 Label msg_address;
3078 Adr(x0, &msg_address);
3079
3080 // Call Printf directly to report the error.
3081 CallPrintf();
3082
3083 // We need a way to stop execution on both the simulator and real hardware,
3084 // and Unreachable() is the best option.
3085 Unreachable();
3086
3087 // Emit the message string directly in the instruction stream.
3088 {
3089 BlockPoolsScope scope(this);
3090 Bind(&msg_address);
3091 EmitStringData(GetAbortReason(reason));
3092 }
3093 }
3094
3095 TmpList()->set_list(old_tmp_list);
3096 }
3097
LoadNativeContextSlot(int index,Register dst)3098 void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
3099 Ldr(dst, NativeContextMemOperand());
3100 Ldr(dst, ContextMemOperand(dst, index));
3101 }
3102
3103
3104 // This is the main Printf implementation. All other Printf variants call
3105 // PrintfNoPreserve after setting up one or more PreserveRegisterScopes.
PrintfNoPreserve(const char * format,const CPURegister & arg0,const CPURegister & arg1,const CPURegister & arg2,const CPURegister & arg3)3106 void MacroAssembler::PrintfNoPreserve(const char * format,
3107 const CPURegister& arg0,
3108 const CPURegister& arg1,
3109 const CPURegister& arg2,
3110 const CPURegister& arg3) {
3111 // We cannot handle a caller-saved stack pointer. It doesn't make much sense
3112 // in most cases anyway, so this restriction shouldn't be too serious.
3113 DCHECK(!kCallerSaved.IncludesAliasOf(sp));
3114
3115 // The provided arguments, and their proper procedure-call standard registers.
3116 CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
3117 CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};
3118
3119 int arg_count = kPrintfMaxArgCount;
3120
3121 // The PCS varargs registers for printf. Note that x0 is used for the printf
3122 // format string.
3123 static const CPURegList kPCSVarargs =
3124 CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
3125 static const CPURegList kPCSVarargsFP =
3126 CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
3127
3128 // We can use caller-saved registers as scratch values, except for the
3129 // arguments and the PCS registers where they might need to go.
3130 CPURegList tmp_list = kCallerSaved;
3131 tmp_list.Remove(x0); // Used to pass the format string.
3132 tmp_list.Remove(kPCSVarargs);
3133 tmp_list.Remove(arg0, arg1, arg2, arg3);
3134
3135 CPURegList fp_tmp_list = kCallerSavedV;
3136 fp_tmp_list.Remove(kPCSVarargsFP);
3137 fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
3138
3139 // Override the MacroAssembler's scratch register list. The lists will be
3140 // reset automatically at the end of the UseScratchRegisterScope.
3141 UseScratchRegisterScope temps(this);
3142 TmpList()->set_list(tmp_list.list());
3143 FPTmpList()->set_list(fp_tmp_list.list());
3144
3145 // Copies of the printf vararg registers that we can pop from.
3146 CPURegList pcs_varargs = kPCSVarargs;
3147 CPURegList pcs_varargs_fp = kPCSVarargsFP;
3148
3149 // Place the arguments. There are lots of clever tricks and optimizations we
3150 // could use here, but Printf is a debug tool so instead we just try to keep
3151 // it simple: Move each input that isn't already in the right place to a
3152 // scratch register, then move everything back.
3153 for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
3154 // Work out the proper PCS register for this argument.
3155 if (args[i].IsRegister()) {
3156 pcs[i] = pcs_varargs.PopLowestIndex().X();
3157 // We might only need a W register here. We need to know the size of the
3158 // argument so we can properly encode it for the simulator call.
3159 if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
3160 } else if (args[i].IsVRegister()) {
3161 // In C, floats are always cast to doubles for varargs calls.
3162 pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
3163 } else {
3164 DCHECK(args[i].IsNone());
3165 arg_count = i;
3166 break;
3167 }
3168
3169 // If the argument is already in the right place, leave it where it is.
3170 if (args[i].Aliases(pcs[i])) continue;
3171
3172 // Otherwise, if the argument is in a PCS argument register, allocate an
3173 // appropriate scratch register and then move it out of the way.
3174 if (kPCSVarargs.IncludesAliasOf(args[i]) ||
3175 kPCSVarargsFP.IncludesAliasOf(args[i])) {
3176 if (args[i].IsRegister()) {
3177 Register old_arg = args[i].Reg();
3178 Register new_arg = temps.AcquireSameSizeAs(old_arg);
3179 Mov(new_arg, old_arg);
3180 args[i] = new_arg;
3181 } else {
3182 VRegister old_arg = args[i].VReg();
3183 VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
3184 Fmov(new_arg, old_arg);
3185 args[i] = new_arg;
3186 }
3187 }
3188 }
3189
3190 // Do a second pass to move values into their final positions and perform any
3191 // conversions that may be required.
3192 for (int i = 0; i < arg_count; i++) {
3193 DCHECK(pcs[i].type() == args[i].type());
3194 if (pcs[i].IsRegister()) {
3195 Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
3196 } else {
3197 DCHECK(pcs[i].IsVRegister());
3198 if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
3199 Fmov(pcs[i].VReg(), args[i].VReg());
3200 } else {
3201 Fcvt(pcs[i].VReg(), args[i].VReg());
3202 }
3203 }
3204 }
3205
3206 // Load the format string into x0, as per the procedure-call standard.
3207 //
3208 // To make the code as portable as possible, the format string is encoded
3209 // directly in the instruction stream. It might be cleaner to encode it in a
3210 // literal pool, but since Printf is usually used for debugging, it is
3211 // beneficial for it to be minimally dependent on other features.
3212 Label format_address;
3213 Adr(x0, &format_address);
3214
3215 // Emit the format string directly in the instruction stream.
3216 { BlockPoolsScope scope(this);
3217 Label after_data;
3218 B(&after_data);
3219 Bind(&format_address);
3220 EmitStringData(format);
3221 Unreachable();
3222 Bind(&after_data);
3223 }
3224
3225 CallPrintf(arg_count, pcs);
3226 }
3227
CallPrintf(int arg_count,const CPURegister * args)3228 void TurboAssembler::CallPrintf(int arg_count, const CPURegister* args) {
3229 // A call to printf needs special handling for the simulator, since the system
3230 // printf function will use a different instruction set and the procedure-call
3231 // standard will not be compatible.
3232 #ifdef USE_SIMULATOR
3233 { InstructionAccurateScope scope(this, kPrintfLength / kInstructionSize);
3234 hlt(kImmExceptionIsPrintf);
3235 dc32(arg_count); // kPrintfArgCountOffset
3236
3237 // Determine the argument pattern.
3238 uint32_t arg_pattern_list = 0;
3239 for (int i = 0; i < arg_count; i++) {
3240 uint32_t arg_pattern;
3241 if (args[i].IsRegister()) {
3242 arg_pattern = args[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
3243 } else {
3244 DCHECK(args[i].Is64Bits());
3245 arg_pattern = kPrintfArgD;
3246 }
3247 DCHECK(arg_pattern < (1 << kPrintfArgPatternBits));
3248 arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
3249 }
3250 dc32(arg_pattern_list); // kPrintfArgPatternListOffset
3251 }
3252 #else
3253 Call(ExternalReference::printf_function());
3254 #endif
3255 }
3256
3257
Printf(const char * format,CPURegister arg0,CPURegister arg1,CPURegister arg2,CPURegister arg3)3258 void MacroAssembler::Printf(const char * format,
3259 CPURegister arg0,
3260 CPURegister arg1,
3261 CPURegister arg2,
3262 CPURegister arg3) {
3263 // Printf is expected to preserve all registers, so make sure that none are
3264 // available as scratch registers until we've preserved them.
3265 RegList old_tmp_list = TmpList()->list();
3266 RegList old_fp_tmp_list = FPTmpList()->list();
3267 TmpList()->set_list(0);
3268 FPTmpList()->set_list(0);
3269
3270 // Preserve all caller-saved registers as well as NZCV.
3271 // PushCPURegList asserts that the size of each list is a multiple of 16
3272 // bytes.
3273 PushCPURegList(kCallerSaved);
3274 PushCPURegList(kCallerSavedV);
3275
3276 // We can use caller-saved registers as scratch values (except for argN).
3277 CPURegList tmp_list = kCallerSaved;
3278 CPURegList fp_tmp_list = kCallerSavedV;
3279 tmp_list.Remove(arg0, arg1, arg2, arg3);
3280 fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
3281 TmpList()->set_list(tmp_list.list());
3282 FPTmpList()->set_list(fp_tmp_list.list());
3283
3284 { UseScratchRegisterScope temps(this);
3285 // If any of the arguments are the current stack pointer, allocate a new
3286 // register for them, and adjust the value to compensate for pushing the
3287 // caller-saved registers.
3288 bool arg0_sp = sp.Aliases(arg0);
3289 bool arg1_sp = sp.Aliases(arg1);
3290 bool arg2_sp = sp.Aliases(arg2);
3291 bool arg3_sp = sp.Aliases(arg3);
3292 if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
3293 // Allocate a register to hold the original stack pointer value, to pass
3294 // to PrintfNoPreserve as an argument.
3295 Register arg_sp = temps.AcquireX();
3296 Add(arg_sp, sp,
3297 kCallerSaved.TotalSizeInBytes() + kCallerSavedV.TotalSizeInBytes());
3298 if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
3299 if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
3300 if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
3301 if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
3302 }
3303
3304 // Preserve NZCV.
3305 { UseScratchRegisterScope temps(this);
3306 Register tmp = temps.AcquireX();
3307 Mrs(tmp, NZCV);
3308 Push(tmp, xzr);
3309 }
3310
3311 PrintfNoPreserve(format, arg0, arg1, arg2, arg3);
3312
3313 // Restore NZCV.
3314 { UseScratchRegisterScope temps(this);
3315 Register tmp = temps.AcquireX();
3316 Pop(xzr, tmp);
3317 Msr(NZCV, tmp);
3318 }
3319 }
3320
3321 PopCPURegList(kCallerSavedV);
3322 PopCPURegList(kCallerSaved);
3323
3324 TmpList()->set_list(old_tmp_list);
3325 FPTmpList()->set_list(old_fp_tmp_list);
3326 }
3327
~UseScratchRegisterScope()3328 UseScratchRegisterScope::~UseScratchRegisterScope() {
3329 available_->set_list(old_available_);
3330 availablefp_->set_list(old_availablefp_);
3331 }
3332
3333
AcquireSameSizeAs(const Register & reg)3334 Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
3335 int code = AcquireNextAvailable(available_).code();
3336 return Register::Create(code, reg.SizeInBits());
3337 }
3338
AcquireSameSizeAs(const VRegister & reg)3339 VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
3340 int code = AcquireNextAvailable(availablefp_).code();
3341 return VRegister::Create(code, reg.SizeInBits());
3342 }
3343
3344
AcquireNextAvailable(CPURegList * available)3345 CPURegister UseScratchRegisterScope::AcquireNextAvailable(
3346 CPURegList* available) {
3347 CHECK(!available->IsEmpty());
3348 CPURegister result = available->PopLowestIndex();
3349 DCHECK(!AreAliased(result, xzr, sp));
3350 return result;
3351 }
3352
3353
ContextMemOperand(Register context,int index)3354 MemOperand ContextMemOperand(Register context, int index) {
3355 return MemOperand(context, Context::SlotOffset(index));
3356 }
3357
NativeContextMemOperand()3358 MemOperand NativeContextMemOperand() {
3359 return ContextMemOperand(cp, Context::NATIVE_CONTEXT_INDEX);
3360 }
3361
3362 #define __ masm->
3363
Emit(MacroAssembler * masm,const Register & reg,const Label * smi_check)3364 void InlineSmiCheckInfo::Emit(MacroAssembler* masm, const Register& reg,
3365 const Label* smi_check) {
3366 Assembler::BlockPoolsScope scope(masm);
3367 if (reg.IsValid()) {
3368 DCHECK(smi_check->is_bound());
3369 DCHECK(reg.Is64Bits());
3370
3371 // Encode the register (x0-x30) in the lowest 5 bits, then the offset to
3372 // 'check' in the other bits. The possible offset is limited in that we
3373 // use BitField to pack the data, and the underlying data type is a
3374 // uint32_t.
3375 uint32_t delta =
3376 static_cast<uint32_t>(__ InstructionsGeneratedSince(smi_check));
3377 __ InlineData(RegisterBits::encode(reg.code()) | DeltaBits::encode(delta));
3378 } else {
3379 DCHECK(!smi_check->is_bound());
3380
3381 // An offset of 0 indicates that there is no patch site.
3382 __ InlineData(0);
3383 }
3384 }
3385
InlineSmiCheckInfo(Address info)3386 InlineSmiCheckInfo::InlineSmiCheckInfo(Address info)
3387 : reg_(NoReg), smi_check_delta_(0), smi_check_(nullptr) {
3388 InstructionSequence* inline_data = InstructionSequence::At(info);
3389 DCHECK(inline_data->IsInlineData());
3390 if (inline_data->IsInlineData()) {
3391 uint64_t payload = inline_data->InlineData();
3392 // We use BitField to decode the payload, and BitField can only handle
3393 // 32-bit values.
3394 DCHECK(is_uint32(payload));
3395 if (payload != 0) {
3396 uint32_t payload32 = static_cast<uint32_t>(payload);
3397 int reg_code = RegisterBits::decode(payload32);
3398 reg_ = Register::XRegFromCode(reg_code);
3399 smi_check_delta_ = DeltaBits::decode(payload32);
3400 DCHECK_NE(0, smi_check_delta_);
3401 smi_check_ = inline_data->preceding(smi_check_delta_);
3402 }
3403 }
3404 }
3405
ComputeCodeStartAddress(const Register & rd)3406 void TurboAssembler::ComputeCodeStartAddress(const Register& rd) {
3407 // We can use adr to load a pc relative location.
3408 adr(rd, -pc_offset());
3409 }
3410
ResetSpeculationPoisonRegister()3411 void TurboAssembler::ResetSpeculationPoisonRegister() {
3412 Mov(kSpeculationPoisonRegister, -1);
3413 }
3414
3415 #undef __
3416
3417
3418 } // namespace internal
3419 } // namespace v8
3420
3421 #endif // V8_TARGET_ARCH_ARM64
3422