1 // Copyright 2015, ARM Limited
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are met:
6 //
7 // * Redistributions of source code must retain the above copyright notice,
8 // this list of conditions and the following disclaimer.
9 // * Redistributions in binary form must reproduce the above copyright notice,
10 // this list of conditions and the following disclaimer in the documentation
11 // and/or other materials provided with the distribution.
12 // * Neither the name of ARM Limited nor the names of its contributors may be
13 // used to endorse or promote products derived from this software without
14 // specific prior written permission.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
17 // ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
20 // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 #include "jit/arm64/vixl/MacroAssembler-vixl.h"
28
29 #include <ctype.h>
30
31 namespace vixl {
32
MacroAssembler()33 MacroAssembler::MacroAssembler()
34 : js::jit::Assembler(),
35 sp_(x28),
36 tmp_list_(ip0, ip1),
37 fptmp_list_(d31)
38 {
39 }
40
41
FinalizeCode()42 void MacroAssembler::FinalizeCode() {
43 Assembler::FinalizeCode();
44 }
45
46
MoveImmediateHelper(MacroAssembler * masm,const Register & rd,uint64_t imm)47 int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm,
48 const Register &rd,
49 uint64_t imm) {
50 bool emit_code = (masm != NULL);
51 VIXL_ASSERT(IsUint32(imm) || IsInt32(imm) || rd.Is64Bits());
52 // The worst case for size is mov 64-bit immediate to sp:
53 // * up to 4 instructions to materialise the constant
54 // * 1 instruction to move to sp
55 MacroEmissionCheckScope guard(masm);
56
57 // Immediates on Aarch64 can be produced using an initial value, and zero to
58 // three move keep operations.
59 //
60 // Initial values can be generated with:
61 // 1. 64-bit move zero (movz).
62 // 2. 32-bit move inverted (movn).
63 // 3. 64-bit move inverted.
64 // 4. 32-bit orr immediate.
65 // 5. 64-bit orr immediate.
66 // Move-keep may then be used to modify each of the 16-bit half words.
67 //
68 // The code below supports all five initial value generators, and
69 // applying move-keep operations to move-zero and move-inverted initial
70 // values.
71
72 // Try to move the immediate in one instruction, and if that fails, switch to
73 // using multiple instructions.
74 if (OneInstrMoveImmediateHelper(masm, rd, imm)) {
75 return 1;
76 } else {
77 int instruction_count = 0;
78 unsigned reg_size = rd.size();
79
80 // Generic immediate case. Imm will be represented by
81 // [imm3, imm2, imm1, imm0], where each imm is 16 bits.
82 // A move-zero or move-inverted is generated for the first non-zero or
83 // non-0xffff immX, and a move-keep for subsequent non-zero immX.
84
85 uint64_t ignored_halfword = 0;
86 bool invert_move = false;
87 // If the number of 0xffff halfwords is greater than the number of 0x0000
88 // halfwords, it's more efficient to use move-inverted.
89 if (CountClearHalfWords(~imm, reg_size) >
90 CountClearHalfWords(imm, reg_size)) {
91 ignored_halfword = 0xffff;
92 invert_move = true;
93 }
94
95 // Mov instructions can't move values into the stack pointer, so set up a
96 // temporary register, if needed.
97 UseScratchRegisterScope temps;
98 Register temp;
99 if (emit_code) {
100 temps.Open(masm);
101 temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
102 }
103
104 // Iterate through the halfwords. Use movn/movz for the first non-ignored
105 // halfword, and movk for subsequent halfwords.
106 VIXL_ASSERT((reg_size % 16) == 0);
107 bool first_mov_done = false;
108 for (unsigned i = 0; i < (temp.size() / 16); i++) {
109 uint64_t imm16 = (imm >> (16 * i)) & 0xffff;
110 if (imm16 != ignored_halfword) {
111 if (!first_mov_done) {
112 if (invert_move) {
113 if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i);
114 instruction_count++;
115 } else {
116 if (emit_code) masm->movz(temp, imm16, 16 * i);
117 instruction_count++;
118 }
119 first_mov_done = true;
120 } else {
121 // Construct a wider constant.
122 if (emit_code) masm->movk(temp, imm16, 16 * i);
123 instruction_count++;
124 }
125 }
126 }
127
128 VIXL_ASSERT(first_mov_done);
129
130 // Move the temporary if the original destination register was the stack
131 // pointer.
132 if (rd.IsSP()) {
133 if (emit_code) masm->mov(rd, temp);
134 instruction_count++;
135 }
136 return instruction_count;
137 }
138 }
139
140
OneInstrMoveImmediateHelper(MacroAssembler * masm,const Register & dst,int64_t imm)141 bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm,
142 const Register& dst,
143 int64_t imm) {
144 bool emit_code = masm != NULL;
145 unsigned n, imm_s, imm_r;
146 int reg_size = dst.size();
147
148 if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
149 // Immediate can be represented in a move zero instruction. Movz can't write
150 // to the stack pointer.
151 if (emit_code) {
152 masm->movz(dst, imm);
153 }
154 return true;
155 } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
156 // Immediate can be represented in a move negative instruction. Movn can't
157 // write to the stack pointer.
158 if (emit_code) {
159 masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
160 }
161 return true;
162 } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
163 // Immediate can be represented in a logical orr instruction.
164 VIXL_ASSERT(!dst.IsZero());
165 if (emit_code) {
166 masm->LogicalImmediate(
167 dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
168 }
169 return true;
170 }
171 return false;
172 }
173
174
B(Label * label,BranchType type,Register reg,int bit)175 void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) {
176 VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) &&
177 ((bit == -1) || (type >= kBranchTypeFirstUsingBit)));
178 if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
179 B(static_cast<Condition>(type), label);
180 } else {
181 switch (type) {
182 case always: B(label); break;
183 case never: break;
184 case reg_zero: Cbz(reg, label); break;
185 case reg_not_zero: Cbnz(reg, label); break;
186 case reg_bit_clear: Tbz(reg, bit, label); break;
187 case reg_bit_set: Tbnz(reg, bit, label); break;
188 default:
189 VIXL_UNREACHABLE();
190 }
191 }
192 }
193
194
B(Label * label)195 void MacroAssembler::B(Label* label) {
196 SingleEmissionCheckScope guard(this);
197 b(label);
198 }
199
200
B(Label * label,Condition cond)201 void MacroAssembler::B(Label* label, Condition cond) {
202 VIXL_ASSERT((cond != al) && (cond != nv));
203 EmissionCheckScope guard(this, 2 * kInstructionSize);
204
205 if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
206 Label done;
207 b(&done, InvertCondition(cond));
208 b(label);
209 bind(&done);
210 } else {
211 b(label, cond);
212 }
213 }
214
215
Cbnz(const Register & rt,Label * label)216 void MacroAssembler::Cbnz(const Register& rt, Label* label) {
217 VIXL_ASSERT(!rt.IsZero());
218 EmissionCheckScope guard(this, 2 * kInstructionSize);
219
220 if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
221 Label done;
222 cbz(rt, &done);
223 b(label);
224 bind(&done);
225 } else {
226 cbnz(rt, label);
227 }
228 }
229
230
Cbz(const Register & rt,Label * label)231 void MacroAssembler::Cbz(const Register& rt, Label* label) {
232 VIXL_ASSERT(!rt.IsZero());
233 EmissionCheckScope guard(this, 2 * kInstructionSize);
234
235 if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
236 Label done;
237 cbnz(rt, &done);
238 b(label);
239 bind(&done);
240 } else {
241 cbz(rt, label);
242 }
243 }
244
245
Tbnz(const Register & rt,unsigned bit_pos,Label * label)246 void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
247 VIXL_ASSERT(!rt.IsZero());
248 EmissionCheckScope guard(this, 2 * kInstructionSize);
249
250 if (label->bound() && LabelIsOutOfRange(label, TestBranchType)) {
251 Label done;
252 tbz(rt, bit_pos, &done);
253 b(label);
254 bind(&done);
255 } else {
256 tbnz(rt, bit_pos, label);
257 }
258 }
259
260
Tbz(const Register & rt,unsigned bit_pos,Label * label)261 void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
262 VIXL_ASSERT(!rt.IsZero());
263 EmissionCheckScope guard(this, 2 * kInstructionSize);
264
265 if (label->bound() && LabelIsOutOfRange(label, TestBranchType)) {
266 Label done;
267 tbnz(rt, bit_pos, &done);
268 b(label);
269 bind(&done);
270 } else {
271 tbz(rt, bit_pos, label);
272 }
273 }
274
275
And(const Register & rd,const Register & rn,const Operand & operand)276 void MacroAssembler::And(const Register& rd,
277 const Register& rn,
278 const Operand& operand) {
279 LogicalMacro(rd, rn, operand, AND);
280 }
281
282
Ands(const Register & rd,const Register & rn,const Operand & operand)283 void MacroAssembler::Ands(const Register& rd,
284 const Register& rn,
285 const Operand& operand) {
286 LogicalMacro(rd, rn, operand, ANDS);
287 }
288
289
Tst(const Register & rn,const Operand & operand)290 void MacroAssembler::Tst(const Register& rn,
291 const Operand& operand) {
292 Ands(AppropriateZeroRegFor(rn), rn, operand);
293 }
294
295
Bic(const Register & rd,const Register & rn,const Operand & operand)296 void MacroAssembler::Bic(const Register& rd,
297 const Register& rn,
298 const Operand& operand) {
299 LogicalMacro(rd, rn, operand, BIC);
300 }
301
302
Bics(const Register & rd,const Register & rn,const Operand & operand)303 void MacroAssembler::Bics(const Register& rd,
304 const Register& rn,
305 const Operand& operand) {
306 LogicalMacro(rd, rn, operand, BICS);
307 }
308
309
Orr(const Register & rd,const Register & rn,const Operand & operand)310 void MacroAssembler::Orr(const Register& rd,
311 const Register& rn,
312 const Operand& operand) {
313 LogicalMacro(rd, rn, operand, ORR);
314 }
315
316
Orn(const Register & rd,const Register & rn,const Operand & operand)317 void MacroAssembler::Orn(const Register& rd,
318 const Register& rn,
319 const Operand& operand) {
320 LogicalMacro(rd, rn, operand, ORN);
321 }
322
323
Eor(const Register & rd,const Register & rn,const Operand & operand)324 void MacroAssembler::Eor(const Register& rd,
325 const Register& rn,
326 const Operand& operand) {
327 LogicalMacro(rd, rn, operand, EOR);
328 }
329
330
Eon(const Register & rd,const Register & rn,const Operand & operand)331 void MacroAssembler::Eon(const Register& rd,
332 const Register& rn,
333 const Operand& operand) {
334 LogicalMacro(rd, rn, operand, EON);
335 }
336
337
LogicalMacro(const Register & rd,const Register & rn,const Operand & operand,LogicalOp op)338 void MacroAssembler::LogicalMacro(const Register& rd,
339 const Register& rn,
340 const Operand& operand,
341 LogicalOp op) {
342 // The worst case for size is logical immediate to sp:
343 // * up to 4 instructions to materialise the constant
344 // * 1 instruction to do the operation
345 // * 1 instruction to move to sp
346 MacroEmissionCheckScope guard(this);
347 UseScratchRegisterScope temps(this);
348
349 if (operand.IsImmediate()) {
350 int64_t immediate = operand.immediate();
351 unsigned reg_size = rd.size();
352
353 // If the operation is NOT, invert the operation and immediate.
354 if ((op & NOT) == NOT) {
355 op = static_cast<LogicalOp>(op & ~NOT);
356 immediate = ~immediate;
357 }
358
359 // Ignore the top 32 bits of an immediate if we're moving to a W register.
360 if (rd.Is32Bits()) {
361 // Check that the top 32 bits are consistent.
362 VIXL_ASSERT(((immediate >> kWRegSize) == 0) ||
363 ((immediate >> kWRegSize) == -1));
364 immediate &= kWRegMask;
365 }
366
367 VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
368
369 // Special cases for all set or all clear immediates.
370 if (immediate == 0) {
371 switch (op) {
372 case AND:
373 Mov(rd, 0);
374 return;
375 case ORR:
376 VIXL_FALLTHROUGH();
377 case EOR:
378 Mov(rd, rn);
379 return;
380 case ANDS:
381 VIXL_FALLTHROUGH();
382 case BICS:
383 break;
384 default:
385 VIXL_UNREACHABLE();
386 }
387 } else if ((rd.Is64Bits() && (immediate == -1)) ||
388 (rd.Is32Bits() && (immediate == 0xffffffff))) {
389 switch (op) {
390 case AND:
391 Mov(rd, rn);
392 return;
393 case ORR:
394 Mov(rd, immediate);
395 return;
396 case EOR:
397 Mvn(rd, rn);
398 return;
399 case ANDS:
400 VIXL_FALLTHROUGH();
401 case BICS:
402 break;
403 default:
404 VIXL_UNREACHABLE();
405 }
406 }
407
408 unsigned n, imm_s, imm_r;
409 if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
410 // Immediate can be encoded in the instruction.
411 LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
412 } else {
413 // Immediate can't be encoded: synthesize using move immediate.
414 Register temp = temps.AcquireSameSizeAs(rn);
415
416 // If the left-hand input is the stack pointer, we can't pre-shift the
417 // immediate, as the encoding won't allow the subsequent post shift.
418 PreShiftImmMode mode = rn.IsSP() ? kNoShift : kAnyShift;
419 Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);
420
421 // VIXL can acquire temp registers. Assert that the caller is aware.
422 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
423 VIXL_ASSERT(!temp.Is(operand.maybeReg()));
424
425 if (rd.Is(sp)) {
426 // If rd is the stack pointer we cannot use it as the destination
427 // register so we use the temp register as an intermediate again.
428 Logical(temp, rn, imm_operand, op);
429 Mov(sp, temp);
430 } else {
431 Logical(rd, rn, imm_operand, op);
432 }
433 }
434 } else if (operand.IsExtendedRegister()) {
435 VIXL_ASSERT(operand.reg().size() <= rd.size());
436 // Add/sub extended supports shift <= 4. We want to support exactly the
437 // same modes here.
438 VIXL_ASSERT(operand.shift_amount() <= 4);
439 VIXL_ASSERT(operand.reg().Is64Bits() ||
440 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
441
442 temps.Exclude(operand.reg());
443 Register temp = temps.AcquireSameSizeAs(rn);
444
445 // VIXL can acquire temp registers. Assert that the caller is aware.
446 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
447 VIXL_ASSERT(!temp.Is(operand.maybeReg()));
448
449 EmitExtendShift(temp, operand.reg(), operand.extend(),
450 operand.shift_amount());
451 Logical(rd, rn, Operand(temp), op);
452 } else {
453 // The operand can be encoded in the instruction.
454 VIXL_ASSERT(operand.IsShiftedRegister());
455 Logical(rd, rn, operand, op);
456 }
457 }
458
459
Mov(const Register & rd,const Operand & operand,DiscardMoveMode discard_mode)460 void MacroAssembler::Mov(const Register& rd,
461 const Operand& operand,
462 DiscardMoveMode discard_mode) {
463 // The worst case for size is mov immediate with up to 4 instructions.
464 MacroEmissionCheckScope guard(this);
465
466 if (operand.IsImmediate()) {
467 // Call the macro assembler for generic immediates.
468 Mov(rd, operand.immediate());
469 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
470 // Emit a shift instruction if moving a shifted register. This operation
471 // could also be achieved using an orr instruction (like orn used by Mvn),
472 // but using a shift instruction makes the disassembly clearer.
473 EmitShift(rd, operand.reg(), operand.shift(), operand.shift_amount());
474 } else if (operand.IsExtendedRegister()) {
475 // Emit an extend instruction if moving an extended register. This handles
476 // extend with post-shift operations, too.
477 EmitExtendShift(rd, operand.reg(), operand.extend(),
478 operand.shift_amount());
479 } else {
480 // Otherwise, emit a register move only if the registers are distinct, or
481 // if they are not X registers.
482 //
483 // Note that mov(w0, w0) is not a no-op because it clears the top word of
484 // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
485 // registers is not required to clear the top word of the X register. In
486 // this case, the instruction is discarded.
487 //
488 // If the sp is an operand, add #0 is emitted, otherwise, orr #0.
489 if (!rd.Is(operand.reg()) || (rd.Is32Bits() &&
490 (discard_mode == kDontDiscardForSameWReg))) {
491 mov(rd, operand.reg());
492 }
493 }
494 }
495
496
Movi16bitHelper(const VRegister & vd,uint64_t imm)497 void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
498 VIXL_ASSERT(IsUint16(imm));
499 int byte1 = (imm & 0xff);
500 int byte2 = ((imm >> 8) & 0xff);
501 if (byte1 == byte2) {
502 movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
503 } else if (byte1 == 0) {
504 movi(vd, byte2, LSL, 8);
505 } else if (byte2 == 0) {
506 movi(vd, byte1);
507 } else if (byte1 == 0xff) {
508 mvni(vd, ~byte2 & 0xff, LSL, 8);
509 } else if (byte2 == 0xff) {
510 mvni(vd, ~byte1 & 0xff);
511 } else {
512 UseScratchRegisterScope temps(this);
513 Register temp = temps.AcquireW();
514 movz(temp, imm);
515 dup(vd, temp);
516 }
517 }
518
519
Movi32bitHelper(const VRegister & vd,uint64_t imm)520 void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
521 VIXL_ASSERT(IsUint32(imm));
522
523 uint8_t bytes[sizeof(imm)];
524 memcpy(bytes, &imm, sizeof(imm));
525
526 // All bytes are either 0x00 or 0xff.
527 {
528 bool all0orff = true;
529 for (int i = 0; i < 4; ++i) {
530 if ((bytes[i] != 0) && (bytes[i] != 0xff)) {
531 all0orff = false;
532 break;
533 }
534 }
535
536 if (all0orff == true) {
537 movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
538 return;
539 }
540 }
541
542 // Of the 4 bytes, only one byte is non-zero.
543 for (int i = 0; i < 4; i++) {
544 if ((imm & (0xff << (i * 8))) == imm) {
545 movi(vd, bytes[i], LSL, i * 8);
546 return;
547 }
548 }
549
550 // Of the 4 bytes, only one byte is not 0xff.
551 for (int i = 0; i < 4; i++) {
552 uint32_t mask = ~(0xff << (i * 8));
553 if ((imm & mask) == mask) {
554 mvni(vd, ~bytes[i] & 0xff, LSL, i * 8);
555 return;
556 }
557 }
558
559 // Immediate is of the form 0x00MMFFFF.
560 if ((imm & 0xff00ffff) == 0x0000ffff) {
561 movi(vd, bytes[2], MSL, 16);
562 return;
563 }
564
565 // Immediate is of the form 0x0000MMFF.
566 if ((imm & 0xffff00ff) == 0x000000ff) {
567 movi(vd, bytes[1], MSL, 8);
568 return;
569 }
570
571 // Immediate is of the form 0xFFMM0000.
572 if ((imm & 0xff00ffff) == 0xff000000) {
573 mvni(vd, ~bytes[2] & 0xff, MSL, 16);
574 return;
575 }
576 // Immediate is of the form 0xFFFFMM00.
577 if ((imm & 0xffff00ff) == 0xffff0000) {
578 mvni(vd, ~bytes[1] & 0xff, MSL, 8);
579 return;
580 }
581
582 // Top and bottom 16-bits are equal.
583 if (((imm >> 16) & 0xffff) == (imm & 0xffff)) {
584 Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff);
585 return;
586 }
587
588 // Default case.
589 {
590 UseScratchRegisterScope temps(this);
591 Register temp = temps.AcquireW();
592 Mov(temp, imm);
593 dup(vd, temp);
594 }
595 }
596
597
Movi64bitHelper(const VRegister & vd,uint64_t imm)598 void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
599 // All bytes are either 0x00 or 0xff.
600 {
601 bool all0orff = true;
602 for (int i = 0; i < 8; ++i) {
603 int byteval = (imm >> (i * 8)) & 0xff;
604 if (byteval != 0 && byteval != 0xff) {
605 all0orff = false;
606 break;
607 }
608 }
609 if (all0orff == true) {
610 movi(vd, imm);
611 return;
612 }
613 }
614
615 // Top and bottom 32-bits are equal.
616 if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) {
617 Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff);
618 return;
619 }
620
621 // Default case.
622 {
623 UseScratchRegisterScope temps(this);
624 Register temp = temps.AcquireX();
625 Mov(temp, imm);
626 if (vd.Is1D()) {
627 mov(vd.D(), 0, temp);
628 } else {
629 dup(vd.V2D(), temp);
630 }
631 }
632 }
633
634
Movi(const VRegister & vd,uint64_t imm,Shift shift,int shift_amount)635 void MacroAssembler::Movi(const VRegister& vd,
636 uint64_t imm,
637 Shift shift,
638 int shift_amount) {
639 MacroEmissionCheckScope guard(this);
640 if (shift_amount != 0 || shift != LSL) {
641 movi(vd, imm, shift, shift_amount);
642 } else if (vd.Is8B() || vd.Is16B()) {
643 // 8-bit immediate.
644 VIXL_ASSERT(IsUint8(imm));
645 movi(vd, imm);
646 } else if (vd.Is4H() || vd.Is8H()) {
647 // 16-bit immediate.
648 Movi16bitHelper(vd, imm);
649 } else if (vd.Is2S() || vd.Is4S()) {
650 // 32-bit immediate.
651 Movi32bitHelper(vd, imm);
652 } else {
653 // 64-bit immediate.
654 Movi64bitHelper(vd, imm);
655 }
656 }
657
658
Movi(const VRegister & vd,uint64_t hi,uint64_t lo)659 void MacroAssembler::Movi(const VRegister& vd,
660 uint64_t hi,
661 uint64_t lo) {
662 VIXL_ASSERT(vd.Is128Bits());
663 UseScratchRegisterScope temps(this);
664
665 // When hi == lo, the following generates good code.
666 //
667 // In situations where the constants are complex and hi != lo, the following
668 // can turn into up to 10 instructions: 2*(mov + 3*movk + dup/insert). To do
669 // any better, we could try to estimate whether splatting the high value and
670 // updating the low value would generate fewer instructions than vice versa
671 // (what we do now).
672 //
673 // (A PC-relative load from memory to the vector register (ADR + LD2) is going
674 // to have fairly high latency but is fairly compact; not clear what the best
675 // tradeoff is.)
676
677 Movi(vd.V2D(), lo);
678 if (hi != lo) {
679 Register temp = temps.AcquireX();
680 Mov(temp, hi);
681 Ins(vd.V2D(), 1, temp);
682 }
683 }
684
685
Mvn(const Register & rd,const Operand & operand)686 void MacroAssembler::Mvn(const Register& rd, const Operand& operand) {
687 // The worst case for size is mvn immediate with up to 4 instructions.
688 MacroEmissionCheckScope guard(this);
689
690 if (operand.IsImmediate()) {
691 // Call the macro assembler for generic immediates.
692 Mvn(rd, operand.immediate());
693 } else if (operand.IsExtendedRegister()) {
694 UseScratchRegisterScope temps(this);
695 temps.Exclude(operand.reg());
696
697 // Emit two instructions for the extend case. This differs from Mov, as
698 // the extend and invert can't be achieved in one instruction.
699 Register temp = temps.AcquireSameSizeAs(rd);
700
701 // VIXL can acquire temp registers. Assert that the caller is aware.
702 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(operand.maybeReg()));
703
704 EmitExtendShift(temp, operand.reg(), operand.extend(),
705 operand.shift_amount());
706 mvn(rd, Operand(temp));
707 } else {
708 // Otherwise, register and shifted register cases can be handled by the
709 // assembler directly, using orn.
710 mvn(rd, operand);
711 }
712 }
713
714
Mov(const Register & rd,uint64_t imm)715 void MacroAssembler::Mov(const Register& rd, uint64_t imm) {
716 MoveImmediateHelper(this, rd, imm);
717 }
718
719
Ccmp(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)720 void MacroAssembler::Ccmp(const Register& rn,
721 const Operand& operand,
722 StatusFlags nzcv,
723 Condition cond) {
724 if (operand.IsImmediate() && (operand.immediate() < 0)) {
725 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMN);
726 } else {
727 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP);
728 }
729 }
730
731
Ccmn(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond)732 void MacroAssembler::Ccmn(const Register& rn,
733 const Operand& operand,
734 StatusFlags nzcv,
735 Condition cond) {
736 if (operand.IsImmediate() && (operand.immediate() < 0)) {
737 ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMP);
738 } else {
739 ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN);
740 }
741 }
742
743
ConditionalCompareMacro(const Register & rn,const Operand & operand,StatusFlags nzcv,Condition cond,ConditionalCompareOp op)744 void MacroAssembler::ConditionalCompareMacro(const Register& rn,
745 const Operand& operand,
746 StatusFlags nzcv,
747 Condition cond,
748 ConditionalCompareOp op) {
749 VIXL_ASSERT((cond != al) && (cond != nv));
750 // The worst case for size is ccmp immediate:
751 // * up to 4 instructions to materialise the constant
752 // * 1 instruction for ccmp
753 MacroEmissionCheckScope guard(this);
754
755 if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
756 (operand.IsImmediate() && IsImmConditionalCompare(operand.immediate()))) {
757 // The immediate can be encoded in the instruction, or the operand is an
758 // unshifted register: call the assembler.
759 ConditionalCompare(rn, operand, nzcv, cond, op);
760 } else {
761 UseScratchRegisterScope temps(this);
762 // The operand isn't directly supported by the instruction: perform the
763 // operation on a temporary register.
764 Register temp = temps.AcquireSameSizeAs(rn);
765 VIXL_ASSERT(!temp.Is(rn) && !temp.Is(operand.maybeReg()));
766 Mov(temp, operand);
767 ConditionalCompare(rn, temp, nzcv, cond, op);
768 }
769 }
770
771
Csel(const Register & rd,const Register & rn,const Operand & operand,Condition cond)772 void MacroAssembler::Csel(const Register& rd,
773 const Register& rn,
774 const Operand& operand,
775 Condition cond) {
776 VIXL_ASSERT(!rd.IsZero());
777 VIXL_ASSERT(!rn.IsZero());
778 VIXL_ASSERT((cond != al) && (cond != nv));
779 // The worst case for size is csel immediate:
780 // * up to 4 instructions to materialise the constant
781 // * 1 instruction for csel
782 MacroEmissionCheckScope guard(this);
783
784 if (operand.IsImmediate()) {
785 // Immediate argument. Handle special cases of 0, 1 and -1 using zero
786 // register.
787 int64_t imm = operand.immediate();
788 Register zr = AppropriateZeroRegFor(rn);
789 if (imm == 0) {
790 csel(rd, rn, zr, cond);
791 } else if (imm == 1) {
792 csinc(rd, rn, zr, cond);
793 } else if (imm == -1) {
794 csinv(rd, rn, zr, cond);
795 } else {
796 UseScratchRegisterScope temps(this);
797 Register temp = temps.AcquireSameSizeAs(rn);
798 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
799 VIXL_ASSERT(!temp.Is(operand.maybeReg()));
800 Mov(temp, operand.immediate());
801 csel(rd, rn, temp, cond);
802 }
803 } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
804 // Unshifted register argument.
805 csel(rd, rn, operand.reg(), cond);
806 } else {
807 // All other arguments.
808 UseScratchRegisterScope temps(this);
809 Register temp = temps.AcquireSameSizeAs(rn);
810 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
811 VIXL_ASSERT(!temp.Is(operand.maybeReg()));
812 Mov(temp, operand);
813 csel(rd, rn, temp, cond);
814 }
815 }
816
817
Add(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S)818 void MacroAssembler::Add(const Register& rd,
819 const Register& rn,
820 const Operand& operand,
821 FlagsUpdate S) {
822 if (operand.IsImmediate() && (operand.immediate() < 0) &&
823 IsImmAddSub(-operand.immediate())) {
824 AddSubMacro(rd, rn, -operand.immediate(), S, SUB);
825 } else {
826 AddSubMacro(rd, rn, operand, S, ADD);
827 }
828 }
829
830
Adds(const Register & rd,const Register & rn,const Operand & operand)831 void MacroAssembler::Adds(const Register& rd,
832 const Register& rn,
833 const Operand& operand) {
834 Add(rd, rn, operand, SetFlags);
835 }
836
837
Sub(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S)838 void MacroAssembler::Sub(const Register& rd,
839 const Register& rn,
840 const Operand& operand,
841 FlagsUpdate S) {
842 if (operand.IsImmediate() && (operand.immediate() < 0) &&
843 IsImmAddSub(-operand.immediate())) {
844 AddSubMacro(rd, rn, -operand.immediate(), S, ADD);
845 } else {
846 AddSubMacro(rd, rn, operand, S, SUB);
847 }
848 }
849
850
Subs(const Register & rd,const Register & rn,const Operand & operand)851 void MacroAssembler::Subs(const Register& rd,
852 const Register& rn,
853 const Operand& operand) {
854 Sub(rd, rn, operand, SetFlags);
855 }
856
857
Cmn(const Register & rn,const Operand & operand)858 void MacroAssembler::Cmn(const Register& rn, const Operand& operand) {
859 Adds(AppropriateZeroRegFor(rn), rn, operand);
860 }
861
862
Cmp(const Register & rn,const Operand & operand)863 void MacroAssembler::Cmp(const Register& rn, const Operand& operand) {
864 Subs(AppropriateZeroRegFor(rn), rn, operand);
865 }
866
867
Fcmp(const FPRegister & fn,double value,FPTrapFlags trap)868 void MacroAssembler::Fcmp(const FPRegister& fn, double value,
869 FPTrapFlags trap) {
870 // The worst case for size is:
871 // * 1 to materialise the constant, using literal pool if necessary
872 // * 1 instruction for fcmp{e}
873 MacroEmissionCheckScope guard(this);
874 if (value != 0.0) {
875 UseScratchRegisterScope temps(this);
876 FPRegister tmp = temps.AcquireSameSizeAs(fn);
877 VIXL_ASSERT(!tmp.Is(fn));
878 Fmov(tmp, value);
879 FPCompareMacro(fn, tmp, trap);
880 } else {
881 FPCompareMacro(fn, value, trap);
882 }
883 }
884
885
Fcmpe(const FPRegister & fn,double value)886 void MacroAssembler::Fcmpe(const FPRegister& fn, double value) {
887 Fcmp(fn, value, EnableTrap);
888 }
889
890
Fmov(VRegister vd,double imm)891 void MacroAssembler::Fmov(VRegister vd, double imm) {
892 // Floating point immediates are loaded through the literal pool.
893 MacroEmissionCheckScope guard(this);
894
895 if (vd.Is1S() || vd.Is2S() || vd.Is4S()) {
896 Fmov(vd, static_cast<float>(imm));
897 return;
898 }
899
900 VIXL_ASSERT(vd.Is1D() || vd.Is2D());
901 if (IsImmFP64(imm)) {
902 fmov(vd, imm);
903 } else {
904 uint64_t rawbits = DoubleToRawbits(imm);
905 if (vd.IsScalar()) {
906 if (rawbits == 0) {
907 fmov(vd, xzr);
908 } else {
909 Assembler::fImmPool64(vd, imm);
910 }
911 } else {
912 // TODO: consider NEON support for load literal.
913 Movi(vd, rawbits);
914 }
915 }
916 }
917
918
Fmov(VRegister vd,float imm)919 void MacroAssembler::Fmov(VRegister vd, float imm) {
920 // Floating point immediates are loaded through the literal pool.
921 MacroEmissionCheckScope guard(this);
922
923 if (vd.Is1D() || vd.Is2D()) {
924 Fmov(vd, static_cast<double>(imm));
925 return;
926 }
927
928 VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S());
929 if (IsImmFP32(imm)) {
930 fmov(vd, imm);
931 } else {
932 uint32_t rawbits = FloatToRawbits(imm);
933 if (vd.IsScalar()) {
934 if (rawbits == 0) {
935 fmov(vd, wzr);
936 } else {
937 Assembler::fImmPool32(vd, imm);
938 }
939 } else {
940 // TODO: consider NEON support for load literal.
941 Movi(vd, rawbits);
942 }
943 }
944 }
945
946
947
Neg(const Register & rd,const Operand & operand)948 void MacroAssembler::Neg(const Register& rd,
949 const Operand& operand) {
950 if (operand.IsImmediate()) {
951 Mov(rd, -operand.immediate());
952 } else {
953 Sub(rd, AppropriateZeroRegFor(rd), operand);
954 }
955 }
956
957
Negs(const Register & rd,const Operand & operand)958 void MacroAssembler::Negs(const Register& rd,
959 const Operand& operand) {
960 Subs(rd, AppropriateZeroRegFor(rd), operand);
961 }
962
963
TryOneInstrMoveImmediate(const Register & dst,int64_t imm)964 bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst,
965 int64_t imm) {
966 return OneInstrMoveImmediateHelper(this, dst, imm);
967 }
968
969
MoveImmediateForShiftedOp(const Register & dst,int64_t imm,PreShiftImmMode mode)970 Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst,
971 int64_t imm,
972 PreShiftImmMode mode) {
973 int reg_size = dst.size();
974
975 // Encode the immediate in a single move instruction, if possible.
976 if (TryOneInstrMoveImmediate(dst, imm)) {
977 // The move was successful; nothing to do here.
978 } else {
979 // Pre-shift the immediate to the least-significant bits of the register.
980 int shift_low = CountTrailingZeros(imm, reg_size);
981 if (mode == kLimitShiftForSP) {
982 // When applied to the stack pointer, the subsequent arithmetic operation
983 // can use the extend form to shift left by a maximum of four bits. Right
984 // shifts are not allowed, so we filter them out later before the new
985 // immediate is tested.
986 shift_low = std::min(shift_low, 4);
987 }
988
989 int64_t imm_low = imm >> shift_low;
990
991 // Pre-shift the immediate to the most-significant bits of the register,
992 // inserting set bits in the least-significant bits.
993 int shift_high = CountLeadingZeros(imm, reg_size);
994 int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
995
996 if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
997 // The new immediate has been moved into the destination's low bits:
998 // return a new leftward-shifting operand.
999 return Operand(dst, LSL, shift_low);
1000 } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
1001 // The new immediate has been moved into the destination's high bits:
1002 // return a new rightward-shifting operand.
1003 return Operand(dst, LSR, shift_high);
1004 } else {
1005 Mov(dst, imm);
1006 }
1007 }
1008 return Operand(dst);
1009 }
1010
1011
ComputeAddress(const Register & dst,const MemOperand & mem_op)1012 void MacroAssembler::ComputeAddress(const Register& dst,
1013 const MemOperand& mem_op) {
1014 // We cannot handle pre-indexing or post-indexing.
1015 VIXL_ASSERT(mem_op.addrmode() == Offset);
1016 Register base = mem_op.base();
1017 if (mem_op.IsImmediateOffset()) {
1018 Add(dst, base, mem_op.offset());
1019 } else {
1020 VIXL_ASSERT(mem_op.IsRegisterOffset());
1021 Register reg_offset = mem_op.regoffset();
1022 Shift shift = mem_op.shift();
1023 Extend extend = mem_op.extend();
1024 if (shift == NO_SHIFT) {
1025 VIXL_ASSERT(extend != NO_EXTEND);
1026 Add(dst, base, Operand(reg_offset, extend, mem_op.shift_amount()));
1027 } else {
1028 VIXL_ASSERT(extend == NO_EXTEND);
1029 Add(dst, base, Operand(reg_offset, shift, mem_op.shift_amount()));
1030 }
1031 }
1032 }
1033
1034
AddSubMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubOp op)1035 void MacroAssembler::AddSubMacro(const Register& rd,
1036 const Register& rn,
1037 const Operand& operand,
1038 FlagsUpdate S,
1039 AddSubOp op) {
1040 // Worst case is add/sub immediate:
1041 // * up to 4 instructions to materialise the constant
1042 // * 1 instruction for add/sub
1043 MacroEmissionCheckScope guard(this);
1044
1045 if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() &&
1046 (S == LeaveFlags)) {
1047 // The instruction would be a nop. Avoid generating useless code.
1048 return;
1049 }
1050
1051 if ((operand.IsImmediate() && !IsImmAddSub(operand.immediate())) ||
1052 (rn.IsZero() && !operand.IsShiftedRegister()) ||
1053 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
1054 UseScratchRegisterScope temps(this);
1055 Register temp = temps.AcquireSameSizeAs(rn);
1056 if (operand.IsImmediate()) {
1057 PreShiftImmMode mode = kAnyShift;
1058
1059 // If the destination or source register is the stack pointer, we can
1060 // only pre-shift the immediate right by values supported in the add/sub
1061 // extend encoding.
1062 if (rd.IsSP()) {
1063 // If the destination is SP and flags will be set, we can't pre-shift
1064 // the immediate at all.
1065 mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
1066 } else if (rn.IsSP()) {
1067 mode = kLimitShiftForSP;
1068 }
1069
1070 Operand imm_operand =
1071 MoveImmediateForShiftedOp(temp, operand.immediate(), mode);
1072 AddSub(rd, rn, imm_operand, S, op);
1073 } else {
1074 Mov(temp, operand);
1075 AddSub(rd, rn, temp, S, op);
1076 }
1077 } else {
1078 AddSub(rd, rn, operand, S, op);
1079 }
1080 }
1081
1082
Adc(const Register & rd,const Register & rn,const Operand & operand)1083 void MacroAssembler::Adc(const Register& rd,
1084 const Register& rn,
1085 const Operand& operand) {
1086 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC);
1087 }
1088
1089
Adcs(const Register & rd,const Register & rn,const Operand & operand)1090 void MacroAssembler::Adcs(const Register& rd,
1091 const Register& rn,
1092 const Operand& operand) {
1093 AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC);
1094 }
1095
1096
Sbc(const Register & rd,const Register & rn,const Operand & operand)1097 void MacroAssembler::Sbc(const Register& rd,
1098 const Register& rn,
1099 const Operand& operand) {
1100 AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC);
1101 }
1102
1103
Sbcs(const Register & rd,const Register & rn,const Operand & operand)1104 void MacroAssembler::Sbcs(const Register& rd,
1105 const Register& rn,
1106 const Operand& operand) {
1107 AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC);
1108 }
1109
1110
Ngc(const Register & rd,const Operand & operand)1111 void MacroAssembler::Ngc(const Register& rd,
1112 const Operand& operand) {
1113 Register zr = AppropriateZeroRegFor(rd);
1114 Sbc(rd, zr, operand);
1115 }
1116
1117
Ngcs(const Register & rd,const Operand & operand)1118 void MacroAssembler::Ngcs(const Register& rd,
1119 const Operand& operand) {
1120 Register zr = AppropriateZeroRegFor(rd);
1121 Sbcs(rd, zr, operand);
1122 }
1123
1124
AddSubWithCarryMacro(const Register & rd,const Register & rn,const Operand & operand,FlagsUpdate S,AddSubWithCarryOp op)1125 void MacroAssembler::AddSubWithCarryMacro(const Register& rd,
1126 const Register& rn,
1127 const Operand& operand,
1128 FlagsUpdate S,
1129 AddSubWithCarryOp op) {
1130 VIXL_ASSERT(rd.size() == rn.size());
1131 // Worst case is addc/subc immediate:
1132 // * up to 4 instructions to materialise the constant
1133 // * 1 instruction for add/sub
1134 MacroEmissionCheckScope guard(this);
1135 UseScratchRegisterScope temps(this);
1136
1137 if (operand.IsImmediate() ||
1138 (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
1139 // Add/sub with carry (immediate or ROR shifted register.)
1140 Register temp = temps.AcquireSameSizeAs(rn);
1141 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
1142 Mov(temp, operand);
1143 AddSubWithCarry(rd, rn, Operand(temp), S, op);
1144 } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
1145 // Add/sub with carry (shifted register).
1146 VIXL_ASSERT(operand.reg().size() == rd.size());
1147 VIXL_ASSERT(operand.shift() != ROR);
1148 VIXL_ASSERT(IsUintN(rd.size() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2,
1149 operand.shift_amount()));
1150 temps.Exclude(operand.reg());
1151 Register temp = temps.AcquireSameSizeAs(rn);
1152 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
1153 EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
1154 AddSubWithCarry(rd, rn, Operand(temp), S, op);
1155 } else if (operand.IsExtendedRegister()) {
1156 // Add/sub with carry (extended register).
1157 VIXL_ASSERT(operand.reg().size() <= rd.size());
1158 // Add/sub extended supports a shift <= 4. We want to support exactly the
1159 // same modes.
1160 VIXL_ASSERT(operand.shift_amount() <= 4);
1161 VIXL_ASSERT(operand.reg().Is64Bits() ||
1162 ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
1163 temps.Exclude(operand.reg());
1164 Register temp = temps.AcquireSameSizeAs(rn);
1165 VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
1166 EmitExtendShift(temp, operand.reg(), operand.extend(),
1167 operand.shift_amount());
1168 AddSubWithCarry(rd, rn, Operand(temp), S, op);
1169 } else {
1170 // The addressing mode is directly supported by the instruction.
1171 AddSubWithCarry(rd, rn, operand, S, op);
1172 }
1173 }
1174
1175
1176 #define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \
1177 void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \
1178 LoadStoreMacro(REG, addr, OP); \
1179 }
LS_MACRO_LIST(DEFINE_FUNCTION)1180 LS_MACRO_LIST(DEFINE_FUNCTION)
1181 #undef DEFINE_FUNCTION
1182
1183
1184 void MacroAssembler::LoadStoreMacro(const CPURegister& rt,
1185 const MemOperand& addr,
1186 LoadStoreOp op) {
1187 // Worst case is ldr/str pre/post index:
1188 // * 1 instruction for ldr/str
1189 // * up to 4 instructions to materialise the constant
1190 // * 1 instruction to update the base
1191 MacroEmissionCheckScope guard(this);
1192
1193 int64_t offset = addr.offset();
1194 unsigned access_size = CalcLSDataSize(op);
1195
1196 // Check if an immediate offset fits in the immediate field of the
1197 // appropriate instruction. If not, emit two instructions to perform
1198 // the operation.
1199 if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) &&
1200 !IsImmLSUnscaled(offset)) {
1201 // Immediate offset that can't be encoded using unsigned or unscaled
1202 // addressing modes.
1203 UseScratchRegisterScope temps(this);
1204 Register temp = temps.AcquireSameSizeAs(addr.base());
1205 VIXL_ASSERT(!temp.Is(rt));
1206 VIXL_ASSERT(!temp.Is(addr.base()) && !temp.Is(addr.regoffset()));
1207 Mov(temp, addr.offset());
1208 LoadStore(rt, MemOperand(addr.base(), temp), op);
1209 } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
1210 // Post-index beyond unscaled addressing range.
1211 LoadStore(rt, MemOperand(addr.base()), op);
1212 Add(addr.base(), addr.base(), Operand(offset));
1213 } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
1214 // Pre-index beyond unscaled addressing range.
1215 Add(addr.base(), addr.base(), Operand(offset));
1216 LoadStore(rt, MemOperand(addr.base()), op);
1217 } else {
1218 // Encodable in one load/store instruction.
1219 LoadStore(rt, addr, op);
1220 }
1221 }
1222
1223
1224 #define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
1225 void MacroAssembler::FN(const REGTYPE REG, \
1226 const REGTYPE REG2, \
1227 const MemOperand& addr) { \
1228 LoadStorePairMacro(REG, REG2, addr, OP); \
1229 }
LSPAIR_MACRO_LIST(DEFINE_FUNCTION)1230 LSPAIR_MACRO_LIST(DEFINE_FUNCTION)
1231 #undef DEFINE_FUNCTION
1232
1233 void MacroAssembler::LoadStorePairMacro(const CPURegister& rt,
1234 const CPURegister& rt2,
1235 const MemOperand& addr,
1236 LoadStorePairOp op) {
1237 // TODO(all): Should we support register offset for load-store-pair?
1238 VIXL_ASSERT(!addr.IsRegisterOffset());
1239 // Worst case is ldp/stp immediate:
1240 // * 1 instruction for ldp/stp
1241 // * up to 4 instructions to materialise the constant
1242 // * 1 instruction to update the base
1243 MacroEmissionCheckScope guard(this);
1244
1245 int64_t offset = addr.offset();
1246 unsigned access_size = CalcLSPairDataSize(op);
1247
1248 // Check if the offset fits in the immediate field of the appropriate
1249 // instruction. If not, emit two instructions to perform the operation.
1250 if (IsImmLSPair(offset, access_size)) {
1251 // Encodable in one load/store pair instruction.
1252 LoadStorePair(rt, rt2, addr, op);
1253 } else {
1254 Register base = addr.base();
1255 if (addr.IsImmediateOffset()) {
1256 UseScratchRegisterScope temps(this);
1257 Register temp = temps.AcquireSameSizeAs(base);
1258 Add(temp, base, offset);
1259 LoadStorePair(rt, rt2, MemOperand(temp), op);
1260 } else if (addr.IsPostIndex()) {
1261 LoadStorePair(rt, rt2, MemOperand(base), op);
1262 Add(base, base, offset);
1263 } else {
1264 VIXL_ASSERT(addr.IsPreIndex());
1265 Add(base, base, offset);
1266 LoadStorePair(rt, rt2, MemOperand(base), op);
1267 }
1268 }
1269 }
1270
1271
Prfm(PrefetchOperation op,const MemOperand & addr)1272 void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) {
1273 MacroEmissionCheckScope guard(this);
1274
1275 // There are no pre- or post-index modes for prfm.
1276 VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset());
1277
1278 // The access size is implicitly 8 bytes for all prefetch operations.
1279 unsigned size = kXRegSizeInBytesLog2;
1280
1281 // Check if an immediate offset fits in the immediate field of the
1282 // appropriate instruction. If not, emit two instructions to perform
1283 // the operation.
1284 if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.offset(), size) &&
1285 !IsImmLSUnscaled(addr.offset())) {
1286 // Immediate offset that can't be encoded using unsigned or unscaled
1287 // addressing modes.
1288 UseScratchRegisterScope temps(this);
1289 Register temp = temps.AcquireSameSizeAs(addr.base());
1290 Mov(temp, addr.offset());
1291 Prefetch(op, MemOperand(addr.base(), temp));
1292 } else {
1293 // Simple register-offsets are encodable in one instruction.
1294 Prefetch(op, addr);
1295 }
1296 }
1297
1298
PushStackPointer()1299 void MacroAssembler::PushStackPointer() {
1300 PrepareForPush(1, 8);
1301
1302 // Pushing a stack pointer leads to implementation-defined
1303 // behavior, which may be surprising. In particular,
1304 // str x28, [x28, #-8]!
1305 // pre-decrements the stack pointer, storing the decremented value.
1306 // Additionally, sp is read as xzr in this context, so it cannot be pushed.
1307 // So we must use a scratch register.
1308 UseScratchRegisterScope temps(this);
1309 Register scratch = temps.AcquireX();
1310
1311 Mov(scratch, GetStackPointer64());
1312 str(scratch, MemOperand(GetStackPointer64(), -8, PreIndex));
1313 }
1314
1315
Push(const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1316 void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1317 const CPURegister& src2, const CPURegister& src3) {
1318 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
1319 VIXL_ASSERT(src0.IsValid());
1320
1321 int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid();
1322 int size = src0.SizeInBytes();
1323
1324 if (src0.Is(GetStackPointer64())) {
1325 VIXL_ASSERT(count == 1);
1326 VIXL_ASSERT(size == 8);
1327 PushStackPointer();
1328 return;
1329 }
1330
1331 PrepareForPush(count, size);
1332 PushHelper(count, size, src0, src1, src2, src3);
1333 }
1334
1335
Pop(const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1336 void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1337 const CPURegister& dst2, const CPURegister& dst3) {
1338 // It is not valid to pop into the same register more than once in one
1339 // instruction, not even into the zero register.
1340 VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3));
1341 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1342 VIXL_ASSERT(dst0.IsValid());
1343
1344 int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid();
1345 int size = dst0.SizeInBytes();
1346
1347 PrepareForPop(count, size);
1348 PopHelper(count, size, dst0, dst1, dst2, dst3);
1349 }
1350
1351
PushCPURegList(CPURegList registers)1352 void MacroAssembler::PushCPURegList(CPURegList registers) {
1353 VIXL_ASSERT(!registers.Overlaps(*TmpList()));
1354 VIXL_ASSERT(!registers.Overlaps(*FPTmpList()));
1355
1356 int reg_size = registers.RegisterSizeInBytes();
1357 PrepareForPush(registers.Count(), reg_size);
1358
1359 // Bump the stack pointer and store two registers at the bottom.
1360 int size = registers.TotalSizeInBytes();
1361 const CPURegister& bottom_0 = registers.PopLowestIndex();
1362 const CPURegister& bottom_1 = registers.PopLowestIndex();
1363 if (bottom_0.IsValid() && bottom_1.IsValid()) {
1364 Stp(bottom_0, bottom_1, MemOperand(GetStackPointer64(), -size, PreIndex));
1365 } else if (bottom_0.IsValid()) {
1366 Str(bottom_0, MemOperand(GetStackPointer64(), -size, PreIndex));
1367 }
1368
1369 int offset = 2 * reg_size;
1370 while (!registers.IsEmpty()) {
1371 const CPURegister& src0 = registers.PopLowestIndex();
1372 const CPURegister& src1 = registers.PopLowestIndex();
1373 if (src1.IsValid()) {
1374 Stp(src0, src1, MemOperand(GetStackPointer64(), offset));
1375 } else {
1376 Str(src0, MemOperand(GetStackPointer64(), offset));
1377 }
1378 offset += 2 * reg_size;
1379 }
1380 }
1381
1382
PopCPURegList(CPURegList registers)1383 void MacroAssembler::PopCPURegList(CPURegList registers) {
1384 VIXL_ASSERT(!registers.Overlaps(*TmpList()));
1385 VIXL_ASSERT(!registers.Overlaps(*FPTmpList()));
1386
1387 int reg_size = registers.RegisterSizeInBytes();
1388 PrepareForPop(registers.Count(), reg_size);
1389
1390
1391 int size = registers.TotalSizeInBytes();
1392 const CPURegister& bottom_0 = registers.PopLowestIndex();
1393 const CPURegister& bottom_1 = registers.PopLowestIndex();
1394
1395 int offset = 2 * reg_size;
1396 while (!registers.IsEmpty()) {
1397 const CPURegister& dst0 = registers.PopLowestIndex();
1398 const CPURegister& dst1 = registers.PopLowestIndex();
1399 if (dst1.IsValid()) {
1400 Ldp(dst0, dst1, MemOperand(GetStackPointer64(), offset));
1401 } else {
1402 Ldr(dst0, MemOperand(GetStackPointer64(), offset));
1403 }
1404 offset += 2 * reg_size;
1405 }
1406
1407 // Load the two registers at the bottom and drop the stack pointer.
1408 if (bottom_0.IsValid() && bottom_1.IsValid()) {
1409 Ldp(bottom_0, bottom_1, MemOperand(GetStackPointer64(), size, PostIndex));
1410 } else if (bottom_0.IsValid()) {
1411 Ldr(bottom_0, MemOperand(GetStackPointer64(), size, PostIndex));
1412 }
1413 }
1414
1415
PushMultipleTimes(int count,Register src)1416 void MacroAssembler::PushMultipleTimes(int count, Register src) {
1417 int size = src.SizeInBytes();
1418
1419 PrepareForPush(count, size);
1420 // Push up to four registers at a time if possible because if the current
1421 // stack pointer is sp and the register size is 32, registers must be pushed
1422 // in blocks of four in order to maintain the 16-byte alignment for sp.
1423 while (count >= 4) {
1424 PushHelper(4, size, src, src, src, src);
1425 count -= 4;
1426 }
1427 if (count >= 2) {
1428 PushHelper(2, size, src, src, NoReg, NoReg);
1429 count -= 2;
1430 }
1431 if (count == 1) {
1432 PushHelper(1, size, src, NoReg, NoReg, NoReg);
1433 count -= 1;
1434 }
1435 VIXL_ASSERT(count == 0);
1436 }
1437
1438
PushHelper(int count,int size,const CPURegister & src0,const CPURegister & src1,const CPURegister & src2,const CPURegister & src3)1439 void MacroAssembler::PushHelper(int count, int size,
1440 const CPURegister& src0,
1441 const CPURegister& src1,
1442 const CPURegister& src2,
1443 const CPURegister& src3) {
1444 // Ensure that we don't unintentionally modify scratch or debug registers.
1445 // Worst case for size is 2 stp.
1446 InstructionAccurateScope scope(this, 2,
1447 InstructionAccurateScope::kMaximumSize);
1448
1449 VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
1450 VIXL_ASSERT(size == src0.SizeInBytes());
1451
1452 // Pushing the stack pointer has unexpected behavior. See PushStackPointer().
1453 VIXL_ASSERT(!src0.Is(GetStackPointer64()) && !src0.Is(sp));
1454 VIXL_ASSERT(!src1.Is(GetStackPointer64()) && !src1.Is(sp));
1455 VIXL_ASSERT(!src2.Is(GetStackPointer64()) && !src2.Is(sp));
1456 VIXL_ASSERT(!src3.Is(GetStackPointer64()) && !src3.Is(sp));
1457
1458 // The JS engine should never push 4 bytes.
1459 VIXL_ASSERT(size >= 8);
1460
1461 // When pushing multiple registers, the store order is chosen such that
1462 // Push(a, b) is equivalent to Push(a) followed by Push(b).
1463 switch (count) {
1464 case 1:
1465 VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone());
1466 str(src0, MemOperand(GetStackPointer64(), -1 * size, PreIndex));
1467 break;
1468 case 2:
1469 VIXL_ASSERT(src2.IsNone() && src3.IsNone());
1470 stp(src1, src0, MemOperand(GetStackPointer64(), -2 * size, PreIndex));
1471 break;
1472 case 3:
1473 VIXL_ASSERT(src3.IsNone());
1474 stp(src2, src1, MemOperand(GetStackPointer64(), -3 * size, PreIndex));
1475 str(src0, MemOperand(GetStackPointer64(), 2 * size));
1476 break;
1477 case 4:
1478 // Skip over 4 * size, then fill in the gap. This allows four W registers
1479 // to be pushed using sp, whilst maintaining 16-byte alignment for sp at
1480 // all times.
1481 stp(src3, src2, MemOperand(GetStackPointer64(), -4 * size, PreIndex));
1482 stp(src1, src0, MemOperand(GetStackPointer64(), 2 * size));
1483 break;
1484 default:
1485 VIXL_UNREACHABLE();
1486 }
1487 }
1488
1489
PopHelper(int count,int size,const CPURegister & dst0,const CPURegister & dst1,const CPURegister & dst2,const CPURegister & dst3)1490 void MacroAssembler::PopHelper(int count, int size,
1491 const CPURegister& dst0,
1492 const CPURegister& dst1,
1493 const CPURegister& dst2,
1494 const CPURegister& dst3) {
1495 // Ensure that we don't unintentionally modify scratch or debug registers.
1496 // Worst case for size is 2 ldp.
1497 InstructionAccurateScope scope(this, 2,
1498 InstructionAccurateScope::kMaximumSize);
1499
1500 VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1501 VIXL_ASSERT(size == dst0.SizeInBytes());
1502
1503 // When popping multiple registers, the load order is chosen such that
1504 // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
1505 switch (count) {
1506 case 1:
1507 VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
1508 ldr(dst0, MemOperand(GetStackPointer64(), 1 * size, PostIndex));
1509 break;
1510 case 2:
1511 VIXL_ASSERT(dst2.IsNone() && dst3.IsNone());
1512 ldp(dst0, dst1, MemOperand(GetStackPointer64(), 2 * size, PostIndex));
1513 break;
1514 case 3:
1515 VIXL_ASSERT(dst3.IsNone());
1516 ldr(dst2, MemOperand(GetStackPointer64(), 2 * size));
1517 ldp(dst0, dst1, MemOperand(GetStackPointer64(), 3 * size, PostIndex));
1518 break;
1519 case 4:
1520 // Load the higher addresses first, then load the lower addresses and skip
1521 // the whole block in the second instruction. This allows four W registers
1522 // to be popped using sp, whilst maintaining 16-byte alignment for sp at
1523 // all times.
1524 ldp(dst2, dst3, MemOperand(GetStackPointer64(), 2 * size));
1525 ldp(dst0, dst1, MemOperand(GetStackPointer64(), 4 * size, PostIndex));
1526 break;
1527 default:
1528 VIXL_UNREACHABLE();
1529 }
1530 }
1531
1532
PrepareForPush(int count,int size)1533 void MacroAssembler::PrepareForPush(int count, int size) {
1534 if (sp.Is(GetStackPointer64())) {
1535 // If the current stack pointer is sp, then it must be aligned to 16 bytes
1536 // on entry and the total size of the specified registers must also be a
1537 // multiple of 16 bytes.
1538 VIXL_ASSERT((count * size) % 16 == 0);
1539 } else {
1540 // Even if the current stack pointer is not the system stack pointer (sp),
1541 // the system stack pointer will still be modified in order to comply with
1542 // ABI rules about accessing memory below the system stack pointer.
1543 BumpSystemStackPointer(count * size);
1544 }
1545 }
1546
1547
PrepareForPop(int count,int size)1548 void MacroAssembler::PrepareForPop(int count, int size) {
1549 USE(count, size);
1550 if (sp.Is(GetStackPointer64())) {
1551 // If the current stack pointer is sp, then it must be aligned to 16 bytes
1552 // on entry and the total size of the specified registers must also be a
1553 // multiple of 16 bytes.
1554 VIXL_ASSERT((count * size) % 16 == 0);
1555 }
1556 }
1557
Poke(const Register & src,const Operand & offset)1558 void MacroAssembler::Poke(const Register& src, const Operand& offset) {
1559 if (offset.IsImmediate()) {
1560 VIXL_ASSERT(offset.immediate() >= 0);
1561 }
1562
1563 Str(src, MemOperand(GetStackPointer64(), offset));
1564 }
1565
1566
Peek(const Register & dst,const Operand & offset)1567 void MacroAssembler::Peek(const Register& dst, const Operand& offset) {
1568 if (offset.IsImmediate()) {
1569 VIXL_ASSERT(offset.immediate() >= 0);
1570 }
1571
1572 Ldr(dst, MemOperand(GetStackPointer64(), offset));
1573 }
1574
1575
Claim(const Operand & size)1576 void MacroAssembler::Claim(const Operand& size) {
1577
1578 if (size.IsZero()) {
1579 return;
1580 }
1581
1582 if (size.IsImmediate()) {
1583 VIXL_ASSERT(size.immediate() > 0);
1584 if (sp.Is(GetStackPointer64())) {
1585 VIXL_ASSERT((size.immediate() % 16) == 0);
1586 }
1587 }
1588
1589 Sub(GetStackPointer64(), GetStackPointer64(), size);
1590
1591 // Make sure the real stack pointer reflects the claimed stack space.
1592 // We can't use stack memory below the stack pointer, it could be clobbered by
1593 // interupts and signal handlers.
1594 if (!sp.Is(GetStackPointer64())) {
1595 Mov(sp, GetStackPointer64());
1596 }
1597 }
1598
1599
Drop(const Operand & size)1600 void MacroAssembler::Drop(const Operand& size) {
1601
1602 if (size.IsZero()) {
1603 return;
1604 }
1605
1606 if (size.IsImmediate()) {
1607 VIXL_ASSERT(size.immediate() > 0);
1608 if (sp.Is(GetStackPointer64())) {
1609 VIXL_ASSERT((size.immediate() % 16) == 0);
1610 }
1611 }
1612
1613 Add(GetStackPointer64(), GetStackPointer64(), size);
1614 }
1615
1616
PushCalleeSavedRegisters()1617 void MacroAssembler::PushCalleeSavedRegisters() {
1618 // Ensure that the macro-assembler doesn't use any scratch registers.
1619 // 10 stp will be emitted.
1620 // TODO(all): Should we use GetCalleeSaved and SavedFP.
1621 InstructionAccurateScope scope(this, 10);
1622
1623 // This method must not be called unless the current stack pointer is sp.
1624 VIXL_ASSERT(sp.Is(GetStackPointer64()));
1625
1626 MemOperand tos(sp, -2 * static_cast<int>(kXRegSizeInBytes), PreIndex);
1627
1628 stp(x29, x30, tos);
1629 stp(x27, x28, tos);
1630 stp(x25, x26, tos);
1631 stp(x23, x24, tos);
1632 stp(x21, x22, tos);
1633 stp(x19, x20, tos);
1634
1635 stp(d14, d15, tos);
1636 stp(d12, d13, tos);
1637 stp(d10, d11, tos);
1638 stp(d8, d9, tos);
1639 }
1640
1641
PopCalleeSavedRegisters()1642 void MacroAssembler::PopCalleeSavedRegisters() {
1643 // Ensure that the macro-assembler doesn't use any scratch registers.
1644 // 10 ldp will be emitted.
1645 // TODO(all): Should we use GetCalleeSaved and SavedFP.
1646 InstructionAccurateScope scope(this, 10);
1647
1648 // This method must not be called unless the current stack pointer is sp.
1649 VIXL_ASSERT(sp.Is(GetStackPointer64()));
1650
1651 MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex);
1652
1653 ldp(d8, d9, tos);
1654 ldp(d10, d11, tos);
1655 ldp(d12, d13, tos);
1656 ldp(d14, d15, tos);
1657
1658 ldp(x19, x20, tos);
1659 ldp(x21, x22, tos);
1660 ldp(x23, x24, tos);
1661 ldp(x25, x26, tos);
1662 ldp(x27, x28, tos);
1663 ldp(x29, x30, tos);
1664 }
1665
LoadCPURegList(CPURegList registers,const MemOperand & src)1666 void MacroAssembler::LoadCPURegList(CPURegList registers,
1667 const MemOperand& src) {
1668 LoadStoreCPURegListHelper(kLoad, registers, src);
1669 }
1670
StoreCPURegList(CPURegList registers,const MemOperand & dst)1671 void MacroAssembler::StoreCPURegList(CPURegList registers,
1672 const MemOperand& dst) {
1673 LoadStoreCPURegListHelper(kStore, registers, dst);
1674 }
1675
1676
LoadStoreCPURegListHelper(LoadStoreCPURegListAction op,CPURegList registers,const MemOperand & mem)1677 void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op,
1678 CPURegList registers,
1679 const MemOperand& mem) {
1680 // We do not handle pre-indexing or post-indexing.
1681 VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex()));
1682 VIXL_ASSERT(!registers.Overlaps(tmp_list_));
1683 VIXL_ASSERT(!registers.Overlaps(fptmp_list_));
1684 VIXL_ASSERT(!registers.IncludesAliasOf(sp));
1685
1686 UseScratchRegisterScope temps(this);
1687
1688 MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers,
1689 mem,
1690 &temps);
1691
1692 while (registers.Count() >= 2) {
1693 const CPURegister& dst0 = registers.PopLowestIndex();
1694 const CPURegister& dst1 = registers.PopLowestIndex();
1695 if (op == kStore) {
1696 Stp(dst0, dst1, loc);
1697 } else {
1698 VIXL_ASSERT(op == kLoad);
1699 Ldp(dst0, dst1, loc);
1700 }
1701 loc.AddOffset(2 * registers.RegisterSizeInBytes());
1702 }
1703 if (!registers.IsEmpty()) {
1704 if (op == kStore) {
1705 Str(registers.PopLowestIndex(), loc);
1706 } else {
1707 VIXL_ASSERT(op == kLoad);
1708 Ldr(registers.PopLowestIndex(), loc);
1709 }
1710 }
1711 }
1712
BaseMemOperandForLoadStoreCPURegList(const CPURegList & registers,const MemOperand & mem,UseScratchRegisterScope * scratch_scope)1713 MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList(
1714 const CPURegList& registers,
1715 const MemOperand& mem,
1716 UseScratchRegisterScope* scratch_scope) {
1717 // If necessary, pre-compute the base address for the accesses.
1718 if (mem.IsRegisterOffset()) {
1719 Register reg_base = scratch_scope->AcquireX();
1720 ComputeAddress(reg_base, mem);
1721 return MemOperand(reg_base);
1722
1723 } else if (mem.IsImmediateOffset()) {
1724 int reg_size = registers.RegisterSizeInBytes();
1725 int total_size = registers.TotalSizeInBytes();
1726 int64_t min_offset = mem.offset();
1727 int64_t max_offset = mem.offset() + std::max(0, total_size - 2 * reg_size);
1728 if ((registers.Count() >= 2) &&
1729 (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) ||
1730 !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) {
1731 Register reg_base = scratch_scope->AcquireX();
1732 ComputeAddress(reg_base, mem);
1733 return MemOperand(reg_base);
1734 }
1735 }
1736
1737 return mem;
1738 }
1739
BumpSystemStackPointer(const Operand & space)1740 void MacroAssembler::BumpSystemStackPointer(const Operand& space) {
1741 VIXL_ASSERT(!sp.Is(GetStackPointer64()));
1742 // TODO: Several callers rely on this not using scratch registers, so we use
1743 // the assembler directly here. However, this means that large immediate
1744 // values of 'space' cannot be handled.
1745 InstructionAccurateScope scope(this, 1);
1746 sub(sp, GetStackPointer64(), space);
1747 }
1748
1749
Trace(TraceParameters parameters,TraceCommand command)1750 void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) {
1751
1752 #ifdef JS_SIMULATOR_ARM64
1753 // The arguments to the trace pseudo instruction need to be contiguous in
1754 // memory, so make sure we don't try to emit a literal pool.
1755 InstructionAccurateScope scope(this, kTraceLength / kInstructionSize);
1756
1757 Label start;
1758 bind(&start);
1759
1760 // Refer to simulator-a64.h for a description of the marker and its
1761 // arguments.
1762 hlt(kTraceOpcode);
1763
1764 // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceParamsOffset);
1765 dc32(parameters);
1766
1767 // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceCommandOffset);
1768 dc32(command);
1769 #else
1770 // Emit nothing on real hardware.
1771 USE(parameters, command);
1772 #endif
1773 }
1774
1775
Log(TraceParameters parameters)1776 void MacroAssembler::Log(TraceParameters parameters) {
1777
1778 #ifdef JS_SIMULATOR_ARM64
1779 // The arguments to the log pseudo instruction need to be contiguous in
1780 // memory, so make sure we don't try to emit a literal pool.
1781 InstructionAccurateScope scope(this, kLogLength / kInstructionSize);
1782
1783 Label start;
1784 bind(&start);
1785
1786 // Refer to simulator-a64.h for a description of the marker and its
1787 // arguments.
1788 hlt(kLogOpcode);
1789
1790 // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kLogParamsOffset);
1791 dc32(parameters);
1792 #else
1793 // Emit nothing on real hardware.
1794 USE(parameters);
1795 #endif
1796 }
1797
1798
EnableInstrumentation()1799 void MacroAssembler::EnableInstrumentation() {
1800 VIXL_ASSERT(!isprint(InstrumentStateEnable));
1801 InstructionAccurateScope scope(this, 1);
1802 movn(xzr, InstrumentStateEnable);
1803 }
1804
1805
DisableInstrumentation()1806 void MacroAssembler::DisableInstrumentation() {
1807 VIXL_ASSERT(!isprint(InstrumentStateDisable));
1808 InstructionAccurateScope scope(this, 1);
1809 movn(xzr, InstrumentStateDisable);
1810 }
1811
1812
AnnotateInstrumentation(const char * marker_name)1813 void MacroAssembler::AnnotateInstrumentation(const char* marker_name) {
1814 VIXL_ASSERT(strlen(marker_name) == 2);
1815
1816 // We allow only printable characters in the marker names. Unprintable
1817 // characters are reserved for controlling features of the instrumentation.
1818 VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1]));
1819
1820 InstructionAccurateScope scope(this, 1);
1821 movn(xzr, (marker_name[1] << 8) | marker_name[0]);
1822 }
1823
1824
Open(MacroAssembler * masm)1825 void UseScratchRegisterScope::Open(MacroAssembler* masm) {
1826 VIXL_ASSERT(!initialised_);
1827 available_ = masm->TmpList();
1828 availablefp_ = masm->FPTmpList();
1829 old_available_ = available_->list();
1830 old_availablefp_ = availablefp_->list();
1831 VIXL_ASSERT(available_->type() == CPURegister::kRegister);
1832 VIXL_ASSERT(availablefp_->type() == CPURegister::kVRegister);
1833 #ifdef DEBUG
1834 initialised_ = true;
1835 #endif
1836 }
1837
1838
Close()1839 void UseScratchRegisterScope::Close() {
1840 if (available_) {
1841 available_->set_list(old_available_);
1842 available_ = NULL;
1843 }
1844 if (availablefp_) {
1845 availablefp_->set_list(old_availablefp_);
1846 availablefp_ = NULL;
1847 }
1848 #ifdef DEBUG
1849 initialised_ = false;
1850 #endif
1851 }
1852
1853
UseScratchRegisterScope(MacroAssembler * masm)1854 UseScratchRegisterScope::UseScratchRegisterScope(MacroAssembler* masm) {
1855 #ifdef DEBUG
1856 initialised_ = false;
1857 #endif
1858 Open(masm);
1859 }
1860
1861 // This allows deferred (and optional) initialisation of the scope.
UseScratchRegisterScope()1862 UseScratchRegisterScope::UseScratchRegisterScope()
1863 : available_(NULL), availablefp_(NULL),
1864 old_available_(0), old_availablefp_(0) {
1865 #ifdef DEBUG
1866 initialised_ = false;
1867 #endif
1868 }
1869
~UseScratchRegisterScope()1870 UseScratchRegisterScope::~UseScratchRegisterScope() {
1871 Close();
1872 }
1873
1874
IsAvailable(const CPURegister & reg) const1875 bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const {
1876 return available_->IncludesAliasOf(reg) || availablefp_->IncludesAliasOf(reg);
1877 }
1878
1879
AcquireSameSizeAs(const Register & reg)1880 Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
1881 int code = AcquireNextAvailable(available_).code();
1882 return Register(code, reg.size());
1883 }
1884
1885
AcquireSameSizeAs(const FPRegister & reg)1886 FPRegister UseScratchRegisterScope::AcquireSameSizeAs(const FPRegister& reg) {
1887 int code = AcquireNextAvailable(availablefp_).code();
1888 return FPRegister(code, reg.size());
1889 }
1890
1891
Release(const CPURegister & reg)1892 void UseScratchRegisterScope::Release(const CPURegister& reg) {
1893 VIXL_ASSERT(initialised_);
1894 if (reg.IsRegister()) {
1895 ReleaseByCode(available_, reg.code());
1896 } else if (reg.IsFPRegister()) {
1897 ReleaseByCode(availablefp_, reg.code());
1898 } else {
1899 VIXL_ASSERT(reg.IsNone());
1900 }
1901 }
1902
1903
Include(const CPURegList & list)1904 void UseScratchRegisterScope::Include(const CPURegList& list) {
1905 VIXL_ASSERT(initialised_);
1906 if (list.type() == CPURegister::kRegister) {
1907 // Make sure that neither sp nor xzr are included the list.
1908 IncludeByRegList(available_, list.list() & ~(xzr.Bit() | sp.Bit()));
1909 } else {
1910 VIXL_ASSERT(list.type() == CPURegister::kVRegister);
1911 IncludeByRegList(availablefp_, list.list());
1912 }
1913 }
1914
1915
Include(const Register & reg1,const Register & reg2,const Register & reg3,const Register & reg4)1916 void UseScratchRegisterScope::Include(const Register& reg1,
1917 const Register& reg2,
1918 const Register& reg3,
1919 const Register& reg4) {
1920 VIXL_ASSERT(initialised_);
1921 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
1922 // Make sure that neither sp nor xzr are included the list.
1923 include &= ~(xzr.Bit() | sp.Bit());
1924
1925 IncludeByRegList(available_, include);
1926 }
1927
1928
Include(const FPRegister & reg1,const FPRegister & reg2,const FPRegister & reg3,const FPRegister & reg4)1929 void UseScratchRegisterScope::Include(const FPRegister& reg1,
1930 const FPRegister& reg2,
1931 const FPRegister& reg3,
1932 const FPRegister& reg4) {
1933 RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
1934 IncludeByRegList(availablefp_, include);
1935 }
1936
1937
Exclude(const CPURegList & list)1938 void UseScratchRegisterScope::Exclude(const CPURegList& list) {
1939 if (list.type() == CPURegister::kRegister) {
1940 ExcludeByRegList(available_, list.list());
1941 } else {
1942 VIXL_ASSERT(list.type() == CPURegister::kVRegister);
1943 ExcludeByRegList(availablefp_, list.list());
1944 }
1945 }
1946
1947
Exclude(const Register & reg1,const Register & reg2,const Register & reg3,const Register & reg4)1948 void UseScratchRegisterScope::Exclude(const Register& reg1,
1949 const Register& reg2,
1950 const Register& reg3,
1951 const Register& reg4) {
1952 RegList exclude = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
1953 ExcludeByRegList(available_, exclude);
1954 }
1955
1956
Exclude(const FPRegister & reg1,const FPRegister & reg2,const FPRegister & reg3,const FPRegister & reg4)1957 void UseScratchRegisterScope::Exclude(const FPRegister& reg1,
1958 const FPRegister& reg2,
1959 const FPRegister& reg3,
1960 const FPRegister& reg4) {
1961 RegList excludefp = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
1962 ExcludeByRegList(availablefp_, excludefp);
1963 }
1964
1965
Exclude(const CPURegister & reg1,const CPURegister & reg2,const CPURegister & reg3,const CPURegister & reg4)1966 void UseScratchRegisterScope::Exclude(const CPURegister& reg1,
1967 const CPURegister& reg2,
1968 const CPURegister& reg3,
1969 const CPURegister& reg4) {
1970 RegList exclude = 0;
1971 RegList excludefp = 0;
1972
1973 const CPURegister regs[] = {reg1, reg2, reg3, reg4};
1974
1975 for (unsigned i = 0; i < (sizeof(regs) / sizeof(regs[0])); i++) {
1976 if (regs[i].IsRegister()) {
1977 exclude |= regs[i].Bit();
1978 } else if (regs[i].IsFPRegister()) {
1979 excludefp |= regs[i].Bit();
1980 } else {
1981 VIXL_ASSERT(regs[i].IsNone());
1982 }
1983 }
1984
1985 ExcludeByRegList(available_, exclude);
1986 ExcludeByRegList(availablefp_, excludefp);
1987 }
1988
1989
ExcludeAll()1990 void UseScratchRegisterScope::ExcludeAll() {
1991 ExcludeByRegList(available_, available_->list());
1992 ExcludeByRegList(availablefp_, availablefp_->list());
1993 }
1994
1995
AcquireNextAvailable(CPURegList * available)1996 CPURegister UseScratchRegisterScope::AcquireNextAvailable(
1997 CPURegList* available) {
1998 VIXL_CHECK(!available->IsEmpty());
1999 CPURegister result = available->PopLowestIndex();
2000 VIXL_ASSERT(!AreAliased(result, xzr, sp));
2001 return result;
2002 }
2003
2004
ReleaseByCode(CPURegList * available,int code)2005 void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) {
2006 ReleaseByRegList(available, static_cast<RegList>(1) << code);
2007 }
2008
2009
ReleaseByRegList(CPURegList * available,RegList regs)2010 void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available,
2011 RegList regs) {
2012 available->set_list(available->list() | regs);
2013 }
2014
2015
IncludeByRegList(CPURegList * available,RegList regs)2016 void UseScratchRegisterScope::IncludeByRegList(CPURegList* available,
2017 RegList regs) {
2018 available->set_list(available->list() | regs);
2019 }
2020
2021
ExcludeByRegList(CPURegList * available,RegList exclude)2022 void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available,
2023 RegList exclude) {
2024 available->set_list(available->list() & ~exclude);
2025 }
2026
2027 } // namespace vixl
2028