1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_arm_MacroAssembler_arm_inl_h
8 #define jit_arm_MacroAssembler_arm_inl_h
9
10 #include "jit/arm/MacroAssembler-arm.h"
11
12 namespace js {
13 namespace jit {
14
15 //{{{ check_macroassembler_style
16
move64(Register64 src,Register64 dest)17 void MacroAssembler::move64(Register64 src, Register64 dest) {
18 move32(src.low, dest.low);
19 move32(src.high, dest.high);
20 }
21
move64(Imm64 imm,Register64 dest)22 void MacroAssembler::move64(Imm64 imm, Register64 dest) {
23 move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
24 move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
25 }
26
moveFloat32ToGPR(FloatRegister src,Register dest)27 void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
28 ma_vxfer(src, dest);
29 }
30
moveGPRToFloat32(Register src,FloatRegister dest)31 void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
32 ma_vxfer(src, dest);
33 }
34
move8SignExtend(Register src,Register dest)35 void MacroAssembler::move8SignExtend(Register src, Register dest) {
36 as_sxtb(dest, src, 0);
37 }
38
move16SignExtend(Register src,Register dest)39 void MacroAssembler::move16SignExtend(Register src, Register dest) {
40 as_sxth(dest, src, 0);
41 }
42
moveDoubleToGPR64(FloatRegister src,Register64 dest)43 void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
44 ma_vxfer(src, dest.low, dest.high);
45 }
46
moveGPR64ToDouble(Register64 src,FloatRegister dest)47 void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
48 ma_vxfer(src.low, src.high, dest);
49 }
50
move64To32(Register64 src,Register dest)51 void MacroAssembler::move64To32(Register64 src, Register dest) {
52 if (src.low != dest) move32(src.low, dest);
53 }
54
move32To64ZeroExtend(Register src,Register64 dest)55 void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
56 if (src != dest.low) move32(src, dest.low);
57 move32(Imm32(0), dest.high);
58 }
59
move8To64SignExtend(Register src,Register64 dest)60 void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
61 as_sxtb(dest.low, src, 0);
62 ma_asr(Imm32(31), dest.low, dest.high);
63 }
64
move16To64SignExtend(Register src,Register64 dest)65 void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
66 as_sxth(dest.low, src, 0);
67 ma_asr(Imm32(31), dest.low, dest.high);
68 }
69
move32To64SignExtend(Register src,Register64 dest)70 void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
71 if (src != dest.low) move32(src, dest.low);
72 ma_asr(Imm32(31), dest.low, dest.high);
73 }
74
75 // ===============================================================
76 // Logical instructions
77
not32(Register reg)78 void MacroAssembler::not32(Register reg) { ma_mvn(reg, reg); }
79
and32(Register src,Register dest)80 void MacroAssembler::and32(Register src, Register dest) {
81 ma_and(src, dest, SetCC);
82 }
83
and32(Imm32 imm,Register dest)84 void MacroAssembler::and32(Imm32 imm, Register dest) {
85 ScratchRegisterScope scratch(*this);
86 ma_and(imm, dest, scratch, SetCC);
87 }
88
and32(Imm32 imm,const Address & dest)89 void MacroAssembler::and32(Imm32 imm, const Address& dest) {
90 ScratchRegisterScope scratch(*this);
91 SecondScratchRegisterScope scratch2(*this);
92
93 ma_ldr(dest, scratch, scratch2);
94 ma_and(imm, scratch, scratch2);
95 ma_str(scratch, dest, scratch2);
96 }
97
and32(const Address & src,Register dest)98 void MacroAssembler::and32(const Address& src, Register dest) {
99 ScratchRegisterScope scratch(*this);
100 SecondScratchRegisterScope scratch2(*this);
101
102 ma_ldr(src, scratch, scratch2);
103 ma_and(scratch, dest, SetCC);
104 }
105
andPtr(Register src,Register dest)106 void MacroAssembler::andPtr(Register src, Register dest) { ma_and(src, dest); }
107
andPtr(Imm32 imm,Register dest)108 void MacroAssembler::andPtr(Imm32 imm, Register dest) {
109 ScratchRegisterScope scratch(*this);
110 ma_and(imm, dest, scratch);
111 }
112
and64(Imm64 imm,Register64 dest)113 void MacroAssembler::and64(Imm64 imm, Register64 dest) {
114 if (imm.low().value != int32_t(0xFFFFFFFF)) and32(imm.low(), dest.low);
115 if (imm.hi().value != int32_t(0xFFFFFFFF)) and32(imm.hi(), dest.high);
116 }
117
or64(Imm64 imm,Register64 dest)118 void MacroAssembler::or64(Imm64 imm, Register64 dest) {
119 if (imm.low().value) or32(imm.low(), dest.low);
120 if (imm.hi().value) or32(imm.hi(), dest.high);
121 }
122
xor64(Imm64 imm,Register64 dest)123 void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
124 if (imm.low().value) xor32(imm.low(), dest.low);
125 if (imm.hi().value) xor32(imm.hi(), dest.high);
126 }
127
or32(Register src,Register dest)128 void MacroAssembler::or32(Register src, Register dest) { ma_orr(src, dest); }
129
or32(Imm32 imm,Register dest)130 void MacroAssembler::or32(Imm32 imm, Register dest) {
131 ScratchRegisterScope scratch(*this);
132 ma_orr(imm, dest, scratch);
133 }
134
or32(Imm32 imm,const Address & dest)135 void MacroAssembler::or32(Imm32 imm, const Address& dest) {
136 ScratchRegisterScope scratch(*this);
137 SecondScratchRegisterScope scratch2(*this);
138
139 ma_ldr(dest, scratch, scratch2);
140 ma_orr(imm, scratch, scratch2);
141 ma_str(scratch, dest, scratch2);
142 }
143
orPtr(Register src,Register dest)144 void MacroAssembler::orPtr(Register src, Register dest) { ma_orr(src, dest); }
145
orPtr(Imm32 imm,Register dest)146 void MacroAssembler::orPtr(Imm32 imm, Register dest) {
147 ScratchRegisterScope scratch(*this);
148 ma_orr(imm, dest, scratch);
149 }
150
and64(Register64 src,Register64 dest)151 void MacroAssembler::and64(Register64 src, Register64 dest) {
152 and32(src.low, dest.low);
153 and32(src.high, dest.high);
154 }
155
or64(Register64 src,Register64 dest)156 void MacroAssembler::or64(Register64 src, Register64 dest) {
157 or32(src.low, dest.low);
158 or32(src.high, dest.high);
159 }
160
xor64(Register64 src,Register64 dest)161 void MacroAssembler::xor64(Register64 src, Register64 dest) {
162 ma_eor(src.low, dest.low);
163 ma_eor(src.high, dest.high);
164 }
165
xor32(Register src,Register dest)166 void MacroAssembler::xor32(Register src, Register dest) {
167 ma_eor(src, dest, SetCC);
168 }
169
xor32(Imm32 imm,Register dest)170 void MacroAssembler::xor32(Imm32 imm, Register dest) {
171 ScratchRegisterScope scratch(*this);
172 ma_eor(imm, dest, scratch, SetCC);
173 }
174
xorPtr(Register src,Register dest)175 void MacroAssembler::xorPtr(Register src, Register dest) { ma_eor(src, dest); }
176
xorPtr(Imm32 imm,Register dest)177 void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
178 ScratchRegisterScope scratch(*this);
179 ma_eor(imm, dest, scratch);
180 }
181
182 // ===============================================================
183 // Arithmetic functions
184
add32(Register src,Register dest)185 void MacroAssembler::add32(Register src, Register dest) {
186 ma_add(src, dest, SetCC);
187 }
188
add32(Imm32 imm,Register dest)189 void MacroAssembler::add32(Imm32 imm, Register dest) {
190 ScratchRegisterScope scratch(*this);
191 ma_add(imm, dest, scratch, SetCC);
192 }
193
add32(Imm32 imm,const Address & dest)194 void MacroAssembler::add32(Imm32 imm, const Address& dest) {
195 ScratchRegisterScope scratch(*this);
196 SecondScratchRegisterScope scratch2(*this);
197
198 ma_ldr(dest, scratch, scratch2);
199 ma_add(imm, scratch, scratch2, SetCC);
200 ma_str(scratch, dest, scratch2);
201 }
202
addPtr(Register src,Register dest)203 void MacroAssembler::addPtr(Register src, Register dest) { ma_add(src, dest); }
204
addPtr(Imm32 imm,Register dest)205 void MacroAssembler::addPtr(Imm32 imm, Register dest) {
206 ScratchRegisterScope scratch(*this);
207 ma_add(imm, dest, scratch);
208 }
209
addPtr(ImmWord imm,Register dest)210 void MacroAssembler::addPtr(ImmWord imm, Register dest) {
211 addPtr(Imm32(imm.value), dest);
212 }
213
addPtr(Imm32 imm,const Address & dest)214 void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
215 ScratchRegisterScope scratch(*this);
216 SecondScratchRegisterScope scratch2(*this);
217
218 ma_ldr(dest, scratch, scratch2);
219 ma_add(imm, scratch, scratch2);
220 ma_str(scratch, dest, scratch2);
221 }
222
addPtr(const Address & src,Register dest)223 void MacroAssembler::addPtr(const Address& src, Register dest) {
224 ScratchRegisterScope scratch(*this);
225 SecondScratchRegisterScope scratch2(*this);
226
227 ma_ldr(src, scratch, scratch2);
228 ma_add(scratch, dest, SetCC);
229 }
230
add64(Register64 src,Register64 dest)231 void MacroAssembler::add64(Register64 src, Register64 dest) {
232 ma_add(src.low, dest.low, SetCC);
233 ma_adc(src.high, dest.high);
234 }
235
add64(Imm32 imm,Register64 dest)236 void MacroAssembler::add64(Imm32 imm, Register64 dest) {
237 ScratchRegisterScope scratch(*this);
238 ma_add(imm, dest.low, scratch, SetCC);
239 as_adc(dest.high, dest.high, Imm8(0), LeaveCC);
240 }
241
add64(Imm64 imm,Register64 dest)242 void MacroAssembler::add64(Imm64 imm, Register64 dest) {
243 ScratchRegisterScope scratch(*this);
244 ma_add(imm.low(), dest.low, scratch, SetCC);
245 ma_adc(imm.hi(), dest.high, scratch, LeaveCC);
246 }
247
sub32FromStackPtrWithPatch(Register dest)248 CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
249 ScratchRegisterScope scratch(*this);
250 CodeOffset offs = CodeOffset(currentOffset());
251 ma_movPatchable(Imm32(0), scratch, Always);
252 ma_sub(getStackPointer(), scratch, dest);
253 return offs;
254 }
255
patchSub32FromStackPtr(CodeOffset offset,Imm32 imm)256 void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
257 ScratchRegisterScope scratch(*this);
258 BufferInstructionIterator iter(BufferOffset(offset.offset()), &m_buffer);
259 iter.maybeSkipAutomaticInstructions();
260 ma_mov_patch(imm, scratch, Always, HasMOVWT() ? L_MOVWT : L_LDR, iter);
261 }
262
addDouble(FloatRegister src,FloatRegister dest)263 void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
264 ma_vadd(dest, src, dest);
265 }
266
addFloat32(FloatRegister src,FloatRegister dest)267 void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
268 ma_vadd_f32(dest, src, dest);
269 }
270
sub32(Register src,Register dest)271 void MacroAssembler::sub32(Register src, Register dest) {
272 ma_sub(src, dest, SetCC);
273 }
274
sub32(Imm32 imm,Register dest)275 void MacroAssembler::sub32(Imm32 imm, Register dest) {
276 ScratchRegisterScope scratch(*this);
277 ma_sub(imm, dest, scratch, SetCC);
278 }
279
sub32(const Address & src,Register dest)280 void MacroAssembler::sub32(const Address& src, Register dest) {
281 ScratchRegisterScope scratch(*this);
282 SecondScratchRegisterScope scratch2(*this);
283
284 ma_ldr(src, scratch, scratch2);
285 ma_sub(scratch, dest, SetCC);
286 }
287
subPtr(Register src,Register dest)288 void MacroAssembler::subPtr(Register src, Register dest) { ma_sub(src, dest); }
289
subPtr(Register src,const Address & dest)290 void MacroAssembler::subPtr(Register src, const Address& dest) {
291 ScratchRegisterScope scratch(*this);
292 SecondScratchRegisterScope scratch2(*this);
293
294 ma_ldr(dest, scratch, scratch2);
295 ma_sub(src, scratch);
296 ma_str(scratch, dest, scratch2);
297 }
298
subPtr(Imm32 imm,Register dest)299 void MacroAssembler::subPtr(Imm32 imm, Register dest) {
300 ScratchRegisterScope scratch(*this);
301 ma_sub(imm, dest, scratch);
302 }
303
subPtr(const Address & addr,Register dest)304 void MacroAssembler::subPtr(const Address& addr, Register dest) {
305 ScratchRegisterScope scratch(*this);
306 SecondScratchRegisterScope scratch2(*this);
307
308 ma_ldr(addr, scratch, scratch2);
309 ma_sub(scratch, dest);
310 }
311
sub64(Register64 src,Register64 dest)312 void MacroAssembler::sub64(Register64 src, Register64 dest) {
313 ma_sub(src.low, dest.low, SetCC);
314 ma_sbc(src.high, dest.high, LeaveCC);
315 }
316
sub64(Imm64 imm,Register64 dest)317 void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
318 ScratchRegisterScope scratch(*this);
319 ma_sub(imm.low(), dest.low, scratch, SetCC);
320 ma_sbc(imm.hi(), dest.high, scratch, LeaveCC);
321 }
322
subDouble(FloatRegister src,FloatRegister dest)323 void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
324 ma_vsub(dest, src, dest);
325 }
326
subFloat32(FloatRegister src,FloatRegister dest)327 void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
328 ma_vsub_f32(dest, src, dest);
329 }
330
mul32(Register rhs,Register srcDest)331 void MacroAssembler::mul32(Register rhs, Register srcDest) {
332 as_mul(srcDest, srcDest, rhs);
333 }
334
mul64(Imm64 imm,const Register64 & dest)335 void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
336 // LOW32 = LOW(LOW(dest) * LOW(imm));
337 // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
338 // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
339 // + HIGH(LOW(dest) * LOW(imm)) [carry]
340
341 ScratchRegisterScope scratch(*this);
342 SecondScratchRegisterScope scratch2(*this);
343
344 // HIGH(dest) = LOW(HIGH(dest) * LOW(imm));
345 ma_mov(Imm32(imm.value & 0xFFFFFFFFL), scratch);
346 as_mul(dest.high, dest.high, scratch);
347
348 // high:low = LOW(dest) * LOW(imm);
349 as_umull(scratch2, scratch, dest.low, scratch);
350
351 // HIGH(dest) += high;
352 as_add(dest.high, dest.high, O2Reg(scratch2));
353
354 // HIGH(dest) += LOW(LOW(dest) * HIGH(imm));
355 if (((imm.value >> 32) & 0xFFFFFFFFL) == 5)
356 as_add(scratch2, dest.low, lsl(dest.low, 2));
357 else
358 MOZ_CRASH("Not supported imm");
359 as_add(dest.high, dest.high, O2Reg(scratch2));
360
361 // LOW(dest) = low;
362 ma_mov(scratch, dest.low);
363 }
364
mul64(Imm64 imm,const Register64 & dest,const Register temp)365 void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
366 const Register temp) {
367 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
368 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
369 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
370 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
371
372 MOZ_ASSERT(temp != dest.high && temp != dest.low);
373
374 // Compute mul64
375 ScratchRegisterScope scratch(*this);
376 ma_mul(dest.high, imm.low(), dest.high, scratch); // (2)
377 ma_mul(dest.low, imm.hi(), temp, scratch); // (3)
378 ma_add(dest.high, temp, temp);
379 ma_umull(dest.low, imm.low(), dest.high, dest.low, scratch); // (4) + (1)
380 ma_add(temp, dest.high, dest.high);
381 }
382
mul64(const Register64 & src,const Register64 & dest,const Register temp)383 void MacroAssembler::mul64(const Register64& src, const Register64& dest,
384 const Register temp) {
385 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
386 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
387 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
388 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
389
390 MOZ_ASSERT(dest != src);
391 MOZ_ASSERT(dest.low != src.high && dest.high != src.low);
392
393 // Compute mul64
394 ma_mul(dest.high, src.low, dest.high); // (2)
395 ma_mul(src.high, dest.low, temp); // (3)
396 ma_add(dest.high, temp, temp);
397 ma_umull(dest.low, src.low, dest.high, dest.low); // (4) + (1)
398 ma_add(temp, dest.high, dest.high);
399 }
400
mulBy3(Register src,Register dest)401 void MacroAssembler::mulBy3(Register src, Register dest) {
402 as_add(dest, src, lsl(src, 1));
403 }
404
mulFloat32(FloatRegister src,FloatRegister dest)405 void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
406 ma_vmul_f32(dest, src, dest);
407 }
408
mulDouble(FloatRegister src,FloatRegister dest)409 void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
410 ma_vmul(dest, src, dest);
411 }
412
mulDoublePtr(ImmPtr imm,Register temp,FloatRegister dest)413 void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
414 FloatRegister dest) {
415 ScratchRegisterScope scratch(*this);
416 ScratchDoubleScope scratchDouble(*this);
417
418 movePtr(imm, scratch);
419 ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
420 mulDouble(scratchDouble, dest);
421 }
422
quotient32(Register rhs,Register srcDest,bool isUnsigned)423 void MacroAssembler::quotient32(Register rhs, Register srcDest,
424 bool isUnsigned) {
425 MOZ_ASSERT(HasIDIV());
426 if (isUnsigned)
427 ma_udiv(srcDest, rhs, srcDest);
428 else
429 ma_sdiv(srcDest, rhs, srcDest);
430 }
431
remainder32(Register rhs,Register srcDest,bool isUnsigned)432 void MacroAssembler::remainder32(Register rhs, Register srcDest,
433 bool isUnsigned) {
434 MOZ_ASSERT(HasIDIV());
435
436 ScratchRegisterScope scratch(*this);
437 if (isUnsigned)
438 ma_umod(srcDest, rhs, srcDest, scratch);
439 else
440 ma_smod(srcDest, rhs, srcDest, scratch);
441 }
442
divFloat32(FloatRegister src,FloatRegister dest)443 void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
444 ma_vdiv_f32(dest, src, dest);
445 }
446
divDouble(FloatRegister src,FloatRegister dest)447 void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
448 ma_vdiv(dest, src, dest);
449 }
450
inc64(AbsoluteAddress dest)451 void MacroAssembler::inc64(AbsoluteAddress dest) {
452 ScratchRegisterScope scratch(*this);
453
454 ma_strd(r0, r1, EDtrAddr(sp, EDtrOffImm(-8)), PreIndex);
455
456 ma_mov(Imm32((int32_t)dest.addr), scratch);
457 ma_ldrd(EDtrAddr(scratch, EDtrOffImm(0)), r0, r1);
458
459 as_add(r0, r0, Imm8(1), SetCC);
460 as_adc(r1, r1, Imm8(0), LeaveCC);
461
462 ma_strd(r0, r1, EDtrAddr(scratch, EDtrOffImm(0)));
463 ma_ldrd(EDtrAddr(sp, EDtrOffImm(8)), r0, r1, PostIndex);
464 }
465
neg32(Register reg)466 void MacroAssembler::neg32(Register reg) { ma_neg(reg, reg, SetCC); }
467
neg64(Register64 reg)468 void MacroAssembler::neg64(Register64 reg) {
469 as_rsb(reg.low, reg.low, Imm8(0), SetCC);
470 as_rsc(reg.high, reg.high, Imm8(0));
471 }
472
negateDouble(FloatRegister reg)473 void MacroAssembler::negateDouble(FloatRegister reg) { ma_vneg(reg, reg); }
474
negateFloat(FloatRegister reg)475 void MacroAssembler::negateFloat(FloatRegister reg) { ma_vneg_f32(reg, reg); }
476
absFloat32(FloatRegister src,FloatRegister dest)477 void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
478 if (src != dest) ma_vmov_f32(src, dest);
479 ma_vabs_f32(dest, dest);
480 }
481
absDouble(FloatRegister src,FloatRegister dest)482 void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
483 if (src != dest) ma_vmov(src, dest);
484 ma_vabs(dest, dest);
485 }
486
sqrtFloat32(FloatRegister src,FloatRegister dest)487 void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
488 ma_vsqrt_f32(src, dest);
489 }
490
sqrtDouble(FloatRegister src,FloatRegister dest)491 void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
492 ma_vsqrt(src, dest);
493 }
494
minFloat32(FloatRegister other,FloatRegister srcDest,bool handleNaN)495 void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
496 bool handleNaN) {
497 minMaxFloat32(srcDest, other, handleNaN, false);
498 }
499
minDouble(FloatRegister other,FloatRegister srcDest,bool handleNaN)500 void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
501 bool handleNaN) {
502 minMaxDouble(srcDest, other, handleNaN, false);
503 }
504
maxFloat32(FloatRegister other,FloatRegister srcDest,bool handleNaN)505 void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
506 bool handleNaN) {
507 minMaxFloat32(srcDest, other, handleNaN, true);
508 }
509
maxDouble(FloatRegister other,FloatRegister srcDest,bool handleNaN)510 void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
511 bool handleNaN) {
512 minMaxDouble(srcDest, other, handleNaN, true);
513 }
514
515 // ===============================================================
516 // Shift functions
517
lshiftPtr(Imm32 imm,Register dest)518 void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
519 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
520 ma_lsl(imm, dest, dest);
521 }
522
lshift64(Imm32 imm,Register64 dest)523 void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
524 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
525 if (imm.value == 0) return;
526
527 if (imm.value < 32) {
528 as_mov(dest.high, lsl(dest.high, imm.value));
529 as_orr(dest.high, dest.high, lsr(dest.low, 32 - imm.value));
530 as_mov(dest.low, lsl(dest.low, imm.value));
531 } else {
532 as_mov(dest.high, lsl(dest.low, imm.value - 32));
533 ma_mov(Imm32(0), dest.low);
534 }
535 }
536
lshift64(Register unmaskedShift,Register64 dest)537 void MacroAssembler::lshift64(Register unmaskedShift, Register64 dest) {
538 // dest.high = dest.high << shift | dest.low << shift - 32 | dest.low >> 32 -
539 // shift Note: one of the two dest.low shift will always yield zero due to
540 // negative shift.
541
542 ScratchRegisterScope shift(*this);
543 as_and(shift, unmaskedShift, Imm8(0x3f));
544 as_mov(dest.high, lsl(dest.high, shift));
545 as_sub(shift, shift, Imm8(32));
546 as_orr(dest.high, dest.high, lsl(dest.low, shift));
547 ma_neg(shift, shift);
548 as_orr(dest.high, dest.high, lsr(dest.low, shift));
549 as_and(shift, unmaskedShift, Imm8(0x3f));
550 as_mov(dest.low, lsl(dest.low, shift));
551 }
552
lshift32(Register src,Register dest)553 void MacroAssembler::lshift32(Register src, Register dest) {
554 ma_lsl(src, dest, dest);
555 }
556
lshift32(Imm32 imm,Register dest)557 void MacroAssembler::lshift32(Imm32 imm, Register dest) {
558 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
559 lshiftPtr(imm, dest);
560 }
561
rshiftPtr(Imm32 imm,Register dest)562 void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
563 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
564 if (imm.value) ma_lsr(imm, dest, dest);
565 }
566
rshift32(Register src,Register dest)567 void MacroAssembler::rshift32(Register src, Register dest) {
568 ma_lsr(src, dest, dest);
569 }
570
rshift32(Imm32 imm,Register dest)571 void MacroAssembler::rshift32(Imm32 imm, Register dest) {
572 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
573 rshiftPtr(imm, dest);
574 }
575
rshiftPtrArithmetic(Imm32 imm,Register dest)576 void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
577 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
578 if (imm.value) ma_asr(imm, dest, dest);
579 }
580
rshift64Arithmetic(Imm32 imm,Register64 dest)581 void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
582 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
583 if (!imm.value) return;
584
585 if (imm.value < 32) {
586 as_mov(dest.low, lsr(dest.low, imm.value));
587 as_orr(dest.low, dest.low, lsl(dest.high, 32 - imm.value));
588 as_mov(dest.high, asr(dest.high, imm.value));
589 } else if (imm.value == 32) {
590 as_mov(dest.low, O2Reg(dest.high));
591 as_mov(dest.high, asr(dest.high, 31));
592 } else {
593 as_mov(dest.low, asr(dest.high, imm.value - 32));
594 as_mov(dest.high, asr(dest.high, 31));
595 }
596 }
597
rshift64Arithmetic(Register unmaskedShift,Register64 dest)598 void MacroAssembler::rshift64Arithmetic(Register unmaskedShift,
599 Register64 dest) {
600 Label proceed;
601
602 // dest.low = dest.low >>> shift | dest.high <<< 32 - shift
603 // if (shift - 32 >= 0)
604 // dest.low |= dest.high >>> shift - 32
605 // Note: Negative shifts yield a zero as result, except for the signed
606 // right shift. Therefore we need to test for it and only do it if
607 // it isn't negative.
608 ScratchRegisterScope shift(*this);
609
610 as_and(shift, unmaskedShift, Imm8(0x3f));
611 as_mov(dest.low, lsr(dest.low, shift));
612 as_rsb(shift, shift, Imm8(32));
613 as_orr(dest.low, dest.low, lsl(dest.high, shift));
614 ma_neg(shift, shift, SetCC);
615 ma_b(&proceed, Signed);
616
617 as_orr(dest.low, dest.low, asr(dest.high, shift));
618
619 bind(&proceed);
620 as_and(shift, unmaskedShift, Imm8(0x3f));
621 as_mov(dest.high, asr(dest.high, shift));
622 }
623
rshift32Arithmetic(Register src,Register dest)624 void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
625 ma_asr(src, dest, dest);
626 }
627
rshift32Arithmetic(Imm32 imm,Register dest)628 void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
629 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
630 rshiftPtrArithmetic(imm, dest);
631 }
632
rshift64(Imm32 imm,Register64 dest)633 void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
634 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
635 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
636 if (!imm.value) return;
637
638 if (imm.value < 32) {
639 as_mov(dest.low, lsr(dest.low, imm.value));
640 as_orr(dest.low, dest.low, lsl(dest.high, 32 - imm.value));
641 as_mov(dest.high, lsr(dest.high, imm.value));
642 } else if (imm.value == 32) {
643 ma_mov(dest.high, dest.low);
644 ma_mov(Imm32(0), dest.high);
645 } else {
646 ma_lsr(Imm32(imm.value - 32), dest.high, dest.low);
647 ma_mov(Imm32(0), dest.high);
648 }
649 }
650
rshift64(Register unmaskedShift,Register64 dest)651 void MacroAssembler::rshift64(Register unmaskedShift, Register64 dest) {
652 // dest.low = dest.low >> shift | dest.high >> shift - 32 | dest.high << 32 -
653 // shift Note: one of the two dest.high shifts will always yield zero due to
654 // negative shift.
655
656 ScratchRegisterScope shift(*this);
657 as_and(shift, unmaskedShift, Imm8(0x3f));
658 as_mov(dest.low, lsr(dest.low, shift));
659 as_sub(shift, shift, Imm8(32));
660 as_orr(dest.low, dest.low, lsr(dest.high, shift));
661 ma_neg(shift, shift);
662 as_orr(dest.low, dest.low, lsl(dest.high, shift));
663 as_and(shift, unmaskedShift, Imm8(0x3f));
664 as_mov(dest.high, lsr(dest.high, shift));
665 }
666
667 // ===============================================================
668 // Rotate functions
rotateLeft(Imm32 count,Register input,Register dest)669 void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
670 if (count.value)
671 ma_rol(count, input, dest);
672 else
673 ma_mov(input, dest);
674 }
675
rotateLeft(Register count,Register input,Register dest)676 void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
677 ScratchRegisterScope scratch(*this);
678 ma_rol(count, input, dest, scratch);
679 }
680
rotateLeft64(Imm32 count,Register64 input,Register64 dest,Register temp)681 void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
682 Register64 dest, Register temp) {
683 MOZ_ASSERT(temp == InvalidReg);
684 MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
685
686 int32_t amount = count.value & 0x3f;
687 if (amount > 32) {
688 rotateRight64(Imm32(64 - amount), input, dest, temp);
689 } else {
690 ScratchRegisterScope scratch(*this);
691 if (amount == 0) {
692 ma_mov(input.low, dest.low);
693 ma_mov(input.high, dest.high);
694 } else if (amount == 32) {
695 ma_mov(input.low, scratch);
696 ma_mov(input.high, dest.low);
697 ma_mov(scratch, dest.high);
698 } else {
699 MOZ_ASSERT(0 < amount && amount < 32);
700 ma_mov(dest.high, scratch);
701 as_mov(dest.high, lsl(dest.high, amount));
702 as_orr(dest.high, dest.high, lsr(dest.low, 32 - amount));
703 as_mov(dest.low, lsl(dest.low, amount));
704 as_orr(dest.low, dest.low, lsr(scratch, 32 - amount));
705 }
706 }
707 }
708
rotateLeft64(Register shift,Register64 src,Register64 dest,Register temp)709 void MacroAssembler::rotateLeft64(Register shift, Register64 src,
710 Register64 dest, Register temp) {
711 MOZ_ASSERT(shift != temp);
712 MOZ_ASSERT(src == dest);
713 MOZ_ASSERT(temp != src.low && temp != src.high);
714 MOZ_ASSERT(shift != src.low && shift != src.high);
715 MOZ_ASSERT(temp != InvalidReg);
716
717 ScratchRegisterScope shift_value(*this);
718 Label high, done;
719
720 ma_mov(src.high, temp);
721 as_and(shift_value, shift, Imm8(0x3f));
722 as_cmp(shift_value, Imm8(32));
723 ma_b(&high, GreaterThanOrEqual);
724
725 // high = high << shift | low >> 32 - shift
726 // low = low << shift | high >> 32 - shift
727 as_mov(dest.high, lsl(src.high, shift_value));
728 as_rsb(shift_value, shift_value, Imm8(32));
729 as_orr(dest.high, dest.high, lsr(src.low, shift_value));
730
731 as_rsb(shift_value, shift_value, Imm8(32));
732 as_mov(dest.low, lsl(src.low, shift_value));
733 as_rsb(shift_value, shift_value, Imm8(32));
734 as_orr(dest.low, dest.low, lsr(temp, shift_value));
735
736 ma_b(&done);
737
738 // A 32 - 64 shift is a 0 - 32 shift in the other direction.
739 bind(&high);
740 as_rsb(shift_value, shift_value, Imm8(64));
741
742 as_mov(dest.high, lsr(src.high, shift_value));
743 as_rsb(shift_value, shift_value, Imm8(32));
744 as_orr(dest.high, dest.high, lsl(src.low, shift_value));
745
746 as_rsb(shift_value, shift_value, Imm8(32));
747 as_mov(dest.low, lsr(src.low, shift_value));
748 as_rsb(shift_value, shift_value, Imm8(32));
749 as_orr(dest.low, dest.low, lsl(temp, shift_value));
750
751 bind(&done);
752 }
753
rotateRight(Imm32 count,Register input,Register dest)754 void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
755 if (count.value)
756 ma_ror(count, input, dest);
757 else
758 ma_mov(input, dest);
759 }
760
rotateRight(Register count,Register input,Register dest)761 void MacroAssembler::rotateRight(Register count, Register input,
762 Register dest) {
763 ma_ror(count, input, dest);
764 }
765
rotateRight64(Imm32 count,Register64 input,Register64 dest,Register temp)766 void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
767 Register64 dest, Register temp) {
768 MOZ_ASSERT(temp == InvalidReg);
769 MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
770
771 int32_t amount = count.value & 0x3f;
772 if (amount > 32) {
773 rotateLeft64(Imm32(64 - amount), input, dest, temp);
774 } else {
775 ScratchRegisterScope scratch(*this);
776 if (amount == 0) {
777 ma_mov(input.low, dest.low);
778 ma_mov(input.high, dest.high);
779 } else if (amount == 32) {
780 ma_mov(input.low, scratch);
781 ma_mov(input.high, dest.low);
782 ma_mov(scratch, dest.high);
783 } else {
784 MOZ_ASSERT(0 < amount && amount < 32);
785 ma_mov(dest.high, scratch);
786 as_mov(dest.high, lsr(dest.high, amount));
787 as_orr(dest.high, dest.high, lsl(dest.low, 32 - amount));
788 as_mov(dest.low, lsr(dest.low, amount));
789 as_orr(dest.low, dest.low, lsl(scratch, 32 - amount));
790 }
791 }
792 }
793
rotateRight64(Register shift,Register64 src,Register64 dest,Register temp)794 void MacroAssembler::rotateRight64(Register shift, Register64 src,
795 Register64 dest, Register temp) {
796 MOZ_ASSERT(shift != temp);
797 MOZ_ASSERT(src == dest);
798 MOZ_ASSERT(temp != src.low && temp != src.high);
799 MOZ_ASSERT(shift != src.low && shift != src.high);
800 MOZ_ASSERT(temp != InvalidReg);
801
802 ScratchRegisterScope shift_value(*this);
803 Label high, done;
804
805 ma_mov(src.high, temp);
806 as_and(shift_value, shift, Imm8(0x3f));
807 as_cmp(shift_value, Imm8(32));
808 ma_b(&high, GreaterThanOrEqual);
809
810 // high = high >> shift | low << 32 - shift
811 // low = low >> shift | high << 32 - shift
812 as_mov(dest.high, lsr(src.high, shift_value));
813 as_rsb(shift_value, shift_value, Imm8(32));
814 as_orr(dest.high, dest.high, lsl(src.low, shift_value));
815
816 as_rsb(shift_value, shift_value, Imm8(32));
817 as_mov(dest.low, lsr(src.low, shift_value));
818 as_rsb(shift_value, shift_value, Imm8(32));
819 as_orr(dest.low, dest.low, lsl(temp, shift_value));
820
821 ma_b(&done);
822
823 // A 32 - 64 shift is a 0 - 32 shift in the other direction.
824 bind(&high);
825 as_rsb(shift_value, shift_value, Imm8(64));
826
827 as_mov(dest.high, lsl(src.high, shift_value));
828 as_rsb(shift_value, shift_value, Imm8(32));
829 as_orr(dest.high, dest.high, lsr(src.low, shift_value));
830
831 as_rsb(shift_value, shift_value, Imm8(32));
832 as_mov(dest.low, lsl(src.low, shift_value));
833 as_rsb(shift_value, shift_value, Imm8(32));
834 as_orr(dest.low, dest.low, lsr(temp, shift_value));
835
836 bind(&done);
837 }
838
839 // ===============================================================
840 // Condition functions
841
842 template <typename T1, typename T2>
cmp32Set(Condition cond,T1 lhs,T2 rhs,Register dest)843 void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
844 cmp32(lhs, rhs);
845 emitSet(cond, dest);
846 }
847
848 template <typename T1, typename T2>
cmpPtrSet(Condition cond,T1 lhs,T2 rhs,Register dest)849 void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
850 cmpPtr(lhs, rhs);
851 emitSet(cond, dest);
852 }
853
854 // ===============================================================
855 // Bit counting functions
856
clz32(Register src,Register dest,bool knownNotZero)857 void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
858 ma_clz(src, dest);
859 }
860
clz64(Register64 src,Register dest)861 void MacroAssembler::clz64(Register64 src, Register dest) {
862 ScratchRegisterScope scratch(*this);
863
864 ma_clz(src.high, scratch);
865 as_cmp(scratch, Imm8(32));
866 ma_mov(scratch, dest, LeaveCC, NotEqual);
867 ma_clz(src.low, dest, Equal);
868 as_add(dest, dest, Imm8(32), LeaveCC, Equal);
869 }
870
ctz32(Register src,Register dest,bool knownNotZero)871 void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
872 ScratchRegisterScope scratch(*this);
873 ma_ctz(src, dest, scratch);
874 }
875
ctz64(Register64 src,Register dest)876 void MacroAssembler::ctz64(Register64 src, Register dest) {
877 Label done, high;
878 as_cmp(src.low, Imm8(0));
879 ma_b(&high, Equal);
880
881 ctz32(src.low, dest, /* knownNotZero = */ true);
882 ma_b(&done);
883
884 bind(&high);
885 ctz32(src.high, dest, /* knownNotZero = */ false);
886 as_add(dest, dest, Imm8(32));
887
888 bind(&done);
889 }
890
popcnt32(Register input,Register output,Register tmp)891 void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
892 // Equivalent to GCC output of mozilla::CountPopulation32()
893
894 ScratchRegisterScope scratch(*this);
895
896 if (input != output) ma_mov(input, output);
897 as_mov(tmp, asr(output, 1));
898 ma_and(Imm32(0x55555555), tmp, scratch);
899 ma_sub(output, tmp, output);
900 as_mov(tmp, asr(output, 2));
901 ma_mov(Imm32(0x33333333), scratch);
902 ma_and(scratch, output);
903 ma_and(scratch, tmp);
904 ma_add(output, tmp, output);
905 as_add(output, output, lsr(output, 4));
906 ma_and(Imm32(0xF0F0F0F), output, scratch);
907 as_add(output, output, lsl(output, 8));
908 as_add(output, output, lsl(output, 16));
909 as_mov(output, asr(output, 24));
910 }
911
popcnt64(Register64 src,Register64 dest,Register tmp)912 void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
913 MOZ_ASSERT(dest.low != tmp);
914 MOZ_ASSERT(dest.high != tmp);
915 MOZ_ASSERT(dest.low != dest.high);
916 // The source and destination can overlap. Therefore make sure we don't
917 // clobber the source before we have the data.
918 if (dest.low != src.high) {
919 popcnt32(src.low, dest.low, tmp);
920 popcnt32(src.high, dest.high, tmp);
921 } else {
922 MOZ_ASSERT(dest.high != src.high);
923 popcnt32(src.low, dest.high, tmp);
924 popcnt32(src.high, dest.low, tmp);
925 }
926 ma_add(dest.high, dest.low);
927 ma_mov(Imm32(0), dest.high);
928 }
929
930 // ===============================================================
931 // Branch functions
932
933 template <class L>
branch32(Condition cond,Register lhs,Register rhs,L label)934 void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
935 L label) {
936 ma_cmp(lhs, rhs);
937 ma_b(label, cond);
938 }
939
940 template <class L>
branch32(Condition cond,Register lhs,Imm32 rhs,L label)941 void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs,
942 L label) {
943 ScratchRegisterScope scratch(*this);
944
945 ma_cmp(lhs, rhs, scratch);
946 ma_b(label, cond);
947 }
948
branch32(Condition cond,const Address & lhs,Register rhs,Label * label)949 void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
950 Label* label) {
951 ScratchRegisterScope scratch(*this);
952 SecondScratchRegisterScope scratch2(*this);
953
954 ma_ldr(lhs, scratch, scratch2);
955 ma_cmp(scratch, rhs);
956 ma_b(label, cond);
957 }
958
branch32(Condition cond,const Address & lhs,Imm32 rhs,Label * label)959 void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
960 Label* label) {
961 ScratchRegisterScope scratch(*this);
962 SecondScratchRegisterScope scratch2(*this);
963
964 ma_ldr(lhs, scratch, scratch2);
965 ma_cmp(scratch, rhs, scratch2);
966 ma_b(label, cond);
967 }
968
branch32(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)969 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
970 Register rhs, Label* label) {
971 ScratchRegisterScope scratch(*this);
972
973 // Load into scratch.
974 movePtr(ImmWord(uintptr_t(lhs.addr)), scratch);
975 ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
976
977 ma_cmp(scratch, rhs);
978 ma_b(label, cond);
979 }
980
branch32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)981 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
982 Imm32 rhs, Label* label) {
983 ScratchRegisterScope scratch(*this);
984 SecondScratchRegisterScope scratch2(*this);
985
986 // Load into scratch.
987 movePtr(ImmWord(uintptr_t(lhs.addr)), scratch);
988 ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
989
990 ma_cmp(scratch, rhs, scratch2);
991 ma_b(label, cond);
992 }
993
branch32(Condition cond,const BaseIndex & lhs,Imm32 rhs,Label * label)994 void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
995 Label* label) {
996 SecondScratchRegisterScope scratch2(*this);
997 {
998 ScratchRegisterScope scratch(*this);
999
1000 Register base = lhs.base;
1001 uint32_t scale = Imm32::ShiftOf(lhs.scale).value;
1002
1003 // Load lhs into scratch2.
1004 if (lhs.offset != 0) {
1005 ma_add(base, Imm32(lhs.offset), scratch, scratch2);
1006 ma_ldr(DTRAddr(scratch, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
1007 } else {
1008 ma_ldr(DTRAddr(base, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
1009 }
1010 }
1011 branch32(cond, scratch2, rhs, label);
1012 }
1013
branch32(Condition cond,wasm::SymbolicAddress lhs,Imm32 rhs,Label * label)1014 void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
1015 Imm32 rhs, Label* label) {
1016 ScratchRegisterScope scratch(*this);
1017 SecondScratchRegisterScope scratch2(*this);
1018
1019 movePtr(lhs, scratch);
1020 ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
1021
1022 ma_cmp(scratch, rhs, scratch2);
1023 ma_b(label, cond);
1024 }
1025
branch64(Condition cond,const Address & lhs,Imm64 val,Label * label)1026 void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
1027 Label* label) {
1028 MOZ_ASSERT(cond == Assembler::NotEqual,
1029 "other condition codes not supported");
1030
1031 branch32(cond, lhs, val.firstHalf(), label);
1032 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
1033 val.secondHalf(), label);
1034 }
1035
branch64(Condition cond,const Address & lhs,const Address & rhs,Register scratch,Label * label)1036 void MacroAssembler::branch64(Condition cond, const Address& lhs,
1037 const Address& rhs, Register scratch,
1038 Label* label) {
1039 MOZ_ASSERT(cond == Assembler::NotEqual,
1040 "other condition codes not supported");
1041 MOZ_ASSERT(lhs.base != scratch);
1042 MOZ_ASSERT(rhs.base != scratch);
1043
1044 load32(rhs, scratch);
1045 branch32(cond, lhs, scratch, label);
1046
1047 load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
1048 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
1049 label);
1050 }
1051
branch64(Condition cond,Register64 lhs,Imm64 val,Label * success,Label * fail)1052 void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
1053 Label* success, Label* fail) {
1054 bool fallthrough = false;
1055 Label fallthroughLabel;
1056
1057 if (!fail) {
1058 fail = &fallthroughLabel;
1059 fallthrough = true;
1060 }
1061
1062 switch (cond) {
1063 case Assembler::Equal:
1064 branch32(Assembler::NotEqual, lhs.low, val.low(), fail);
1065 branch32(Assembler::Equal, lhs.high, val.hi(), success);
1066 if (!fallthrough) jump(fail);
1067 break;
1068 case Assembler::NotEqual:
1069 branch32(Assembler::NotEqual, lhs.low, val.low(), success);
1070 branch32(Assembler::NotEqual, lhs.high, val.hi(), success);
1071 if (!fallthrough) jump(fail);
1072 break;
1073 case Assembler::LessThan:
1074 case Assembler::LessThanOrEqual:
1075 case Assembler::GreaterThan:
1076 case Assembler::GreaterThanOrEqual:
1077 case Assembler::Below:
1078 case Assembler::BelowOrEqual:
1079 case Assembler::Above:
1080 case Assembler::AboveOrEqual: {
1081 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
1082 Assembler::Condition cond2 =
1083 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
1084 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
1085
1086 cmp32(lhs.high, val.hi());
1087 ma_b(success, cond1);
1088 ma_b(fail, cond2);
1089 cmp32(lhs.low, val.low());
1090 ma_b(success, cond3);
1091 if (!fallthrough) jump(fail);
1092 break;
1093 }
1094 default:
1095 MOZ_CRASH("Condition code not supported");
1096 break;
1097 }
1098
1099 if (fallthrough) bind(fail);
1100 }
1101
branch64(Condition cond,Register64 lhs,Register64 rhs,Label * success,Label * fail)1102 void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
1103 Label* success, Label* fail) {
1104 bool fallthrough = false;
1105 Label fallthroughLabel;
1106
1107 if (!fail) {
1108 fail = &fallthroughLabel;
1109 fallthrough = true;
1110 }
1111
1112 switch (cond) {
1113 case Assembler::Equal:
1114 branch32(Assembler::NotEqual, lhs.low, rhs.low, fail);
1115 branch32(Assembler::Equal, lhs.high, rhs.high, success);
1116 if (!fallthrough) jump(fail);
1117 break;
1118 case Assembler::NotEqual:
1119 branch32(Assembler::NotEqual, lhs.low, rhs.low, success);
1120 branch32(Assembler::NotEqual, lhs.high, rhs.high, success);
1121 if (!fallthrough) jump(fail);
1122 break;
1123 case Assembler::LessThan:
1124 case Assembler::LessThanOrEqual:
1125 case Assembler::GreaterThan:
1126 case Assembler::GreaterThanOrEqual:
1127 case Assembler::Below:
1128 case Assembler::BelowOrEqual:
1129 case Assembler::Above:
1130 case Assembler::AboveOrEqual: {
1131 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
1132 Assembler::Condition cond2 =
1133 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
1134 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
1135
1136 cmp32(lhs.high, rhs.high);
1137 ma_b(success, cond1);
1138 ma_b(fail, cond2);
1139 cmp32(lhs.low, rhs.low);
1140 ma_b(success, cond3);
1141 if (!fallthrough) jump(fail);
1142 break;
1143 }
1144 default:
1145 MOZ_CRASH("Condition code not supported");
1146 break;
1147 }
1148
1149 if (fallthrough) bind(fail);
1150 }
1151
1152 template <class L>
branchPtr(Condition cond,Register lhs,Register rhs,L label)1153 void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
1154 L label) {
1155 branch32(cond, lhs, rhs, label);
1156 }
1157
branchPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1158 void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
1159 Label* label) {
1160 branch32(cond, lhs, rhs, label);
1161 }
1162
branchPtr(Condition cond,Register lhs,ImmPtr rhs,Label * label)1163 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
1164 Label* label) {
1165 branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label);
1166 }
1167
branchPtr(Condition cond,Register lhs,ImmGCPtr rhs,Label * label)1168 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
1169 Label* label) {
1170 ScratchRegisterScope scratch(*this);
1171 movePtr(rhs, scratch);
1172 branchPtr(cond, lhs, scratch, label);
1173 }
1174
branchPtr(Condition cond,Register lhs,ImmWord rhs,Label * label)1175 void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
1176 Label* label) {
1177 branch32(cond, lhs, Imm32(rhs.value), label);
1178 }
1179
1180 template <class L>
branchPtr(Condition cond,const Address & lhs,Register rhs,L label)1181 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
1182 L label) {
1183 branch32(cond, lhs, rhs, label);
1184 }
1185
branchPtr(Condition cond,const Address & lhs,ImmPtr rhs,Label * label)1186 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
1187 Label* label) {
1188 branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label);
1189 }
1190
branchPtr(Condition cond,const Address & lhs,ImmGCPtr rhs,Label * label)1191 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
1192 Label* label) {
1193 SecondScratchRegisterScope scratch2(*this);
1194 loadPtr(lhs, scratch2);
1195 branchPtr(cond, scratch2, rhs, label);
1196 }
1197
branchPtr(Condition cond,const Address & lhs,ImmWord rhs,Label * label)1198 void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
1199 Label* label) {
1200 SecondScratchRegisterScope scratch2(*this);
1201 loadPtr(lhs, scratch2);
1202 branchPtr(cond, scratch2, rhs, label);
1203 }
1204
branchPtr(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)1205 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
1206 Register rhs, Label* label) {
1207 SecondScratchRegisterScope scratch2(*this);
1208 loadPtr(lhs, scratch2);
1209 branchPtr(cond, scratch2, rhs, label);
1210 }
1211
branchPtr(Condition cond,const AbsoluteAddress & lhs,ImmWord rhs,Label * label)1212 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
1213 ImmWord rhs, Label* label) {
1214 SecondScratchRegisterScope scratch2(*this);
1215 loadPtr(lhs, scratch2);
1216 branchPtr(cond, scratch2, rhs, label);
1217 }
1218
branchPtr(Condition cond,wasm::SymbolicAddress lhs,Register rhs,Label * label)1219 void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
1220 Register rhs, Label* label) {
1221 SecondScratchRegisterScope scratch2(*this);
1222 loadPtr(lhs, scratch2);
1223 branchPtr(cond, scratch2, rhs, label);
1224 }
1225
branchPtr(Condition cond,const BaseIndex & lhs,ImmWord rhs,Label * label)1226 void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
1227 ImmWord rhs, Label* label) {
1228 branch32(cond, lhs, Imm32(rhs.value), label);
1229 }
1230
1231 template <typename T>
branchPtrWithPatch(Condition cond,Register lhs,T rhs,RepatchLabel * label)1232 inline CodeOffsetJump MacroAssembler::branchPtrWithPatch(Condition cond,
1233 Register lhs, T rhs,
1234 RepatchLabel* label) {
1235 cmpPtr(lhs, rhs);
1236 return jumpWithPatch(label, cond);
1237 }
1238
1239 template <typename T>
branchPtrWithPatch(Condition cond,Address lhs,T rhs,RepatchLabel * label)1240 inline CodeOffsetJump MacroAssembler::branchPtrWithPatch(Condition cond,
1241 Address lhs, T rhs,
1242 RepatchLabel* label) {
1243 SecondScratchRegisterScope scratch2(*this);
1244 {
1245 ScratchRegisterScope scratch(*this);
1246 ma_ldr(lhs, scratch2, scratch);
1247 }
1248 cmpPtr(scratch2, rhs);
1249 return jumpWithPatch(label, cond);
1250 }
1251
branchPrivatePtr(Condition cond,const Address & lhs,Register rhs,Label * label)1252 void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
1253 Register rhs, Label* label) {
1254 branchPtr(cond, lhs, rhs, label);
1255 }
1256
branchFloat(DoubleCondition cond,FloatRegister lhs,FloatRegister rhs,Label * label)1257 void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
1258 FloatRegister rhs, Label* label) {
1259 compareFloat(lhs, rhs);
1260
1261 if (cond == DoubleNotEqual) {
1262 // Force the unordered cases not to jump.
1263 Label unordered;
1264 ma_b(&unordered, VFP_Unordered);
1265 ma_b(label, VFP_NotEqualOrUnordered);
1266 bind(&unordered);
1267 return;
1268 }
1269
1270 if (cond == DoubleEqualOrUnordered) {
1271 ma_b(label, VFP_Unordered);
1272 ma_b(label, VFP_Equal);
1273 return;
1274 }
1275
1276 ma_b(label, ConditionFromDoubleCondition(cond));
1277 }
1278
branchTruncateFloat32MaybeModUint32(FloatRegister src,Register dest,Label * fail)1279 void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
1280 Register dest,
1281 Label* fail) {
1282 branchTruncateFloat32ToInt32(src, dest, fail);
1283 }
1284
branchTruncateFloat32ToInt32(FloatRegister src,Register dest,Label * fail)1285 void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
1286 Register dest, Label* fail) {
1287 ScratchFloat32Scope scratchFloat32(*this);
1288 ScratchRegisterScope scratch(*this);
1289
1290 ma_vcvt_F32_I32(src, scratchFloat32.sintOverlay());
1291 ma_vxfer(scratchFloat32, dest);
1292 ma_cmp(dest, Imm32(0x7fffffff), scratch);
1293 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
1294 ma_b(fail, Assembler::Equal);
1295 }
1296
branchDouble(DoubleCondition cond,FloatRegister lhs,FloatRegister rhs,Label * label)1297 void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
1298 FloatRegister rhs, Label* label) {
1299 compareDouble(lhs, rhs);
1300
1301 if (cond == DoubleNotEqual) {
1302 // Force the unordered cases not to jump.
1303 Label unordered;
1304 ma_b(&unordered, VFP_Unordered);
1305 ma_b(label, VFP_NotEqualOrUnordered);
1306 bind(&unordered);
1307 return;
1308 }
1309
1310 if (cond == DoubleEqualOrUnordered) {
1311 ma_b(label, VFP_Unordered);
1312 ma_b(label, VFP_Equal);
1313 return;
1314 }
1315
1316 ma_b(label, ConditionFromDoubleCondition(cond));
1317 }
1318
branchTruncateDoubleMaybeModUint32(FloatRegister src,Register dest,Label * fail)1319 void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
1320 Register dest,
1321 Label* fail) {
1322 branchTruncateDoubleToInt32(src, dest, fail);
1323 }
1324
1325 // There are two options for implementing branchTruncateDoubleToInt32:
1326 //
1327 // 1. Convert the floating point value to an integer, if it did not fit, then it
1328 // was clamped to INT_MIN/INT_MAX, and we can test it. NOTE: if the value
1329 // really was supposed to be INT_MAX / INT_MIN then it will be wrong.
1330 //
1331 // 2. Convert the floating point value to an integer, if it did not fit, then it
1332 // set one or two bits in the fpcsr. Check those.
branchTruncateDoubleToInt32(FloatRegister src,Register dest,Label * fail)1333 void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
1334 Register dest, Label* fail) {
1335 ScratchDoubleScope scratchDouble(*this);
1336 FloatRegister scratchSIntReg = scratchDouble.sintOverlay();
1337 ScratchRegisterScope scratch(*this);
1338
1339 ma_vcvt_F64_I32(src, scratchSIntReg);
1340 ma_vxfer(scratchSIntReg, dest);
1341 ma_cmp(dest, Imm32(0x7fffffff), scratch);
1342 ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
1343 ma_b(fail, Assembler::Equal);
1344 }
1345
1346 template <typename T, typename L>
branchAdd32(Condition cond,T src,Register dest,L label)1347 void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
1348 L label) {
1349 add32(src, dest);
1350 as_b(label, cond);
1351 }
1352
1353 template <typename T>
branchSub32(Condition cond,T src,Register dest,Label * label)1354 void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
1355 Label* label) {
1356 sub32(src, dest);
1357 j(cond, label);
1358 }
1359
decBranchPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1360 void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
1361 Label* label) {
1362 ScratchRegisterScope scratch(*this);
1363 ma_sub(rhs, lhs, scratch, SetCC);
1364 as_b(label, cond);
1365 }
1366
1367 template <class L>
branchTest32(Condition cond,Register lhs,Register rhs,L label)1368 void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
1369 L label) {
1370 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
1371 cond == NotSigned);
1372 // x86 likes test foo, foo rather than cmp foo, #0.
1373 // Convert the former into the latter.
1374 if (lhs == rhs && (cond == Zero || cond == NonZero))
1375 as_cmp(lhs, Imm8(0));
1376 else
1377 ma_tst(lhs, rhs);
1378 ma_b(label, cond);
1379 }
1380
1381 template <class L>
branchTest32(Condition cond,Register lhs,Imm32 rhs,L label)1382 void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
1383 L label) {
1384 MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
1385 cond == NotSigned);
1386 ScratchRegisterScope scratch(*this);
1387 ma_tst(lhs, rhs, scratch);
1388 ma_b(label, cond);
1389 }
1390
branchTest32(Condition cond,const Address & lhs,Imm32 rhs,Label * label)1391 void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
1392 Label* label) {
1393 SecondScratchRegisterScope scratch2(*this);
1394 load32(lhs, scratch2);
1395 branchTest32(cond, scratch2, rhs, label);
1396 }
1397
branchTest32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)1398 void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
1399 Imm32 rhs, Label* label) {
1400 SecondScratchRegisterScope scratch2(*this);
1401 load32(lhs, scratch2);
1402 branchTest32(cond, scratch2, rhs, label);
1403 }
1404
1405 template <class L>
branchTestPtr(Condition cond,Register lhs,Register rhs,L label)1406 void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
1407 L label) {
1408 branchTest32(cond, lhs, rhs, label);
1409 }
1410
branchTestPtr(Condition cond,Register lhs,Imm32 rhs,Label * label)1411 void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
1412 Label* label) {
1413 branchTest32(cond, lhs, rhs, label);
1414 }
1415
branchTestPtr(Condition cond,const Address & lhs,Imm32 rhs,Label * label)1416 void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
1417 Imm32 rhs, Label* label) {
1418 branchTest32(cond, lhs, rhs, label);
1419 }
1420
1421 template <class L>
branchTest64(Condition cond,Register64 lhs,Register64 rhs,Register temp,L label)1422 void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
1423 Register64 rhs, Register temp, L label) {
1424 ScratchRegisterScope scratch(*this);
1425
1426 if (cond == Assembler::Zero || cond == Assembler::NonZero) {
1427 MOZ_ASSERT(lhs.low == rhs.low);
1428 MOZ_ASSERT(lhs.high == rhs.high);
1429 ma_orr(lhs.low, lhs.high, scratch);
1430 branchTestPtr(cond, scratch, scratch, label);
1431 } else {
1432 MOZ_CRASH("Unsupported condition");
1433 }
1434 }
1435
branchTestUndefined(Condition cond,Register tag,Label * label)1436 void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
1437 Label* label) {
1438 branchTestUndefinedImpl(cond, tag, label);
1439 }
1440
branchTestUndefined(Condition cond,const Address & address,Label * label)1441 void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
1442 Label* label) {
1443 branchTestUndefinedImpl(cond, address, label);
1444 }
1445
branchTestUndefined(Condition cond,const BaseIndex & address,Label * label)1446 void MacroAssembler::branchTestUndefined(Condition cond,
1447 const BaseIndex& address,
1448 Label* label) {
1449 branchTestUndefinedImpl(cond, address, label);
1450 }
1451
branchTestUndefined(Condition cond,const ValueOperand & value,Label * label)1452 void MacroAssembler::branchTestUndefined(Condition cond,
1453 const ValueOperand& value,
1454 Label* label) {
1455 branchTestUndefinedImpl(cond, value, label);
1456 }
1457
1458 template <typename T>
branchTestUndefinedImpl(Condition cond,const T & t,Label * label)1459 void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
1460 Label* label) {
1461 Condition c = testUndefined(cond, t);
1462 ma_b(label, c);
1463 }
1464
branchTestInt32(Condition cond,Register tag,Label * label)1465 void MacroAssembler::branchTestInt32(Condition cond, Register tag,
1466 Label* label) {
1467 branchTestInt32Impl(cond, tag, label);
1468 }
1469
branchTestInt32(Condition cond,const Address & address,Label * label)1470 void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
1471 Label* label) {
1472 branchTestInt32Impl(cond, address, label);
1473 }
1474
branchTestInt32(Condition cond,const BaseIndex & address,Label * label)1475 void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
1476 Label* label) {
1477 branchTestInt32Impl(cond, address, label);
1478 }
1479
branchTestInt32(Condition cond,const ValueOperand & value,Label * label)1480 void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
1481 Label* label) {
1482 branchTestInt32Impl(cond, value, label);
1483 }
1484
1485 template <typename T>
branchTestInt32Impl(Condition cond,const T & t,Label * label)1486 void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
1487 Label* label) {
1488 Condition c = testInt32(cond, t);
1489 ma_b(label, c);
1490 }
1491
branchTestInt32Truthy(bool truthy,const ValueOperand & value,Label * label)1492 void MacroAssembler::branchTestInt32Truthy(bool truthy,
1493 const ValueOperand& value,
1494 Label* label) {
1495 Condition c = testInt32Truthy(truthy, value);
1496 ma_b(label, c);
1497 }
1498
branchTestDouble(Condition cond,Register tag,Label * label)1499 void MacroAssembler::branchTestDouble(Condition cond, Register tag,
1500 Label* label) {
1501 branchTestDoubleImpl(cond, tag, label);
1502 }
1503
branchTestDouble(Condition cond,const Address & address,Label * label)1504 void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
1505 Label* label) {
1506 branchTestDoubleImpl(cond, address, label);
1507 }
1508
branchTestDouble(Condition cond,const BaseIndex & address,Label * label)1509 void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
1510 Label* label) {
1511 branchTestDoubleImpl(cond, address, label);
1512 }
1513
branchTestDouble(Condition cond,const ValueOperand & value,Label * label)1514 void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
1515 Label* label) {
1516 branchTestDoubleImpl(cond, value, label);
1517 }
1518
1519 template <typename T>
branchTestDoubleImpl(Condition cond,const T & t,Label * label)1520 void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
1521 Label* label) {
1522 Condition c = testDouble(cond, t);
1523 ma_b(label, c);
1524 }
1525
branchTestDoubleTruthy(bool truthy,FloatRegister reg,Label * label)1526 void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
1527 Label* label) {
1528 Condition c = testDoubleTruthy(truthy, reg);
1529 ma_b(label, c);
1530 }
1531
branchTestNumber(Condition cond,Register tag,Label * label)1532 void MacroAssembler::branchTestNumber(Condition cond, Register tag,
1533 Label* label) {
1534 branchTestNumberImpl(cond, tag, label);
1535 }
1536
branchTestNumber(Condition cond,const ValueOperand & value,Label * label)1537 void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
1538 Label* label) {
1539 branchTestNumberImpl(cond, value, label);
1540 }
1541
1542 template <typename T>
branchTestNumberImpl(Condition cond,const T & t,Label * label)1543 void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
1544 Label* label) {
1545 cond = testNumber(cond, t);
1546 ma_b(label, cond);
1547 }
1548
branchTestBoolean(Condition cond,Register tag,Label * label)1549 void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
1550 Label* label) {
1551 branchTestBooleanImpl(cond, tag, label);
1552 }
1553
branchTestBoolean(Condition cond,const Address & address,Label * label)1554 void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
1555 Label* label) {
1556 branchTestBooleanImpl(cond, address, label);
1557 }
1558
branchTestBoolean(Condition cond,const BaseIndex & address,Label * label)1559 void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
1560 Label* label) {
1561 branchTestBooleanImpl(cond, address, label);
1562 }
1563
branchTestBoolean(Condition cond,const ValueOperand & value,Label * label)1564 void MacroAssembler::branchTestBoolean(Condition cond,
1565 const ValueOperand& value,
1566 Label* label) {
1567 branchTestBooleanImpl(cond, value, label);
1568 }
1569
1570 template <typename T>
branchTestBooleanImpl(Condition cond,const T & t,Label * label)1571 void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& t,
1572 Label* label) {
1573 Condition c = testBoolean(cond, t);
1574 ma_b(label, c);
1575 }
1576
branchTestBooleanTruthy(bool truthy,const ValueOperand & value,Label * label)1577 void MacroAssembler::branchTestBooleanTruthy(bool truthy,
1578 const ValueOperand& value,
1579 Label* label) {
1580 Condition c = testBooleanTruthy(truthy, value);
1581 ma_b(label, c);
1582 }
1583
branchTestString(Condition cond,Register tag,Label * label)1584 void MacroAssembler::branchTestString(Condition cond, Register tag,
1585 Label* label) {
1586 branchTestStringImpl(cond, tag, label);
1587 }
1588
branchTestString(Condition cond,const Address & address,Label * label)1589 void MacroAssembler::branchTestString(Condition cond, const Address& address,
1590 Label* label) {
1591 branchTestStringImpl(cond, address, label);
1592 }
1593
branchTestString(Condition cond,const BaseIndex & address,Label * label)1594 void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
1595 Label* label) {
1596 branchTestStringImpl(cond, address, label);
1597 }
1598
branchTestString(Condition cond,const ValueOperand & value,Label * label)1599 void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
1600 Label* label) {
1601 branchTestStringImpl(cond, value, label);
1602 }
1603
1604 template <typename T>
branchTestStringImpl(Condition cond,const T & t,Label * label)1605 void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
1606 Label* label) {
1607 Condition c = testString(cond, t);
1608 ma_b(label, c);
1609 }
1610
branchTestStringTruthy(bool truthy,const ValueOperand & value,Label * label)1611 void MacroAssembler::branchTestStringTruthy(bool truthy,
1612 const ValueOperand& value,
1613 Label* label) {
1614 Condition c = testStringTruthy(truthy, value);
1615 ma_b(label, c);
1616 }
1617
branchTestSymbol(Condition cond,Register tag,Label * label)1618 void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
1619 Label* label) {
1620 branchTestSymbolImpl(cond, tag, label);
1621 }
1622
branchTestSymbol(Condition cond,const BaseIndex & address,Label * label)1623 void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
1624 Label* label) {
1625 branchTestSymbolImpl(cond, address, label);
1626 }
1627
branchTestSymbol(Condition cond,const ValueOperand & value,Label * label)1628 void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
1629 Label* label) {
1630 branchTestSymbolImpl(cond, value, label);
1631 }
1632
1633 template <typename T>
branchTestSymbolImpl(Condition cond,const T & t,Label * label)1634 void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
1635 Label* label) {
1636 Condition c = testSymbol(cond, t);
1637 ma_b(label, c);
1638 }
1639
branchTestNull(Condition cond,Register tag,Label * label)1640 void MacroAssembler::branchTestNull(Condition cond, Register tag,
1641 Label* label) {
1642 branchTestNullImpl(cond, tag, label);
1643 }
1644
branchTestNull(Condition cond,const Address & address,Label * label)1645 void MacroAssembler::branchTestNull(Condition cond, const Address& address,
1646 Label* label) {
1647 branchTestNullImpl(cond, address, label);
1648 }
1649
branchTestNull(Condition cond,const BaseIndex & address,Label * label)1650 void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
1651 Label* label) {
1652 branchTestNullImpl(cond, address, label);
1653 }
1654
branchTestNull(Condition cond,const ValueOperand & value,Label * label)1655 void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
1656 Label* label) {
1657 branchTestNullImpl(cond, value, label);
1658 }
1659
1660 template <typename T>
branchTestNullImpl(Condition cond,const T & t,Label * label)1661 void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
1662 Label* label) {
1663 Condition c = testNull(cond, t);
1664 ma_b(label, c);
1665 }
1666
branchTestObject(Condition cond,Register tag,Label * label)1667 void MacroAssembler::branchTestObject(Condition cond, Register tag,
1668 Label* label) {
1669 branchTestObjectImpl(cond, tag, label);
1670 }
1671
branchTestObject(Condition cond,const Address & address,Label * label)1672 void MacroAssembler::branchTestObject(Condition cond, const Address& address,
1673 Label* label) {
1674 branchTestObjectImpl(cond, address, label);
1675 }
1676
branchTestObject(Condition cond,const BaseIndex & address,Label * label)1677 void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
1678 Label* label) {
1679 branchTestObjectImpl(cond, address, label);
1680 }
1681
branchTestObject(Condition cond,const ValueOperand & value,Label * label)1682 void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
1683 Label* label) {
1684 branchTestObjectImpl(cond, value, label);
1685 }
1686
1687 template <typename T>
branchTestObjectImpl(Condition cond,const T & t,Label * label)1688 void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
1689 Label* label) {
1690 Condition c = testObject(cond, t);
1691 ma_b(label, c);
1692 }
1693
branchTestGCThing(Condition cond,const Address & address,Label * label)1694 void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
1695 Label* label) {
1696 branchTestGCThingImpl(cond, address, label);
1697 }
1698
branchTestGCThing(Condition cond,const BaseIndex & address,Label * label)1699 void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
1700 Label* label) {
1701 branchTestGCThingImpl(cond, address, label);
1702 }
1703
1704 template <typename T>
branchTestGCThingImpl(Condition cond,const T & t,Label * label)1705 void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& t,
1706 Label* label) {
1707 Condition c = testGCThing(cond, t);
1708 ma_b(label, c);
1709 }
1710
branchTestPrimitive(Condition cond,Register tag,Label * label)1711 void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
1712 Label* label) {
1713 branchTestPrimitiveImpl(cond, tag, label);
1714 }
1715
branchTestPrimitive(Condition cond,const ValueOperand & value,Label * label)1716 void MacroAssembler::branchTestPrimitive(Condition cond,
1717 const ValueOperand& value,
1718 Label* label) {
1719 branchTestPrimitiveImpl(cond, value, label);
1720 }
1721
1722 template <typename T>
branchTestPrimitiveImpl(Condition cond,const T & t,Label * label)1723 void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
1724 Label* label) {
1725 Condition c = testPrimitive(cond, t);
1726 ma_b(label, c);
1727 }
1728
branchTestMagic(Condition cond,Register tag,Label * label)1729 void MacroAssembler::branchTestMagic(Condition cond, Register tag,
1730 Label* label) {
1731 branchTestMagicImpl(cond, tag, label);
1732 }
1733
branchTestMagic(Condition cond,const Address & address,Label * label)1734 void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
1735 Label* label) {
1736 branchTestMagicImpl(cond, address, label);
1737 }
1738
branchTestMagic(Condition cond,const BaseIndex & address,Label * label)1739 void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
1740 Label* label) {
1741 branchTestMagicImpl(cond, address, label);
1742 }
1743
1744 template <class L>
branchTestMagic(Condition cond,const ValueOperand & value,L label)1745 void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
1746 L label) {
1747 branchTestMagicImpl(cond, value, label);
1748 }
1749
1750 template <typename T, class L>
branchTestMagicImpl(Condition cond,const T & t,L label)1751 void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
1752 cond = testMagic(cond, t);
1753 ma_b(label, cond);
1754 }
1755
branchTestMagic(Condition cond,const Address & valaddr,JSWhyMagic why,Label * label)1756 void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
1757 JSWhyMagic why, Label* label) {
1758 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
1759
1760 Label notMagic;
1761 if (cond == Assembler::Equal)
1762 branchTestMagic(Assembler::NotEqual, valaddr, ¬Magic);
1763 else
1764 branchTestMagic(Assembler::NotEqual, valaddr, label);
1765
1766 branch32(cond, ToPayload(valaddr), Imm32(why), label);
1767 bind(¬Magic);
1768 }
1769
branchToComputedAddress(const BaseIndex & addr)1770 void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
1771 MOZ_ASSERT(addr.base == pc, "Unsupported jump from any other addresses.");
1772 MOZ_ASSERT(
1773 addr.offset == 0,
1774 "NYI: offsets from pc should be shifted by the number of instructions.");
1775
1776 Register base = addr.base;
1777 uint32_t scale = Imm32::ShiftOf(addr.scale).value;
1778
1779 ma_ldr(DTRAddr(base, DtrRegImmShift(addr.index, LSL, scale)), pc);
1780 // When loading from pc, the pc is shifted to the next instruction, we
1781 // add one extra instruction to accomodate for this shifted offset.
1782 breakpoint();
1783 }
1784
cmp32Move32(Condition cond,Register lhs,Register rhs,Register src,Register dest)1785 void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
1786 Register src, Register dest) {
1787 cmp32(lhs, rhs);
1788 ma_mov(src, dest, LeaveCC, cond);
1789 }
1790
cmp32MovePtr(Condition cond,Register lhs,Imm32 rhs,Register src,Register dest)1791 void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
1792 Register src, Register dest) {
1793 cmp32(lhs, rhs);
1794 ma_mov(src, dest, LeaveCC, cond);
1795 }
1796
cmp32Move32(Condition cond,Register lhs,const Address & rhs,Register src,Register dest)1797 void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
1798 const Address& rhs, Register src,
1799 Register dest) {
1800 ScratchRegisterScope scratch(*this);
1801 SecondScratchRegisterScope scratch2(*this);
1802 ma_ldr(rhs, scratch, scratch2);
1803 cmp32Move32(cond, lhs, scratch, src, dest);
1804 }
1805
test32LoadPtr(Condition cond,const Address & addr,Imm32 mask,const Address & src,Register dest)1806 void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
1807 Imm32 mask, const Address& src,
1808 Register dest) {
1809 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
1810 test32(addr, mask);
1811 ScratchRegisterScope scratch(*this);
1812 ma_ldr(src, dest, scratch, Offset, cond);
1813 }
1814
test32MovePtr(Condition cond,const Address & addr,Imm32 mask,Register src,Register dest)1815 void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
1816 Imm32 mask, Register src, Register dest) {
1817 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
1818 test32(addr, mask);
1819 ma_mov(src, dest, LeaveCC, cond);
1820 }
1821
spectreMovePtr(Condition cond,Register src,Register dest)1822 void MacroAssembler::spectreMovePtr(Condition cond, Register src,
1823 Register dest) {
1824 ma_mov(src, dest, LeaveCC, cond);
1825 }
1826
spectreZeroRegister(Condition cond,Register,Register dest)1827 void MacroAssembler::spectreZeroRegister(Condition cond, Register,
1828 Register dest) {
1829 ma_mov(Imm32(0), dest, cond);
1830 }
1831
spectreBoundsCheck32(Register index,Register length,Register maybeScratch,Label * failure)1832 void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
1833 Register maybeScratch,
1834 Label* failure) {
1835 MOZ_ASSERT(length != maybeScratch);
1836 MOZ_ASSERT(index != maybeScratch);
1837
1838 branch32(Assembler::BelowOrEqual, length, index, failure);
1839
1840 if (JitOptions.spectreIndexMasking)
1841 ma_mov(Imm32(0), index, Assembler::BelowOrEqual);
1842 }
1843
spectreBoundsCheck32(Register index,const Address & length,Register maybeScratch,Label * failure)1844 void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
1845 Register maybeScratch,
1846 Label* failure) {
1847 MOZ_ASSERT(index != length.base);
1848 MOZ_ASSERT(length.base != maybeScratch);
1849 MOZ_ASSERT(index != maybeScratch);
1850
1851 branch32(Assembler::BelowOrEqual, length, index, failure);
1852
1853 if (JitOptions.spectreIndexMasking)
1854 ma_mov(Imm32(0), index, Assembler::BelowOrEqual);
1855 }
1856
1857 // ========================================================================
1858 // Memory access primitives.
storeUncanonicalizedDouble(FloatRegister src,const Address & addr)1859 void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
1860 const Address& addr) {
1861 ScratchRegisterScope scratch(*this);
1862 ma_vstr(src, addr, scratch);
1863 }
storeUncanonicalizedDouble(FloatRegister src,const BaseIndex & addr)1864 void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
1865 const BaseIndex& addr) {
1866 ScratchRegisterScope scratch(*this);
1867 SecondScratchRegisterScope scratch2(*this);
1868 uint32_t scale = Imm32::ShiftOf(addr.scale).value;
1869 ma_vstr(src, addr.base, addr.index, scratch, scratch2, scale, addr.offset);
1870 }
1871
storeUncanonicalizedFloat32(FloatRegister src,const Address & addr)1872 void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
1873 const Address& addr) {
1874 ScratchRegisterScope scratch(*this);
1875 ma_vstr(src.asSingle(), addr, scratch);
1876 }
storeUncanonicalizedFloat32(FloatRegister src,const BaseIndex & addr)1877 void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
1878 const BaseIndex& addr) {
1879 ScratchRegisterScope scratch(*this);
1880 SecondScratchRegisterScope scratch2(*this);
1881 uint32_t scale = Imm32::ShiftOf(addr.scale).value;
1882 ma_vstr(src.asSingle(), addr.base, addr.index, scratch, scratch2, scale,
1883 addr.offset);
1884 }
1885
storeFloat32x3(FloatRegister src,const Address & dest)1886 void MacroAssembler::storeFloat32x3(FloatRegister src, const Address& dest) {
1887 MOZ_CRASH("NYI");
1888 }
storeFloat32x3(FloatRegister src,const BaseIndex & dest)1889 void MacroAssembler::storeFloat32x3(FloatRegister src, const BaseIndex& dest) {
1890 MOZ_CRASH("NYI");
1891 }
1892
memoryBarrier(MemoryBarrierBits barrier)1893 void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
1894 // On ARMv6 the optional argument (BarrierST, etc) is ignored.
1895 if (barrier == (MembarStoreStore | MembarSynchronizing))
1896 ma_dsb(BarrierST);
1897 else if (barrier & MembarSynchronizing)
1898 ma_dsb();
1899 else if (barrier == MembarStoreStore)
1900 ma_dmb(BarrierST);
1901 else if (barrier)
1902 ma_dmb();
1903 }
1904
1905 // ===============================================================
1906 // Clamping functions.
1907
clampIntToUint8(Register reg)1908 void MacroAssembler::clampIntToUint8(Register reg) {
1909 // Look at (reg >> 8) if it is 0, then reg shouldn't be clamped if it is
1910 // <0, then we want to clamp to 0, otherwise, we wish to clamp to 255
1911 ScratchRegisterScope scratch(*this);
1912 as_mov(scratch, asr(reg, 8), SetCC);
1913 ma_mov(Imm32(0xff), reg, NotEqual);
1914 ma_mov(Imm32(0), reg, Signed);
1915 }
1916
1917 // ========================================================================
1918 // wasm support
1919
1920 template <class L>
wasmBoundsCheck(Condition cond,Register index,Register boundsCheckLimit,L label)1921 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
1922 Register boundsCheckLimit, L label) {
1923 as_cmp(index, O2Reg(boundsCheckLimit));
1924 as_b(label, cond);
1925 if (JitOptions.spectreIndexMasking)
1926 ma_mov(boundsCheckLimit, index, LeaveCC, cond);
1927 }
1928
1929 template <class L>
wasmBoundsCheck(Condition cond,Register index,Address boundsCheckLimit,L label)1930 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
1931 Address boundsCheckLimit, L label) {
1932 ScratchRegisterScope scratch(*this);
1933 MOZ_ASSERT(boundsCheckLimit.offset ==
1934 offsetof(wasm::TlsData, boundsCheckLimit));
1935 ma_ldr(DTRAddr(boundsCheckLimit.base, DtrOffImm(boundsCheckLimit.offset)),
1936 scratch);
1937 as_cmp(index, O2Reg(scratch));
1938 as_b(label, cond);
1939 if (JitOptions.spectreIndexMasking) ma_mov(scratch, index, LeaveCC, cond);
1940 }
1941
1942 //}}} check_macroassembler_style
1943 // ===============================================================
1944
incrementInt32Value(const Address & addr)1945 void MacroAssemblerARMCompat::incrementInt32Value(const Address& addr) {
1946 asMasm().add32(Imm32(1), ToPayload(addr));
1947 }
1948
1949 } // namespace jit
1950 } // namespace js
1951
1952 #endif /* jit_arm_MacroAssembler_arm_inl_h */
1953