1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_x86_MacroAssembler_x86_inl_h
8 #define jit_x86_MacroAssembler_x86_inl_h
9
10 #include "jit/x86/MacroAssembler-x86.h"
11
12 #include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
13
14 namespace js {
15 namespace jit {
16
17 //{{{ check_macroassembler_style
18
move64(Imm64 imm,Register64 dest)19 void MacroAssembler::move64(Imm64 imm, Register64 dest) {
20 move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
21 move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
22 }
23
move64(Register64 src,Register64 dest)24 void MacroAssembler::move64(Register64 src, Register64 dest) {
25 movl(src.low, dest.low);
26 movl(src.high, dest.high);
27 }
28
moveDoubleToGPR64(FloatRegister src,Register64 dest)29 void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
30 ScratchDoubleScope scratch(*this);
31
32 if (Assembler::HasSSE41()) {
33 vmovd(src, dest.low);
34 vpextrd(1, src, dest.high);
35 } else {
36 vmovd(src, dest.low);
37 moveDouble(src, scratch);
38 vpsrldq(Imm32(4), scratch, scratch);
39 vmovd(scratch, dest.high);
40 }
41 }
42
moveGPR64ToDouble(Register64 src,FloatRegister dest)43 void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
44 if (Assembler::HasSSE41()) {
45 vmovd(src.low, dest);
46 vpinsrd(1, src.high, dest, dest);
47 } else {
48 ScratchDoubleScope fpscratch(*this);
49 vmovd(src.low, dest);
50 vmovd(src.high, fpscratch);
51 vunpcklps(fpscratch, dest, dest);
52 }
53 }
54
move64To32(Register64 src,Register dest)55 void MacroAssembler::move64To32(Register64 src, Register dest) {
56 if (src.low != dest) {
57 movl(src.low, dest);
58 }
59 }
60
move32To64ZeroExtend(Register src,Register64 dest)61 void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
62 if (src != dest.low) {
63 movl(src, dest.low);
64 }
65 movl(Imm32(0), dest.high);
66 }
67
move8To64SignExtend(Register src,Register64 dest)68 void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
69 move8SignExtend(src, dest.low);
70 if (dest.low == eax && dest.high == edx) {
71 masm.cdq();
72 } else {
73 movl(dest.low, dest.high);
74 sarl(Imm32(31), dest.high);
75 }
76 }
77
move16To64SignExtend(Register src,Register64 dest)78 void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
79 move16SignExtend(src, dest.low);
80 if (dest.low == eax && dest.high == edx) {
81 masm.cdq();
82 } else {
83 movl(dest.low, dest.high);
84 sarl(Imm32(31), dest.high);
85 }
86 }
87
move32To64SignExtend(Register src,Register64 dest)88 void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
89 if (src != dest.low) {
90 movl(src, dest.low);
91 }
92 if (dest.low == eax && dest.high == edx) {
93 masm.cdq();
94 } else {
95 movl(dest.low, dest.high);
96 sarl(Imm32(31), dest.high);
97 }
98 }
99
move32SignExtendToPtr(Register src,Register dest)100 void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
101 movl(src, dest);
102 }
103
move32ZeroExtendToPtr(Register src,Register dest)104 void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
105 movl(src, dest);
106 }
107
108 // ===============================================================
109 // Load instructions
110
load32SignExtendToPtr(const Address & src,Register dest)111 void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
112 load32(src, dest);
113 }
114
115 // ===============================================================
116 // Logical functions
117
notPtr(Register reg)118 void MacroAssembler::notPtr(Register reg) { notl(reg); }
119
andPtr(Register src,Register dest)120 void MacroAssembler::andPtr(Register src, Register dest) { andl(src, dest); }
121
andPtr(Imm32 imm,Register dest)122 void MacroAssembler::andPtr(Imm32 imm, Register dest) { andl(imm, dest); }
123
and64(Imm64 imm,Register64 dest)124 void MacroAssembler::and64(Imm64 imm, Register64 dest) {
125 if (imm.low().value != int32_t(0xFFFFFFFF)) {
126 andl(imm.low(), dest.low);
127 }
128 if (imm.hi().value != int32_t(0xFFFFFFFF)) {
129 andl(imm.hi(), dest.high);
130 }
131 }
132
or64(Imm64 imm,Register64 dest)133 void MacroAssembler::or64(Imm64 imm, Register64 dest) {
134 if (imm.low().value != 0) {
135 orl(imm.low(), dest.low);
136 }
137 if (imm.hi().value != 0) {
138 orl(imm.hi(), dest.high);
139 }
140 }
141
xor64(Imm64 imm,Register64 dest)142 void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
143 if (imm.low().value != 0) {
144 xorl(imm.low(), dest.low);
145 }
146 if (imm.hi().value != 0) {
147 xorl(imm.hi(), dest.high);
148 }
149 }
150
orPtr(Register src,Register dest)151 void MacroAssembler::orPtr(Register src, Register dest) { orl(src, dest); }
152
orPtr(Imm32 imm,Register dest)153 void MacroAssembler::orPtr(Imm32 imm, Register dest) { orl(imm, dest); }
154
and64(Register64 src,Register64 dest)155 void MacroAssembler::and64(Register64 src, Register64 dest) {
156 andl(src.low, dest.low);
157 andl(src.high, dest.high);
158 }
159
or64(Register64 src,Register64 dest)160 void MacroAssembler::or64(Register64 src, Register64 dest) {
161 orl(src.low, dest.low);
162 orl(src.high, dest.high);
163 }
164
xor64(Register64 src,Register64 dest)165 void MacroAssembler::xor64(Register64 src, Register64 dest) {
166 xorl(src.low, dest.low);
167 xorl(src.high, dest.high);
168 }
169
xorPtr(Register src,Register dest)170 void MacroAssembler::xorPtr(Register src, Register dest) { xorl(src, dest); }
171
xorPtr(Imm32 imm,Register dest)172 void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorl(imm, dest); }
173
174 // ===============================================================
175 // Swap instructions
176
byteSwap64(Register64 reg)177 void MacroAssembler::byteSwap64(Register64 reg) {
178 bswapl(reg.low);
179 bswapl(reg.high);
180 xchgl(reg.low, reg.high);
181 }
182
183 // ===============================================================
184 // Arithmetic functions
185
addPtr(Register src,Register dest)186 void MacroAssembler::addPtr(Register src, Register dest) { addl(src, dest); }
187
addPtr(Imm32 imm,Register dest)188 void MacroAssembler::addPtr(Imm32 imm, Register dest) { addl(imm, dest); }
189
addPtr(ImmWord imm,Register dest)190 void MacroAssembler::addPtr(ImmWord imm, Register dest) {
191 addl(Imm32(imm.value), dest);
192 }
193
addPtr(Imm32 imm,const Address & dest)194 void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
195 addl(imm, Operand(dest));
196 }
197
addPtr(Imm32 imm,const AbsoluteAddress & dest)198 void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
199 addl(imm, Operand(dest));
200 }
201
addPtr(const Address & src,Register dest)202 void MacroAssembler::addPtr(const Address& src, Register dest) {
203 addl(Operand(src), dest);
204 }
205
add64(Register64 src,Register64 dest)206 void MacroAssembler::add64(Register64 src, Register64 dest) {
207 addl(src.low, dest.low);
208 adcl(src.high, dest.high);
209 }
210
add64(Imm32 imm,Register64 dest)211 void MacroAssembler::add64(Imm32 imm, Register64 dest) {
212 addl(imm, dest.low);
213 adcl(Imm32(0), dest.high);
214 }
215
add64(Imm64 imm,Register64 dest)216 void MacroAssembler::add64(Imm64 imm, Register64 dest) {
217 if (imm.low().value == 0) {
218 addl(imm.hi(), dest.high);
219 return;
220 }
221 addl(imm.low(), dest.low);
222 adcl(imm.hi(), dest.high);
223 }
224
addConstantDouble(double d,FloatRegister dest)225 void MacroAssembler::addConstantDouble(double d, FloatRegister dest) {
226 Double* dbl = getDouble(d);
227 if (!dbl) {
228 return;
229 }
230 masm.vaddsd_mr(nullptr, dest.encoding(), dest.encoding());
231 propagateOOM(dbl->uses.append(CodeOffset(masm.size())));
232 }
233
sub32FromStackPtrWithPatch(Register dest)234 CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
235 moveStackPtrTo(dest);
236 addlWithPatch(Imm32(0), dest);
237 return CodeOffset(currentOffset());
238 }
239
patchSub32FromStackPtr(CodeOffset offset,Imm32 imm)240 void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
241 patchAddl(offset, -imm.value);
242 }
243
subPtr(Register src,Register dest)244 void MacroAssembler::subPtr(Register src, Register dest) { subl(src, dest); }
245
subPtr(Register src,const Address & dest)246 void MacroAssembler::subPtr(Register src, const Address& dest) {
247 subl(src, Operand(dest));
248 }
249
subPtr(Imm32 imm,Register dest)250 void MacroAssembler::subPtr(Imm32 imm, Register dest) { subl(imm, dest); }
251
subPtr(const Address & addr,Register dest)252 void MacroAssembler::subPtr(const Address& addr, Register dest) {
253 subl(Operand(addr), dest);
254 }
255
sub64(Register64 src,Register64 dest)256 void MacroAssembler::sub64(Register64 src, Register64 dest) {
257 subl(src.low, dest.low);
258 sbbl(src.high, dest.high);
259 }
260
sub64(Imm64 imm,Register64 dest)261 void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
262 if (imm.low().value == 0) {
263 subl(imm.hi(), dest.high);
264 return;
265 }
266 subl(imm.low(), dest.low);
267 sbbl(imm.hi(), dest.high);
268 }
269
mulPtr(Register rhs,Register srcDest)270 void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
271 imull(rhs, srcDest);
272 }
273
274 // Note: this function clobbers eax and edx.
mul64(Imm64 imm,const Register64 & dest)275 void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
276 // LOW32 = LOW(LOW(dest) * LOW(imm));
277 // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
278 // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
279 // + HIGH(LOW(dest) * LOW(imm)) [carry]
280
281 MOZ_ASSERT(dest.low != eax && dest.low != edx);
282 MOZ_ASSERT(dest.high != eax && dest.high != edx);
283
284 // HIGH(dest) = LOW(HIGH(dest) * LOW(imm));
285 movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
286 imull(edx, dest.high);
287
288 // edx:eax = LOW(dest) * LOW(imm);
289 movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
290 movl(dest.low, eax);
291 mull(edx);
292
293 // HIGH(dest) += edx;
294 addl(edx, dest.high);
295
296 // HIGH(dest) += LOW(LOW(dest) * HIGH(imm));
297 if (((imm.value >> 32) & 0xFFFFFFFFL) == 5) {
298 leal(Operand(dest.low, dest.low, TimesFour), edx);
299 } else {
300 MOZ_CRASH("Unsupported imm");
301 }
302 addl(edx, dest.high);
303
304 // LOW(dest) = eax;
305 movl(eax, dest.low);
306 }
307
mul64(Imm64 imm,const Register64 & dest,const Register temp)308 void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
309 const Register temp) {
310 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
311 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
312 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
313 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
314
315 MOZ_ASSERT(dest == Register64(edx, eax));
316 MOZ_ASSERT(temp != edx && temp != eax);
317
318 movl(dest.low, temp);
319
320 // Compute mul64
321 imull(imm.low(), dest.high); // (2)
322 imull(imm.hi(), temp); // (3)
323 addl(dest.high, temp);
324 movl(imm.low(), dest.high);
325 mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
326 // (dest_hi:dest_lo)
327 addl(temp, dest.high);
328 }
329
mul64(const Register64 & src,const Register64 & dest,const Register temp)330 void MacroAssembler::mul64(const Register64& src, const Register64& dest,
331 const Register temp) {
332 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
333 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
334 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
335 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
336
337 MOZ_ASSERT(dest == Register64(edx, eax));
338 MOZ_ASSERT(src != Register64(edx, eax) && src != Register64(eax, edx));
339
340 // Make sure the rhs.high isn't the dest.high register anymore.
341 // This saves us from doing other register moves.
342 movl(dest.low, temp);
343
344 // Compute mul64
345 imull(src.low, dest.high); // (2)
346 imull(src.high, temp); // (3)
347 addl(dest.high, temp);
348 movl(src.low, dest.high);
349 mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
350 // (dest_hi:dest_lo)
351 addl(temp, dest.high);
352 }
353
mulBy3(Register src,Register dest)354 void MacroAssembler::mulBy3(Register src, Register dest) {
355 lea(Operand(src, src, TimesTwo), dest);
356 }
357
mulDoublePtr(ImmPtr imm,Register temp,FloatRegister dest)358 void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
359 FloatRegister dest) {
360 movl(imm, temp);
361 vmulsd(Operand(temp, 0), dest, dest);
362 }
363
inc64(AbsoluteAddress dest)364 void MacroAssembler::inc64(AbsoluteAddress dest) {
365 addl(Imm32(1), Operand(dest));
366 Label noOverflow;
367 j(NonZero, &noOverflow);
368 addl(Imm32(1), Operand(dest.offset(4)));
369 bind(&noOverflow);
370 }
371
neg64(Register64 reg)372 void MacroAssembler::neg64(Register64 reg) {
373 negl(reg.low);
374 adcl(Imm32(0), reg.high);
375 negl(reg.high);
376 }
377
negPtr(Register reg)378 void MacroAssembler::negPtr(Register reg) { negl(reg); }
379
380 // ===============================================================
381 // Shift functions
382
lshiftPtr(Imm32 imm,Register dest)383 void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
384 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
385 shll(imm, dest);
386 }
387
lshiftPtr(Register shift,Register srcDest)388 void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
389 if (HasBMI2()) {
390 shlxl(srcDest, shift, srcDest);
391 return;
392 }
393 MOZ_ASSERT(shift == ecx);
394 shll_cl(srcDest);
395 }
396
lshift64(Imm32 imm,Register64 dest)397 void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
398 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
399 if (imm.value < 32) {
400 shldl(imm, dest.low, dest.high);
401 shll(imm, dest.low);
402 return;
403 }
404
405 mov(dest.low, dest.high);
406 shll(Imm32(imm.value & 0x1f), dest.high);
407 xorl(dest.low, dest.low);
408 }
409
lshift64(Register shift,Register64 srcDest)410 void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
411 MOZ_ASSERT(shift == ecx);
412 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
413
414 Label done;
415
416 shldl_cl(srcDest.low, srcDest.high);
417 shll_cl(srcDest.low);
418
419 testl(Imm32(0x20), ecx);
420 j(Condition::Equal, &done);
421
422 // 32 - 63 bit shift
423 movl(srcDest.low, srcDest.high);
424 xorl(srcDest.low, srcDest.low);
425
426 bind(&done);
427 }
428
rshiftPtr(Imm32 imm,Register dest)429 void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
430 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
431 shrl(imm, dest);
432 }
433
rshiftPtr(Register shift,Register srcDest)434 void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
435 if (HasBMI2()) {
436 shrxl(srcDest, shift, srcDest);
437 return;
438 }
439 MOZ_ASSERT(shift == ecx);
440 shrl_cl(srcDest);
441 }
442
rshift64(Imm32 imm,Register64 dest)443 void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
444 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
445 if (imm.value < 32) {
446 shrdl(imm, dest.high, dest.low);
447 shrl(imm, dest.high);
448 return;
449 }
450
451 movl(dest.high, dest.low);
452 shrl(Imm32(imm.value & 0x1f), dest.low);
453 xorl(dest.high, dest.high);
454 }
455
rshift64(Register shift,Register64 srcDest)456 void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
457 MOZ_ASSERT(shift == ecx);
458 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
459
460 Label done;
461
462 shrdl_cl(srcDest.high, srcDest.low);
463 shrl_cl(srcDest.high);
464
465 testl(Imm32(0x20), ecx);
466 j(Condition::Equal, &done);
467
468 // 32 - 63 bit shift
469 movl(srcDest.high, srcDest.low);
470 xorl(srcDest.high, srcDest.high);
471
472 bind(&done);
473 }
474
rshiftPtrArithmetic(Imm32 imm,Register dest)475 void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
476 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
477 sarl(imm, dest);
478 }
479
rshift64Arithmetic(Imm32 imm,Register64 dest)480 void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
481 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
482 if (imm.value < 32) {
483 shrdl(imm, dest.high, dest.low);
484 sarl(imm, dest.high);
485 return;
486 }
487
488 movl(dest.high, dest.low);
489 sarl(Imm32(imm.value & 0x1f), dest.low);
490 sarl(Imm32(0x1f), dest.high);
491 }
492
rshift64Arithmetic(Register shift,Register64 srcDest)493 void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
494 MOZ_ASSERT(shift == ecx);
495 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
496
497 Label done;
498
499 shrdl_cl(srcDest.high, srcDest.low);
500 sarl_cl(srcDest.high);
501
502 testl(Imm32(0x20), ecx);
503 j(Condition::Equal, &done);
504
505 // 32 - 63 bit shift
506 movl(srcDest.high, srcDest.low);
507 sarl(Imm32(0x1f), srcDest.high);
508
509 bind(&done);
510 }
511
512 // ===============================================================
513 // Rotation functions
514
rotateLeft64(Register count,Register64 src,Register64 dest,Register temp)515 void MacroAssembler::rotateLeft64(Register count, Register64 src,
516 Register64 dest, Register temp) {
517 MOZ_ASSERT(src == dest, "defineReuseInput");
518 MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
519
520 Label done;
521
522 movl(dest.high, temp);
523 shldl_cl(dest.low, dest.high);
524 shldl_cl(temp, dest.low);
525
526 testl(Imm32(0x20), count);
527 j(Condition::Equal, &done);
528 xchgl(dest.high, dest.low);
529
530 bind(&done);
531 }
532
rotateRight64(Register count,Register64 src,Register64 dest,Register temp)533 void MacroAssembler::rotateRight64(Register count, Register64 src,
534 Register64 dest, Register temp) {
535 MOZ_ASSERT(src == dest, "defineReuseInput");
536 MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
537
538 Label done;
539
540 movl(dest.high, temp);
541 shrdl_cl(dest.low, dest.high);
542 shrdl_cl(temp, dest.low);
543
544 testl(Imm32(0x20), count);
545 j(Condition::Equal, &done);
546 xchgl(dest.high, dest.low);
547
548 bind(&done);
549 }
550
rotateLeft64(Imm32 count,Register64 src,Register64 dest,Register temp)551 void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
552 Register temp) {
553 MOZ_ASSERT(src == dest, "defineReuseInput");
554
555 int32_t amount = count.value & 0x3f;
556 if ((amount & 0x1f) != 0) {
557 movl(dest.high, temp);
558 shldl(Imm32(amount & 0x1f), dest.low, dest.high);
559 shldl(Imm32(amount & 0x1f), temp, dest.low);
560 }
561
562 if (!!(amount & 0x20)) {
563 xchgl(dest.high, dest.low);
564 }
565 }
566
rotateRight64(Imm32 count,Register64 src,Register64 dest,Register temp)567 void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
568 Register temp) {
569 MOZ_ASSERT(src == dest, "defineReuseInput");
570
571 int32_t amount = count.value & 0x3f;
572 if ((amount & 0x1f) != 0) {
573 movl(dest.high, temp);
574 shrdl(Imm32(amount & 0x1f), dest.low, dest.high);
575 shrdl(Imm32(amount & 0x1f), temp, dest.low);
576 }
577
578 if (!!(amount & 0x20)) {
579 xchgl(dest.high, dest.low);
580 }
581 }
582
583 // ===============================================================
584 // Bit counting functions
585
clz64(Register64 src,Register dest)586 void MacroAssembler::clz64(Register64 src, Register dest) {
587 if (AssemblerX86Shared::HasLZCNT()) {
588 Label nonzero, zero;
589
590 testl(src.high, src.high);
591 j(Assembler::Zero, &zero);
592
593 lzcntl(src.high, dest);
594 jump(&nonzero);
595
596 bind(&zero);
597 lzcntl(src.low, dest);
598 addl(Imm32(32), dest);
599
600 bind(&nonzero);
601 return;
602 }
603
604 // Because |dest| may be equal to |src.low|, we rely on BSR not modifying its
605 // output when the input is zero. AMD ISA documents BSR not modifying the
606 // output and current Intel CPUs follow AMD.
607
608 Label nonzero, zero;
609
610 bsrl(src.high, dest);
611 j(Assembler::Zero, &zero);
612 orl(Imm32(32), dest);
613 jump(&nonzero);
614
615 bind(&zero);
616 bsrl(src.low, dest);
617 j(Assembler::NonZero, &nonzero);
618 movl(Imm32(0x7F), dest);
619
620 bind(&nonzero);
621 xorl(Imm32(0x3F), dest);
622 }
623
ctz64(Register64 src,Register dest)624 void MacroAssembler::ctz64(Register64 src, Register dest) {
625 if (AssemblerX86Shared::HasBMI1()) {
626 Label nonzero, zero;
627
628 testl(src.low, src.low);
629 j(Assembler::Zero, &zero);
630
631 tzcntl(src.low, dest);
632 jump(&nonzero);
633
634 bind(&zero);
635 tzcntl(src.high, dest);
636 addl(Imm32(32), dest);
637
638 bind(&nonzero);
639 return;
640 }
641
642 // Because |dest| may be equal to |src.low|, we rely on BSF not modifying its
643 // output when the input is zero. AMD ISA documents BSF not modifying the
644 // output and current Intel CPUs follow AMD.
645
646 Label done, nonzero;
647
648 bsfl(src.low, dest);
649 j(Assembler::NonZero, &done);
650 bsfl(src.high, dest);
651 j(Assembler::NonZero, &nonzero);
652 movl(Imm32(64), dest);
653 jump(&done);
654
655 bind(&nonzero);
656 orl(Imm32(32), dest);
657
658 bind(&done);
659 }
660
popcnt64(Register64 src,Register64 dest,Register tmp)661 void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
662 // The tmp register is only needed if there is no native POPCNT.
663
664 MOZ_ASSERT(src.low != tmp && src.high != tmp);
665 MOZ_ASSERT(dest.low != tmp && dest.high != tmp);
666
667 if (dest.low != src.high) {
668 popcnt32(src.low, dest.low, tmp);
669 popcnt32(src.high, dest.high, tmp);
670 } else {
671 MOZ_ASSERT(dest.high != src.high);
672 popcnt32(src.low, dest.high, tmp);
673 popcnt32(src.high, dest.low, tmp);
674 }
675 addl(dest.high, dest.low);
676 xorl(dest.high, dest.high);
677 }
678
679 // ===============================================================
680 // Condition functions
681
682 template <typename T1, typename T2>
cmpPtrSet(Condition cond,T1 lhs,T2 rhs,Register dest)683 void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
684 cmpPtr(lhs, rhs);
685 emitSet(cond, dest);
686 }
687
688 // ===============================================================
689 // Branch functions
690
branch32(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)691 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
692 Register rhs, Label* label) {
693 cmp32(Operand(lhs), rhs);
694 j(cond, label);
695 }
696
branch32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)697 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
698 Imm32 rhs, Label* label) {
699 cmp32(Operand(lhs), rhs);
700 j(cond, label);
701 }
702
branch32(Condition cond,wasm::SymbolicAddress lhs,Imm32 rhs,Label * label)703 void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
704 Imm32 rhs, Label* label) {
705 cmpl(rhs, lhs);
706 j(cond, label);
707 }
708
branch64(Condition cond,Register64 lhs,Imm64 val,Label * success,Label * fail)709 void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
710 Label* success, Label* fail) {
711 bool fallthrough = false;
712 Label fallthroughLabel;
713
714 if (!fail) {
715 fail = &fallthroughLabel;
716 fallthrough = true;
717 }
718
719 switch (cond) {
720 case Assembler::Equal:
721 branch32(Assembler::NotEqual, lhs.low, val.low(), fail);
722 branch32(Assembler::Equal, lhs.high, val.hi(), success);
723 if (!fallthrough) {
724 jump(fail);
725 }
726 break;
727 case Assembler::NotEqual:
728 branch32(Assembler::NotEqual, lhs.low, val.low(), success);
729 branch32(Assembler::NotEqual, lhs.high, val.hi(), success);
730 if (!fallthrough) {
731 jump(fail);
732 }
733 break;
734 case Assembler::LessThan:
735 case Assembler::LessThanOrEqual:
736 case Assembler::GreaterThan:
737 case Assembler::GreaterThanOrEqual:
738 case Assembler::Below:
739 case Assembler::BelowOrEqual:
740 case Assembler::Above:
741 case Assembler::AboveOrEqual: {
742 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
743 Assembler::Condition cond2 =
744 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
745 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
746
747 cmp32(lhs.high, val.hi());
748 j(cond1, success);
749 j(cond2, fail);
750 cmp32(lhs.low, val.low());
751 j(cond3, success);
752 if (!fallthrough) {
753 jump(fail);
754 }
755 break;
756 }
757 default:
758 MOZ_CRASH("Condition code not supported");
759 break;
760 }
761
762 if (fallthrough) {
763 bind(fail);
764 }
765 }
766
branch64(Condition cond,Register64 lhs,Register64 rhs,Label * success,Label * fail)767 void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
768 Label* success, Label* fail) {
769 bool fallthrough = false;
770 Label fallthroughLabel;
771
772 if (!fail) {
773 fail = &fallthroughLabel;
774 fallthrough = true;
775 }
776
777 switch (cond) {
778 case Assembler::Equal:
779 branch32(Assembler::NotEqual, lhs.low, rhs.low, fail);
780 branch32(Assembler::Equal, lhs.high, rhs.high, success);
781 if (!fallthrough) {
782 jump(fail);
783 }
784 break;
785 case Assembler::NotEqual:
786 branch32(Assembler::NotEqual, lhs.low, rhs.low, success);
787 branch32(Assembler::NotEqual, lhs.high, rhs.high, success);
788 if (!fallthrough) {
789 jump(fail);
790 }
791 break;
792 case Assembler::LessThan:
793 case Assembler::LessThanOrEqual:
794 case Assembler::GreaterThan:
795 case Assembler::GreaterThanOrEqual:
796 case Assembler::Below:
797 case Assembler::BelowOrEqual:
798 case Assembler::Above:
799 case Assembler::AboveOrEqual: {
800 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
801 Assembler::Condition cond2 =
802 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
803 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
804
805 cmp32(lhs.high, rhs.high);
806 j(cond1, success);
807 j(cond2, fail);
808 cmp32(lhs.low, rhs.low);
809 j(cond3, success);
810 if (!fallthrough) {
811 jump(fail);
812 }
813 break;
814 }
815 default:
816 MOZ_CRASH("Condition code not supported");
817 break;
818 }
819
820 if (fallthrough) {
821 bind(fail);
822 }
823 }
824
branch64(Condition cond,const Address & lhs,Imm64 val,Label * label)825 void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
826 Label* label) {
827 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
828 "other condition codes not supported");
829
830 Label done;
831
832 if (cond == Assembler::Equal) {
833 branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done);
834 } else {
835 branch32(Assembler::NotEqual, lhs, val.firstHalf(), label);
836 }
837 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
838 val.secondHalf(), label);
839
840 bind(&done);
841 }
842
branch64(Condition cond,const Address & lhs,const Address & rhs,Register scratch,Label * label)843 void MacroAssembler::branch64(Condition cond, const Address& lhs,
844 const Address& rhs, Register scratch,
845 Label* label) {
846 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
847 "other condition codes not supported");
848 MOZ_ASSERT(lhs.base != scratch);
849 MOZ_ASSERT(rhs.base != scratch);
850
851 Label done;
852
853 load32(rhs, scratch);
854 if (cond == Assembler::Equal) {
855 branch32(Assembler::NotEqual, lhs, scratch, &done);
856 } else {
857 branch32(Assembler::NotEqual, lhs, scratch, label);
858 }
859
860 load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
861 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
862 label);
863
864 bind(&done);
865 }
866
branchPtr(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)867 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
868 Register rhs, Label* label) {
869 branchPtrImpl(cond, lhs, rhs, label);
870 }
871
branchPtr(Condition cond,const AbsoluteAddress & lhs,ImmWord rhs,Label * label)872 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
873 ImmWord rhs, Label* label) {
874 branchPtrImpl(cond, lhs, rhs, label);
875 }
876
branchPtr(Condition cond,wasm::SymbolicAddress lhs,Register rhs,Label * label)877 void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
878 Register rhs, Label* label) {
879 cmpl(rhs, lhs);
880 j(cond, label);
881 }
882
branchPrivatePtr(Condition cond,const Address & lhs,Register rhs,Label * label)883 void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
884 Register rhs, Label* label) {
885 branchPtr(cond, lhs, rhs, label);
886 }
887
branchTruncateFloat32ToPtr(FloatRegister src,Register dest,Label * fail)888 void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
889 Register dest, Label* fail) {
890 branchTruncateFloat32ToInt32(src, dest, fail);
891 }
892
branchTruncateFloat32MaybeModUint32(FloatRegister src,Register dest,Label * fail)893 void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
894 Register dest,
895 Label* fail) {
896 branchTruncateFloat32ToInt32(src, dest, fail);
897 }
898
branchTruncateFloat32ToInt32(FloatRegister src,Register dest,Label * fail)899 void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
900 Register dest, Label* fail) {
901 vcvttss2si(src, dest);
902
903 // vcvttss2si returns 0x80000000 on failure. Test for it by
904 // subtracting 1 and testing overflow (this permits the use of a
905 // smaller immediate field).
906 cmp32(dest, Imm32(1));
907 j(Assembler::Overflow, fail);
908 }
909
branchTruncateDoubleToPtr(FloatRegister src,Register dest,Label * fail)910 void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
911 Label* fail) {
912 branchTruncateDoubleToInt32(src, dest, fail);
913 }
914
branchTruncateDoubleMaybeModUint32(FloatRegister src,Register dest,Label * fail)915 void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
916 Register dest,
917 Label* fail) {
918 // TODO: X64 supports supports integers up till 64bits. Here we only support
919 // 32bits, before failing. Implementing this for x86 might give a x86 kraken
920 // win.
921 branchTruncateDoubleToInt32(src, dest, fail);
922 }
923
branchTruncateDoubleToInt32(FloatRegister src,Register dest,Label * fail)924 void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
925 Register dest, Label* fail) {
926 vcvttsd2si(src, dest);
927
928 // vcvttsd2si returns 0x80000000 on failure. Test for it by
929 // subtracting 1 and testing overflow (this permits the use of a
930 // smaller immediate field).
931 cmp32(dest, Imm32(1));
932 j(Assembler::Overflow, fail);
933 }
934
branchTest32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)935 void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
936 Imm32 rhs, Label* label) {
937 test32(Operand(lhs), rhs);
938 j(cond, label);
939 }
940
941 template <class L>
branchTest64(Condition cond,Register64 lhs,Register64 rhs,Register temp,L label)942 void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
943 Register64 rhs, Register temp, L label) {
944 if (cond == Assembler::Zero || cond == Assembler::NonZero) {
945 MOZ_ASSERT(lhs.low == rhs.low);
946 MOZ_ASSERT(lhs.high == rhs.high);
947 movl(lhs.low, temp);
948 orl(lhs.high, temp);
949 branchTestPtr(cond, temp, temp, label);
950 } else if (cond == Assembler::Signed || cond == Assembler::NotSigned) {
951 branchTest32(cond, lhs.high, rhs.high, label);
952 } else {
953 MOZ_CRASH("Unsupported condition");
954 }
955 }
956
branchTestBooleanTruthy(bool truthy,const ValueOperand & value,Label * label)957 void MacroAssembler::branchTestBooleanTruthy(bool truthy,
958 const ValueOperand& value,
959 Label* label) {
960 test32(value.payloadReg(), value.payloadReg());
961 j(truthy ? NonZero : Zero, label);
962 }
963
branchTestMagic(Condition cond,const Address & valaddr,JSWhyMagic why,Label * label)964 void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
965 JSWhyMagic why, Label* label) {
966 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
967
968 Label notMagic;
969 if (cond == Assembler::Equal) {
970 branchTestMagic(Assembler::NotEqual, valaddr, ¬Magic);
971 } else {
972 branchTestMagic(Assembler::NotEqual, valaddr, label);
973 }
974
975 branch32(cond, ToPayload(valaddr), Imm32(why), label);
976 bind(¬Magic);
977 }
978
branchTestValue(Condition cond,const BaseIndex & lhs,const ValueOperand & rhs,Label * label)979 void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
980 const ValueOperand& rhs, Label* label) {
981 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
982
983 Label notSameValue;
984 if (cond == Assembler::Equal) {
985 branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), ¬SameValue);
986 } else {
987 branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), label);
988 }
989
990 branch32(cond, ToPayload(lhs), rhs.payloadReg(), label);
991 bind(¬SameValue);
992 }
993
branchToComputedAddress(const BaseIndex & addr)994 void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
995 jmp(Operand(addr));
996 }
997
cmp32MovePtr(Condition cond,Register lhs,Imm32 rhs,Register src,Register dest)998 void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
999 Register src, Register dest) {
1000 cmp32(lhs, rhs);
1001 cmovCCl(cond, Operand(src), dest);
1002 }
1003
cmp32LoadPtr(Condition cond,const Address & lhs,Imm32 rhs,const Address & src,Register dest)1004 void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
1005 const Address& src, Register dest) {
1006 cmp32(lhs, rhs);
1007 cmovCCl(cond, Operand(src), dest);
1008 }
1009
cmpPtrMovePtr(Condition cond,Register lhs,Register rhs,Register src,Register dest)1010 void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
1011 Register src, Register dest) {
1012 cmp32Move32(cond, lhs, rhs, src, dest);
1013 }
1014
cmpPtrMovePtr(Condition cond,Register lhs,const Address & rhs,Register src,Register dest)1015 void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
1016 const Address& rhs, Register src,
1017 Register dest) {
1018 cmp32Move32(cond, lhs, rhs, src, dest);
1019 }
1020
test32LoadPtr(Condition cond,const Address & addr,Imm32 mask,const Address & src,Register dest)1021 void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
1022 Imm32 mask, const Address& src,
1023 Register dest) {
1024 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
1025 test32(addr, mask);
1026 cmovCCl(cond, Operand(src), dest);
1027 }
1028
test32MovePtr(Condition cond,const Address & addr,Imm32 mask,Register src,Register dest)1029 void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
1030 Imm32 mask, Register src, Register dest) {
1031 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
1032 test32(addr, mask);
1033 cmovCCl(cond, Operand(src), dest);
1034 }
1035
spectreMovePtr(Condition cond,Register src,Register dest)1036 void MacroAssembler::spectreMovePtr(Condition cond, Register src,
1037 Register dest) {
1038 cmovCCl(cond, Operand(src), dest);
1039 }
1040
spectreBoundsCheck32(Register index,const Operand & length,Register maybeScratch,Label * failure)1041 void MacroAssembler::spectreBoundsCheck32(Register index, const Operand& length,
1042 Register maybeScratch,
1043 Label* failure) {
1044 Label failurePopValue;
1045 bool pushedValue = false;
1046 if (JitOptions.spectreIndexMasking) {
1047 if (maybeScratch == InvalidReg) {
1048 push(Imm32(0));
1049 pushedValue = true;
1050 } else {
1051 move32(Imm32(0), maybeScratch);
1052 }
1053 }
1054
1055 cmp32(index, length);
1056 j(Assembler::AboveOrEqual, pushedValue ? &failurePopValue : failure);
1057
1058 if (JitOptions.spectreIndexMasking) {
1059 if (maybeScratch == InvalidReg) {
1060 Label done;
1061 cmovCCl(Assembler::AboveOrEqual, Operand(StackPointer, 0), index);
1062 lea(Operand(StackPointer, sizeof(void*)), StackPointer);
1063 jump(&done);
1064
1065 bind(&failurePopValue);
1066 lea(Operand(StackPointer, sizeof(void*)), StackPointer);
1067 jump(failure);
1068
1069 bind(&done);
1070 } else {
1071 cmovCCl(Assembler::AboveOrEqual, maybeScratch, index);
1072 }
1073 }
1074 }
1075
spectreBoundsCheck32(Register index,Register length,Register maybeScratch,Label * failure)1076 void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
1077 Register maybeScratch,
1078 Label* failure) {
1079 MOZ_ASSERT(length != maybeScratch);
1080 MOZ_ASSERT(index != maybeScratch);
1081
1082 spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
1083 }
1084
spectreBoundsCheck32(Register index,const Address & length,Register maybeScratch,Label * failure)1085 void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
1086 Register maybeScratch,
1087 Label* failure) {
1088 MOZ_ASSERT(index != length.base);
1089 MOZ_ASSERT(length.base != maybeScratch);
1090 MOZ_ASSERT(index != maybeScratch);
1091
1092 spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
1093 }
1094
spectreBoundsCheckPtr(Register index,Register length,Register maybeScratch,Label * failure)1095 void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
1096 Register maybeScratch,
1097 Label* failure) {
1098 spectreBoundsCheck32(index, length, maybeScratch, failure);
1099 }
1100
spectreBoundsCheckPtr(Register index,const Address & length,Register maybeScratch,Label * failure)1101 void MacroAssembler::spectreBoundsCheckPtr(Register index,
1102 const Address& length,
1103 Register maybeScratch,
1104 Label* failure) {
1105 spectreBoundsCheck32(index, length, maybeScratch, failure);
1106 }
1107
1108 // ========================================================================
1109 // SIMD
1110
anyTrueSimd128(FloatRegister src,Register dest)1111 void MacroAssembler::anyTrueSimd128(FloatRegister src, Register dest) {
1112 Label done;
1113 movl(Imm32(1), dest);
1114 vptest(src, src); // SSE4.1
1115 j(NonZero, &done);
1116 movl(Imm32(0), dest);
1117 bind(&done);
1118 }
1119
extractLaneInt64x2(uint32_t lane,FloatRegister src,Register64 dest)1120 void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
1121 Register64 dest) {
1122 vpextrd(2 * lane, src, dest.low);
1123 vpextrd(2 * lane + 1, src, dest.high);
1124 }
1125
replaceLaneInt64x2(unsigned lane,Register64 rhs,FloatRegister lhsDest)1126 void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
1127 FloatRegister lhsDest) {
1128 vpinsrd(2 * lane, rhs.low, lhsDest, lhsDest);
1129 vpinsrd(2 * lane + 1, rhs.high, lhsDest, lhsDest);
1130 }
1131
splatX2(Register64 src,FloatRegister dest)1132 void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
1133 replaceLaneInt64x2(0, src, dest);
1134 replaceLaneInt64x2(1, src, dest);
1135 }
1136
1137 // ========================================================================
1138 // Truncate floating point.
1139
truncateFloat32ToUInt64(Address src,Address dest,Register temp,FloatRegister floatTemp)1140 void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
1141 Register temp,
1142 FloatRegister floatTemp) {
1143 Label done;
1144
1145 loadFloat32(src, floatTemp);
1146
1147 truncateFloat32ToInt64(src, dest, temp);
1148
1149 // For unsigned conversion the case of [INT64, UINT64] needs to get handle
1150 // seperately.
1151 load32(HighWord(dest), temp);
1152 branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
1153
1154 // Move the value inside INT64 range.
1155 storeFloat32(floatTemp, dest);
1156 loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
1157 vaddss(Operand(dest), floatTemp, floatTemp);
1158 storeFloat32(floatTemp, dest);
1159 truncateFloat32ToInt64(dest, dest, temp);
1160
1161 load32(HighWord(dest), temp);
1162 orl(Imm32(0x80000000), temp);
1163 store32(temp, HighWord(dest));
1164
1165 bind(&done);
1166 }
1167
truncateDoubleToUInt64(Address src,Address dest,Register temp,FloatRegister floatTemp)1168 void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
1169 Register temp,
1170 FloatRegister floatTemp) {
1171 Label done;
1172
1173 loadDouble(src, floatTemp);
1174
1175 truncateDoubleToInt64(src, dest, temp);
1176
1177 // For unsigned conversion the case of [INT64, UINT64] needs to get handle
1178 // seperately.
1179 load32(HighWord(dest), temp);
1180 branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
1181
1182 // Move the value inside INT64 range.
1183 storeDouble(floatTemp, dest);
1184 loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
1185 vaddsd(Operand(dest), floatTemp, floatTemp);
1186 storeDouble(floatTemp, dest);
1187 truncateDoubleToInt64(dest, dest, temp);
1188
1189 load32(HighWord(dest), temp);
1190 orl(Imm32(0x80000000), temp);
1191 store32(temp, HighWord(dest));
1192
1193 bind(&done);
1194 }
1195
1196 template <typename T>
fallibleUnboxPtrImpl(const T & src,Register dest,JSValueType type,Label * fail)1197 void MacroAssemblerX86::fallibleUnboxPtrImpl(const T& src, Register dest,
1198 JSValueType type, Label* fail) {
1199 switch (type) {
1200 case JSVAL_TYPE_OBJECT:
1201 asMasm().branchTestObject(Assembler::NotEqual, src, fail);
1202 break;
1203 case JSVAL_TYPE_STRING:
1204 asMasm().branchTestString(Assembler::NotEqual, src, fail);
1205 break;
1206 case JSVAL_TYPE_SYMBOL:
1207 asMasm().branchTestSymbol(Assembler::NotEqual, src, fail);
1208 break;
1209 case JSVAL_TYPE_BIGINT:
1210 asMasm().branchTestBigInt(Assembler::NotEqual, src, fail);
1211 break;
1212 default:
1213 MOZ_CRASH("Unexpected type");
1214 }
1215 unboxNonDouble(src, dest, type);
1216 }
1217
fallibleUnboxPtr(const ValueOperand & src,Register dest,JSValueType type,Label * fail)1218 void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
1219 JSValueType type, Label* fail) {
1220 fallibleUnboxPtrImpl(src, dest, type, fail);
1221 }
1222
fallibleUnboxPtr(const Address & src,Register dest,JSValueType type,Label * fail)1223 void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
1224 JSValueType type, Label* fail) {
1225 fallibleUnboxPtrImpl(src, dest, type, fail);
1226 }
1227
fallibleUnboxPtr(const BaseIndex & src,Register dest,JSValueType type,Label * fail)1228 void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
1229 JSValueType type, Label* fail) {
1230 fallibleUnboxPtrImpl(src, dest, type, fail);
1231 }
1232
1233 //}}} check_macroassembler_style
1234 // ===============================================================
1235
1236 // Note: this function clobbers the source register.
convertUInt32ToDouble(Register src,FloatRegister dest)1237 void MacroAssemblerX86::convertUInt32ToDouble(Register src,
1238 FloatRegister dest) {
1239 // src is [0, 2^32-1]
1240 subl(Imm32(0x80000000), src);
1241
1242 // Now src is [-2^31, 2^31-1] - int range, but not the same value.
1243 convertInt32ToDouble(src, dest);
1244
1245 // dest is now a double with the int range.
1246 // correct the double value by adding 0x80000000.
1247 asMasm().addConstantDouble(2147483648.0, dest);
1248 }
1249
1250 // Note: this function clobbers the source register.
convertUInt32ToFloat32(Register src,FloatRegister dest)1251 void MacroAssemblerX86::convertUInt32ToFloat32(Register src,
1252 FloatRegister dest) {
1253 convertUInt32ToDouble(src, dest);
1254 convertDoubleToFloat32(dest, dest);
1255 }
1256
unboxValue(const ValueOperand & src,AnyRegister dest,JSValueType)1257 void MacroAssemblerX86::unboxValue(const ValueOperand& src, AnyRegister dest,
1258 JSValueType) {
1259 if (dest.isFloat()) {
1260 Label notInt32, end;
1261 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
1262 convertInt32ToDouble(src.payloadReg(), dest.fpu());
1263 jump(&end);
1264 bind(¬Int32);
1265 unboxDouble(src, dest.fpu());
1266 bind(&end);
1267 } else {
1268 if (src.payloadReg() != dest.gpr()) {
1269 movl(src.payloadReg(), dest.gpr());
1270 }
1271 }
1272 }
1273
1274 template <typename T>
loadInt32OrDouble(const T & src,FloatRegister dest)1275 void MacroAssemblerX86::loadInt32OrDouble(const T& src, FloatRegister dest) {
1276 Label notInt32, end;
1277 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
1278 convertInt32ToDouble(ToPayload(src), dest);
1279 jump(&end);
1280 bind(¬Int32);
1281 loadDouble(src, dest);
1282 bind(&end);
1283 }
1284
1285 template <typename T>
loadUnboxedValue(const T & src,MIRType type,AnyRegister dest)1286 void MacroAssemblerX86::loadUnboxedValue(const T& src, MIRType type,
1287 AnyRegister dest) {
1288 if (dest.isFloat()) {
1289 loadInt32OrDouble(src, dest.fpu());
1290 } else {
1291 movl(Operand(src), dest.gpr());
1292 }
1293 }
1294
1295 // If source is a double, load it into dest. If source is int32,
1296 // convert it to double. Else, branch to failure.
ensureDouble(const ValueOperand & source,FloatRegister dest,Label * failure)1297 void MacroAssemblerX86::ensureDouble(const ValueOperand& source,
1298 FloatRegister dest, Label* failure) {
1299 Label isDouble, done;
1300 asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
1301 asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
1302
1303 convertInt32ToDouble(source.payloadReg(), dest);
1304 jump(&done);
1305
1306 bind(&isDouble);
1307 unboxDouble(source, dest);
1308
1309 bind(&done);
1310 }
1311
1312 } // namespace jit
1313 } // namespace js
1314
1315 #endif /* jit_x86_MacroAssembler_x86_inl_h */
1316