1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_x86_MacroAssembler_x86_inl_h
8 #define jit_x86_MacroAssembler_x86_inl_h
9
10 #include "jit/x86/MacroAssembler-x86.h"
11
12 #include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
13
14 namespace js {
15 namespace jit {
16
17 //{{{ check_macroassembler_style
18
move64(Imm64 imm,Register64 dest)19 void MacroAssembler::move64(Imm64 imm, Register64 dest) {
20 movl(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
21 movl(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
22 }
23
move64(Register64 src,Register64 dest)24 void MacroAssembler::move64(Register64 src, Register64 dest) {
25 movl(src.low, dest.low);
26 movl(src.high, dest.high);
27 }
28
moveDoubleToGPR64(FloatRegister src,Register64 dest)29 void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
30 ScratchDoubleScope scratch(*this);
31
32 if (Assembler::HasSSE41()) {
33 vmovd(src, dest.low);
34 vpextrd(1, src, dest.high);
35 } else {
36 vmovd(src, dest.low);
37 moveDouble(src, scratch);
38 vpsrldq(Imm32(4), scratch, scratch);
39 vmovd(scratch, dest.high);
40 }
41 }
42
moveGPR64ToDouble(Register64 src,FloatRegister dest)43 void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
44 ScratchDoubleScope scratch(*this);
45
46 if (Assembler::HasSSE41()) {
47 vmovd(src.low, dest);
48 vpinsrd(1, src.high, dest, dest);
49 } else {
50 vmovd(src.low, dest);
51 vmovd(src.high, ScratchDoubleReg);
52 vunpcklps(ScratchDoubleReg, dest, dest);
53 }
54 }
55
move64To32(Register64 src,Register dest)56 void MacroAssembler::move64To32(Register64 src, Register dest) {
57 if (src.low != dest) movl(src.low, dest);
58 }
59
move32To64ZeroExtend(Register src,Register64 dest)60 void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
61 if (src != dest.low) movl(src, dest.low);
62 movl(Imm32(0), dest.high);
63 }
64
move8To64SignExtend(Register src,Register64 dest)65 void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
66 MOZ_ASSERT(dest.low == eax);
67 MOZ_ASSERT(dest.high == edx);
68 move8SignExtend(src, eax);
69 masm.cdq();
70 }
71
move16To64SignExtend(Register src,Register64 dest)72 void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
73 MOZ_ASSERT(dest.low == eax);
74 MOZ_ASSERT(dest.high == edx);
75 move16SignExtend(src, eax);
76 masm.cdq();
77 }
78
move32To64SignExtend(Register src,Register64 dest)79 void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
80 MOZ_ASSERT(dest.low == eax);
81 MOZ_ASSERT(dest.high == edx);
82 if (src != eax) movl(src, eax);
83 masm.cdq();
84 }
85
86 // ===============================================================
87 // Logical functions
88
andPtr(Register src,Register dest)89 void MacroAssembler::andPtr(Register src, Register dest) { andl(src, dest); }
90
andPtr(Imm32 imm,Register dest)91 void MacroAssembler::andPtr(Imm32 imm, Register dest) { andl(imm, dest); }
92
and64(Imm64 imm,Register64 dest)93 void MacroAssembler::and64(Imm64 imm, Register64 dest) {
94 if (imm.low().value != int32_t(0xFFFFFFFF)) andl(imm.low(), dest.low);
95 if (imm.hi().value != int32_t(0xFFFFFFFF)) andl(imm.hi(), dest.high);
96 }
97
or64(Imm64 imm,Register64 dest)98 void MacroAssembler::or64(Imm64 imm, Register64 dest) {
99 if (imm.low().value != 0) orl(imm.low(), dest.low);
100 if (imm.hi().value != 0) orl(imm.hi(), dest.high);
101 }
102
xor64(Imm64 imm,Register64 dest)103 void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
104 if (imm.low().value != 0) xorl(imm.low(), dest.low);
105 if (imm.hi().value != 0) xorl(imm.hi(), dest.high);
106 }
107
orPtr(Register src,Register dest)108 void MacroAssembler::orPtr(Register src, Register dest) { orl(src, dest); }
109
orPtr(Imm32 imm,Register dest)110 void MacroAssembler::orPtr(Imm32 imm, Register dest) { orl(imm, dest); }
111
and64(Register64 src,Register64 dest)112 void MacroAssembler::and64(Register64 src, Register64 dest) {
113 andl(src.low, dest.low);
114 andl(src.high, dest.high);
115 }
116
or64(Register64 src,Register64 dest)117 void MacroAssembler::or64(Register64 src, Register64 dest) {
118 orl(src.low, dest.low);
119 orl(src.high, dest.high);
120 }
121
xor64(Register64 src,Register64 dest)122 void MacroAssembler::xor64(Register64 src, Register64 dest) {
123 xorl(src.low, dest.low);
124 xorl(src.high, dest.high);
125 }
126
xorPtr(Register src,Register dest)127 void MacroAssembler::xorPtr(Register src, Register dest) { xorl(src, dest); }
128
xorPtr(Imm32 imm,Register dest)129 void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorl(imm, dest); }
130
131 // ===============================================================
132 // Arithmetic functions
133
addPtr(Register src,Register dest)134 void MacroAssembler::addPtr(Register src, Register dest) { addl(src, dest); }
135
addPtr(Imm32 imm,Register dest)136 void MacroAssembler::addPtr(Imm32 imm, Register dest) { addl(imm, dest); }
137
addPtr(ImmWord imm,Register dest)138 void MacroAssembler::addPtr(ImmWord imm, Register dest) {
139 addl(Imm32(imm.value), dest);
140 }
141
addPtr(Imm32 imm,const Address & dest)142 void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
143 addl(imm, Operand(dest));
144 }
145
addPtr(Imm32 imm,const AbsoluteAddress & dest)146 void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
147 addl(imm, Operand(dest));
148 }
149
addPtr(const Address & src,Register dest)150 void MacroAssembler::addPtr(const Address& src, Register dest) {
151 addl(Operand(src), dest);
152 }
153
add64(Register64 src,Register64 dest)154 void MacroAssembler::add64(Register64 src, Register64 dest) {
155 addl(src.low, dest.low);
156 adcl(src.high, dest.high);
157 }
158
add64(Imm32 imm,Register64 dest)159 void MacroAssembler::add64(Imm32 imm, Register64 dest) {
160 addl(imm, dest.low);
161 adcl(Imm32(0), dest.high);
162 }
163
add64(Imm64 imm,Register64 dest)164 void MacroAssembler::add64(Imm64 imm, Register64 dest) {
165 if (imm.low().value == 0) {
166 addl(imm.hi(), dest.high);
167 return;
168 }
169 addl(imm.low(), dest.low);
170 adcl(imm.hi(), dest.high);
171 }
172
addConstantDouble(double d,FloatRegister dest)173 void MacroAssembler::addConstantDouble(double d, FloatRegister dest) {
174 Double* dbl = getDouble(d);
175 if (!dbl) return;
176 masm.vaddsd_mr(nullptr, dest.encoding(), dest.encoding());
177 propagateOOM(dbl->uses.append(CodeOffset(masm.size())));
178 }
179
sub32FromStackPtrWithPatch(Register dest)180 CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
181 moveStackPtrTo(dest);
182 addlWithPatch(Imm32(0), dest);
183 return CodeOffset(currentOffset());
184 }
185
patchSub32FromStackPtr(CodeOffset offset,Imm32 imm)186 void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
187 patchAddl(offset, -imm.value);
188 }
189
subPtr(Register src,Register dest)190 void MacroAssembler::subPtr(Register src, Register dest) { subl(src, dest); }
191
subPtr(Register src,const Address & dest)192 void MacroAssembler::subPtr(Register src, const Address& dest) {
193 subl(src, Operand(dest));
194 }
195
subPtr(Imm32 imm,Register dest)196 void MacroAssembler::subPtr(Imm32 imm, Register dest) { subl(imm, dest); }
197
subPtr(const Address & addr,Register dest)198 void MacroAssembler::subPtr(const Address& addr, Register dest) {
199 subl(Operand(addr), dest);
200 }
201
sub64(Register64 src,Register64 dest)202 void MacroAssembler::sub64(Register64 src, Register64 dest) {
203 subl(src.low, dest.low);
204 sbbl(src.high, dest.high);
205 }
206
sub64(Imm64 imm,Register64 dest)207 void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
208 if (imm.low().value == 0) {
209 subl(imm.hi(), dest.high);
210 return;
211 }
212 subl(imm.low(), dest.low);
213 sbbl(imm.hi(), dest.high);
214 }
215
216 // Note: this function clobbers eax and edx.
mul64(Imm64 imm,const Register64 & dest)217 void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
218 // LOW32 = LOW(LOW(dest) * LOW(imm));
219 // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
220 // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
221 // + HIGH(LOW(dest) * LOW(imm)) [carry]
222
223 MOZ_ASSERT(dest.low != eax && dest.low != edx);
224 MOZ_ASSERT(dest.high != eax && dest.high != edx);
225
226 // HIGH(dest) = LOW(HIGH(dest) * LOW(imm));
227 movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
228 imull(edx, dest.high);
229
230 // edx:eax = LOW(dest) * LOW(imm);
231 movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
232 movl(dest.low, eax);
233 mull(edx);
234
235 // HIGH(dest) += edx;
236 addl(edx, dest.high);
237
238 // HIGH(dest) += LOW(LOW(dest) * HIGH(imm));
239 if (((imm.value >> 32) & 0xFFFFFFFFL) == 5)
240 leal(Operand(dest.low, dest.low, TimesFour), edx);
241 else
242 MOZ_CRASH("Unsupported imm");
243 addl(edx, dest.high);
244
245 // LOW(dest) = eax;
246 movl(eax, dest.low);
247 }
248
mul64(Imm64 imm,const Register64 & dest,const Register temp)249 void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
250 const Register temp) {
251 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
252 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
253 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
254 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
255
256 MOZ_ASSERT(dest == Register64(edx, eax));
257 MOZ_ASSERT(temp != edx && temp != eax);
258
259 movl(dest.low, temp);
260
261 // Compute mul64
262 imull(imm.low(), dest.high); // (2)
263 imull(imm.hi(), temp); // (3)
264 addl(dest.high, temp);
265 movl(imm.low(), dest.high);
266 mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
267 // (dest_hi:dest_lo)
268 addl(temp, dest.high);
269 }
270
mul64(const Register64 & src,const Register64 & dest,const Register temp)271 void MacroAssembler::mul64(const Register64& src, const Register64& dest,
272 const Register temp) {
273 // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
274 // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
275 // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
276 // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
277
278 MOZ_ASSERT(dest == Register64(edx, eax));
279 MOZ_ASSERT(src != Register64(edx, eax) && src != Register64(eax, edx));
280
281 // Make sure the rhs.high isn't the dest.high register anymore.
282 // This saves us from doing other register moves.
283 movl(dest.low, temp);
284
285 // Compute mul64
286 imull(src.low, dest.high); // (2)
287 imull(src.high, temp); // (3)
288 addl(dest.high, temp);
289 movl(src.low, dest.high);
290 mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
291 // (dest_hi:dest_lo)
292 addl(temp, dest.high);
293 }
294
mulBy3(Register src,Register dest)295 void MacroAssembler::mulBy3(Register src, Register dest) {
296 lea(Operand(src, src, TimesTwo), dest);
297 }
298
mulDoublePtr(ImmPtr imm,Register temp,FloatRegister dest)299 void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
300 FloatRegister dest) {
301 movl(imm, temp);
302 vmulsd(Operand(temp, 0), dest, dest);
303 }
304
inc64(AbsoluteAddress dest)305 void MacroAssembler::inc64(AbsoluteAddress dest) {
306 addl(Imm32(1), Operand(dest));
307 Label noOverflow;
308 j(NonZero, &noOverflow);
309 addl(Imm32(1), Operand(dest.offset(4)));
310 bind(&noOverflow);
311 }
312
neg64(Register64 reg)313 void MacroAssembler::neg64(Register64 reg) {
314 negl(reg.low);
315 adcl(Imm32(0), reg.high);
316 negl(reg.high);
317 }
318
319 // ===============================================================
320 // Shift functions
321
lshiftPtr(Imm32 imm,Register dest)322 void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
323 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
324 shll(imm, dest);
325 }
326
lshift64(Imm32 imm,Register64 dest)327 void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
328 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
329 if (imm.value < 32) {
330 shldl(imm, dest.low, dest.high);
331 shll(imm, dest.low);
332 return;
333 }
334
335 mov(dest.low, dest.high);
336 shll(Imm32(imm.value & 0x1f), dest.high);
337 xorl(dest.low, dest.low);
338 }
339
lshift64(Register shift,Register64 srcDest)340 void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
341 MOZ_ASSERT(shift == ecx);
342 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
343
344 Label done;
345
346 shldl_cl(srcDest.low, srcDest.high);
347 shll_cl(srcDest.low);
348
349 testl(Imm32(0x20), ecx);
350 j(Condition::Equal, &done);
351
352 // 32 - 63 bit shift
353 movl(srcDest.low, srcDest.high);
354 xorl(srcDest.low, srcDest.low);
355
356 bind(&done);
357 }
358
rshiftPtr(Imm32 imm,Register dest)359 void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
360 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
361 shrl(imm, dest);
362 }
363
rshift64(Imm32 imm,Register64 dest)364 void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
365 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
366 if (imm.value < 32) {
367 shrdl(imm, dest.high, dest.low);
368 shrl(imm, dest.high);
369 return;
370 }
371
372 movl(dest.high, dest.low);
373 shrl(Imm32(imm.value & 0x1f), dest.low);
374 xorl(dest.high, dest.high);
375 }
376
rshift64(Register shift,Register64 srcDest)377 void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
378 MOZ_ASSERT(shift == ecx);
379 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
380
381 Label done;
382
383 shrdl_cl(srcDest.high, srcDest.low);
384 shrl_cl(srcDest.high);
385
386 testl(Imm32(0x20), ecx);
387 j(Condition::Equal, &done);
388
389 // 32 - 63 bit shift
390 movl(srcDest.high, srcDest.low);
391 xorl(srcDest.high, srcDest.high);
392
393 bind(&done);
394 }
395
rshiftPtrArithmetic(Imm32 imm,Register dest)396 void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
397 MOZ_ASSERT(0 <= imm.value && imm.value < 32);
398 sarl(imm, dest);
399 }
400
rshift64Arithmetic(Imm32 imm,Register64 dest)401 void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
402 MOZ_ASSERT(0 <= imm.value && imm.value < 64);
403 if (imm.value < 32) {
404 shrdl(imm, dest.high, dest.low);
405 sarl(imm, dest.high);
406 return;
407 }
408
409 movl(dest.high, dest.low);
410 sarl(Imm32(imm.value & 0x1f), dest.low);
411 sarl(Imm32(0x1f), dest.high);
412 }
413
rshift64Arithmetic(Register shift,Register64 srcDest)414 void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
415 MOZ_ASSERT(shift == ecx);
416 MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
417
418 Label done;
419
420 shrdl_cl(srcDest.high, srcDest.low);
421 sarl_cl(srcDest.high);
422
423 testl(Imm32(0x20), ecx);
424 j(Condition::Equal, &done);
425
426 // 32 - 63 bit shift
427 movl(srcDest.high, srcDest.low);
428 sarl(Imm32(0x1f), srcDest.high);
429
430 bind(&done);
431 }
432
433 // ===============================================================
434 // Rotation functions
435
rotateLeft64(Register count,Register64 src,Register64 dest,Register temp)436 void MacroAssembler::rotateLeft64(Register count, Register64 src,
437 Register64 dest, Register temp) {
438 MOZ_ASSERT(src == dest, "defineReuseInput");
439 MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
440
441 Label done;
442
443 movl(dest.high, temp);
444 shldl_cl(dest.low, dest.high);
445 shldl_cl(temp, dest.low);
446
447 testl(Imm32(0x20), count);
448 j(Condition::Equal, &done);
449 xchgl(dest.high, dest.low);
450
451 bind(&done);
452 }
453
rotateRight64(Register count,Register64 src,Register64 dest,Register temp)454 void MacroAssembler::rotateRight64(Register count, Register64 src,
455 Register64 dest, Register temp) {
456 MOZ_ASSERT(src == dest, "defineReuseInput");
457 MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
458
459 Label done;
460
461 movl(dest.high, temp);
462 shrdl_cl(dest.low, dest.high);
463 shrdl_cl(temp, dest.low);
464
465 testl(Imm32(0x20), count);
466 j(Condition::Equal, &done);
467 xchgl(dest.high, dest.low);
468
469 bind(&done);
470 }
471
rotateLeft64(Imm32 count,Register64 src,Register64 dest,Register temp)472 void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
473 Register temp) {
474 MOZ_ASSERT(src == dest, "defineReuseInput");
475
476 int32_t amount = count.value & 0x3f;
477 if ((amount & 0x1f) != 0) {
478 movl(dest.high, temp);
479 shldl(Imm32(amount & 0x1f), dest.low, dest.high);
480 shldl(Imm32(amount & 0x1f), temp, dest.low);
481 }
482
483 if (!!(amount & 0x20)) xchgl(dest.high, dest.low);
484 }
485
rotateRight64(Imm32 count,Register64 src,Register64 dest,Register temp)486 void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
487 Register temp) {
488 MOZ_ASSERT(src == dest, "defineReuseInput");
489
490 int32_t amount = count.value & 0x3f;
491 if ((amount & 0x1f) != 0) {
492 movl(dest.high, temp);
493 shrdl(Imm32(amount & 0x1f), dest.low, dest.high);
494 shrdl(Imm32(amount & 0x1f), temp, dest.low);
495 }
496
497 if (!!(amount & 0x20)) xchgl(dest.high, dest.low);
498 }
499
500 // ===============================================================
501 // Bit counting functions
502
clz64(Register64 src,Register dest)503 void MacroAssembler::clz64(Register64 src, Register dest) {
504 Label nonzero, zero;
505
506 bsrl(src.high, dest);
507 j(Assembler::Zero, &zero);
508 orl(Imm32(32), dest);
509 jump(&nonzero);
510
511 bind(&zero);
512 bsrl(src.low, dest);
513 j(Assembler::NonZero, &nonzero);
514 movl(Imm32(0x7F), dest);
515
516 bind(&nonzero);
517 xorl(Imm32(0x3F), dest);
518 }
519
ctz64(Register64 src,Register dest)520 void MacroAssembler::ctz64(Register64 src, Register dest) {
521 Label done, nonzero;
522
523 bsfl(src.low, dest);
524 j(Assembler::NonZero, &done);
525 bsfl(src.high, dest);
526 j(Assembler::NonZero, &nonzero);
527 movl(Imm32(64), dest);
528 jump(&done);
529
530 bind(&nonzero);
531 orl(Imm32(32), dest);
532
533 bind(&done);
534 }
535
popcnt64(Register64 src,Register64 dest,Register tmp)536 void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
537 // The tmp register is only needed if there is no native POPCNT.
538
539 MOZ_ASSERT(src.low != tmp && src.high != tmp);
540 MOZ_ASSERT(dest.low != tmp && dest.high != tmp);
541
542 if (dest.low != src.high) {
543 popcnt32(src.low, dest.low, tmp);
544 popcnt32(src.high, dest.high, tmp);
545 } else {
546 MOZ_ASSERT(dest.high != src.high);
547 popcnt32(src.low, dest.high, tmp);
548 popcnt32(src.high, dest.low, tmp);
549 }
550 addl(dest.high, dest.low);
551 xorl(dest.high, dest.high);
552 }
553
554 // ===============================================================
555 // Condition functions
556
557 template <typename T1, typename T2>
cmpPtrSet(Condition cond,T1 lhs,T2 rhs,Register dest)558 void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
559 cmpPtr(lhs, rhs);
560 emitSet(cond, dest);
561 }
562
563 // ===============================================================
564 // Branch functions
565
branch32(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)566 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
567 Register rhs, Label* label) {
568 cmp32(Operand(lhs), rhs);
569 j(cond, label);
570 }
571
branch32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)572 void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
573 Imm32 rhs, Label* label) {
574 cmp32(Operand(lhs), rhs);
575 j(cond, label);
576 }
577
branch32(Condition cond,wasm::SymbolicAddress lhs,Imm32 rhs,Label * label)578 void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
579 Imm32 rhs, Label* label) {
580 cmpl(rhs, lhs);
581 j(cond, label);
582 }
583
branch64(Condition cond,Register64 lhs,Imm64 val,Label * success,Label * fail)584 void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
585 Label* success, Label* fail) {
586 bool fallthrough = false;
587 Label fallthroughLabel;
588
589 if (!fail) {
590 fail = &fallthroughLabel;
591 fallthrough = true;
592 }
593
594 switch (cond) {
595 case Assembler::Equal:
596 branch32(Assembler::NotEqual, lhs.low, val.low(), fail);
597 branch32(Assembler::Equal, lhs.high, val.hi(), success);
598 if (!fallthrough) jump(fail);
599 break;
600 case Assembler::NotEqual:
601 branch32(Assembler::NotEqual, lhs.low, val.low(), success);
602 branch32(Assembler::NotEqual, lhs.high, val.hi(), success);
603 if (!fallthrough) jump(fail);
604 break;
605 case Assembler::LessThan:
606 case Assembler::LessThanOrEqual:
607 case Assembler::GreaterThan:
608 case Assembler::GreaterThanOrEqual:
609 case Assembler::Below:
610 case Assembler::BelowOrEqual:
611 case Assembler::Above:
612 case Assembler::AboveOrEqual: {
613 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
614 Assembler::Condition cond2 =
615 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
616 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
617
618 cmp32(lhs.high, val.hi());
619 j(cond1, success);
620 j(cond2, fail);
621 cmp32(lhs.low, val.low());
622 j(cond3, success);
623 if (!fallthrough) jump(fail);
624 break;
625 }
626 default:
627 MOZ_CRASH("Condition code not supported");
628 break;
629 }
630
631 if (fallthrough) bind(fail);
632 }
633
branch64(Condition cond,Register64 lhs,Register64 rhs,Label * success,Label * fail)634 void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
635 Label* success, Label* fail) {
636 bool fallthrough = false;
637 Label fallthroughLabel;
638
639 if (!fail) {
640 fail = &fallthroughLabel;
641 fallthrough = true;
642 }
643
644 switch (cond) {
645 case Assembler::Equal:
646 branch32(Assembler::NotEqual, lhs.low, rhs.low, fail);
647 branch32(Assembler::Equal, lhs.high, rhs.high, success);
648 if (!fallthrough) jump(fail);
649 break;
650 case Assembler::NotEqual:
651 branch32(Assembler::NotEqual, lhs.low, rhs.low, success);
652 branch32(Assembler::NotEqual, lhs.high, rhs.high, success);
653 if (!fallthrough) jump(fail);
654 break;
655 case Assembler::LessThan:
656 case Assembler::LessThanOrEqual:
657 case Assembler::GreaterThan:
658 case Assembler::GreaterThanOrEqual:
659 case Assembler::Below:
660 case Assembler::BelowOrEqual:
661 case Assembler::Above:
662 case Assembler::AboveOrEqual: {
663 Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
664 Assembler::Condition cond2 =
665 Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
666 Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
667
668 cmp32(lhs.high, rhs.high);
669 j(cond1, success);
670 j(cond2, fail);
671 cmp32(lhs.low, rhs.low);
672 j(cond3, success);
673 if (!fallthrough) jump(fail);
674 break;
675 }
676 default:
677 MOZ_CRASH("Condition code not supported");
678 break;
679 }
680
681 if (fallthrough) bind(fail);
682 }
683
branch64(Condition cond,const Address & lhs,Imm64 val,Label * label)684 void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
685 Label* label) {
686 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
687 "other condition codes not supported");
688
689 Label done;
690
691 if (cond == Assembler::Equal)
692 branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done);
693 else
694 branch32(Assembler::NotEqual, lhs, val.firstHalf(), label);
695 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
696 val.secondHalf(), label);
697
698 bind(&done);
699 }
700
branch64(Condition cond,const Address & lhs,const Address & rhs,Register scratch,Label * label)701 void MacroAssembler::branch64(Condition cond, const Address& lhs,
702 const Address& rhs, Register scratch,
703 Label* label) {
704 MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
705 "other condition codes not supported");
706 MOZ_ASSERT(lhs.base != scratch);
707 MOZ_ASSERT(rhs.base != scratch);
708
709 Label done;
710
711 load32(rhs, scratch);
712 if (cond == Assembler::Equal)
713 branch32(Assembler::NotEqual, lhs, scratch, &done);
714 else
715 branch32(Assembler::NotEqual, lhs, scratch, label);
716
717 load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
718 branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
719 label);
720
721 bind(&done);
722 }
723
branchPtr(Condition cond,const AbsoluteAddress & lhs,Register rhs,Label * label)724 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
725 Register rhs, Label* label) {
726 branchPtrImpl(cond, lhs, rhs, label);
727 }
728
branchPtr(Condition cond,const AbsoluteAddress & lhs,ImmWord rhs,Label * label)729 void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
730 ImmWord rhs, Label* label) {
731 branchPtrImpl(cond, lhs, rhs, label);
732 }
733
branchPtr(Condition cond,wasm::SymbolicAddress lhs,Register rhs,Label * label)734 void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
735 Register rhs, Label* label) {
736 cmpl(rhs, lhs);
737 j(cond, label);
738 }
739
branchPrivatePtr(Condition cond,const Address & lhs,Register rhs,Label * label)740 void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
741 Register rhs, Label* label) {
742 branchPtr(cond, lhs, rhs, label);
743 }
744
branchTruncateFloat32ToPtr(FloatRegister src,Register dest,Label * fail)745 void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
746 Register dest, Label* fail) {
747 branchTruncateFloat32ToInt32(src, dest, fail);
748 }
749
branchTruncateFloat32MaybeModUint32(FloatRegister src,Register dest,Label * fail)750 void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
751 Register dest,
752 Label* fail) {
753 branchTruncateFloat32ToInt32(src, dest, fail);
754 }
755
branchTruncateFloat32ToInt32(FloatRegister src,Register dest,Label * fail)756 void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
757 Register dest, Label* fail) {
758 vcvttss2si(src, dest);
759
760 // vcvttss2si returns 0x80000000 on failure. Test for it by
761 // subtracting 1 and testing overflow (this permits the use of a
762 // smaller immediate field).
763 cmp32(dest, Imm32(1));
764 j(Assembler::Overflow, fail);
765 }
766
branchTruncateDoubleToPtr(FloatRegister src,Register dest,Label * fail)767 void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
768 Label* fail) {
769 branchTruncateDoubleToInt32(src, dest, fail);
770 }
771
branchTruncateDoubleMaybeModUint32(FloatRegister src,Register dest,Label * fail)772 void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
773 Register dest,
774 Label* fail) {
775 // TODO: X64 supports supports integers up till 64bits. Here we only support
776 // 32bits, before failing. Implementing this for x86 might give a x86 kraken
777 // win.
778 branchTruncateDoubleToInt32(src, dest, fail);
779 }
780
branchTruncateDoubleToInt32(FloatRegister src,Register dest,Label * fail)781 void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
782 Register dest, Label* fail) {
783 vcvttsd2si(src, dest);
784
785 // vcvttsd2si returns 0x80000000 on failure. Test for it by
786 // subtracting 1 and testing overflow (this permits the use of a
787 // smaller immediate field).
788 cmp32(dest, Imm32(1));
789 j(Assembler::Overflow, fail);
790 }
791
branchTest32(Condition cond,const AbsoluteAddress & lhs,Imm32 rhs,Label * label)792 void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
793 Imm32 rhs, Label* label) {
794 test32(Operand(lhs), rhs);
795 j(cond, label);
796 }
797
798 template <class L>
branchTest64(Condition cond,Register64 lhs,Register64 rhs,Register temp,L label)799 void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
800 Register64 rhs, Register temp, L label) {
801 if (cond == Assembler::Zero || cond == Assembler::NonZero) {
802 MOZ_ASSERT(lhs.low == rhs.low);
803 MOZ_ASSERT(lhs.high == rhs.high);
804 movl(lhs.low, temp);
805 orl(lhs.high, temp);
806 branchTestPtr(cond, temp, temp, label);
807 } else {
808 MOZ_CRASH("Unsupported condition");
809 }
810 }
811
branchTestBooleanTruthy(bool truthy,const ValueOperand & value,Label * label)812 void MacroAssembler::branchTestBooleanTruthy(bool truthy,
813 const ValueOperand& value,
814 Label* label) {
815 test32(value.payloadReg(), value.payloadReg());
816 j(truthy ? NonZero : Zero, label);
817 }
818
branchTestMagic(Condition cond,const Address & valaddr,JSWhyMagic why,Label * label)819 void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
820 JSWhyMagic why, Label* label) {
821 MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
822
823 Label notMagic;
824 if (cond == Assembler::Equal)
825 branchTestMagic(Assembler::NotEqual, valaddr, ¬Magic);
826 else
827 branchTestMagic(Assembler::NotEqual, valaddr, label);
828
829 branch32(cond, ToPayload(valaddr), Imm32(why), label);
830 bind(¬Magic);
831 }
832
branchToComputedAddress(const BaseIndex & addr)833 void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
834 jmp(Operand(addr));
835 }
836
cmp32MovePtr(Condition cond,Register lhs,Imm32 rhs,Register src,Register dest)837 void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
838 Register src, Register dest) {
839 cmp32(lhs, rhs);
840 cmovCCl(cond, Operand(src), dest);
841 }
842
test32LoadPtr(Condition cond,const Address & addr,Imm32 mask,const Address & src,Register dest)843 void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
844 Imm32 mask, const Address& src,
845 Register dest) {
846 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
847 test32(addr, mask);
848 cmovCCl(cond, Operand(src), dest);
849 }
850
test32MovePtr(Condition cond,const Address & addr,Imm32 mask,Register src,Register dest)851 void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
852 Imm32 mask, Register src, Register dest) {
853 MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
854 test32(addr, mask);
855 cmovCCl(cond, Operand(src), dest);
856 }
857
spectreMovePtr(Condition cond,Register src,Register dest)858 void MacroAssembler::spectreMovePtr(Condition cond, Register src,
859 Register dest) {
860 cmovCCl(cond, Operand(src), dest);
861 }
862
spectreBoundsCheck32(Register index,const Operand & length,Register maybeScratch,Label * failure)863 void MacroAssembler::spectreBoundsCheck32(Register index, const Operand& length,
864 Register maybeScratch,
865 Label* failure) {
866 Label failurePopValue;
867 bool pushedValue = false;
868 if (JitOptions.spectreIndexMasking) {
869 if (maybeScratch == InvalidReg) {
870 push(Imm32(0));
871 pushedValue = true;
872 } else {
873 move32(Imm32(0), maybeScratch);
874 }
875 }
876
877 cmp32(index, length);
878 j(Assembler::AboveOrEqual, pushedValue ? &failurePopValue : failure);
879
880 if (JitOptions.spectreIndexMasking) {
881 if (maybeScratch == InvalidReg) {
882 Label done;
883 cmovCCl(Assembler::AboveOrEqual, Operand(StackPointer, 0), index);
884 lea(Operand(StackPointer, sizeof(void*)), StackPointer);
885 jump(&done);
886
887 bind(&failurePopValue);
888 lea(Operand(StackPointer, sizeof(void*)), StackPointer);
889 jump(failure);
890
891 bind(&done);
892 } else {
893 cmovCCl(Assembler::AboveOrEqual, maybeScratch, index);
894 }
895 }
896 }
897
spectreBoundsCheck32(Register index,Register length,Register maybeScratch,Label * failure)898 void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
899 Register maybeScratch,
900 Label* failure) {
901 MOZ_ASSERT(length != maybeScratch);
902 MOZ_ASSERT(index != maybeScratch);
903
904 spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
905 }
906
spectreBoundsCheck32(Register index,const Address & length,Register maybeScratch,Label * failure)907 void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
908 Register maybeScratch,
909 Label* failure) {
910 MOZ_ASSERT(index != length.base);
911 MOZ_ASSERT(length.base != maybeScratch);
912 MOZ_ASSERT(index != maybeScratch);
913
914 spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
915 }
916
917 // ========================================================================
918 // Truncate floating point.
919
truncateFloat32ToUInt64(Address src,Address dest,Register temp,FloatRegister floatTemp)920 void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
921 Register temp,
922 FloatRegister floatTemp) {
923 Label done;
924
925 loadFloat32(src, floatTemp);
926
927 truncateFloat32ToInt64(src, dest, temp);
928
929 // For unsigned conversion the case of [INT64, UINT64] needs to get handle
930 // seperately.
931 load32(HighWord(dest), temp);
932 branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
933
934 // Move the value inside INT64 range.
935 storeFloat32(floatTemp, dest);
936 loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
937 vaddss(Operand(dest), floatTemp, floatTemp);
938 storeFloat32(floatTemp, dest);
939 truncateFloat32ToInt64(dest, dest, temp);
940
941 load32(HighWord(dest), temp);
942 orl(Imm32(0x80000000), temp);
943 store32(temp, HighWord(dest));
944
945 bind(&done);
946 }
947
truncateDoubleToUInt64(Address src,Address dest,Register temp,FloatRegister floatTemp)948 void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
949 Register temp,
950 FloatRegister floatTemp) {
951 Label done;
952
953 loadDouble(src, floatTemp);
954
955 truncateDoubleToInt64(src, dest, temp);
956
957 // For unsigned conversion the case of [INT64, UINT64] needs to get handle
958 // seperately.
959 load32(HighWord(dest), temp);
960 branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
961
962 // Move the value inside INT64 range.
963 storeDouble(floatTemp, dest);
964 loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
965 vaddsd(Operand(dest), floatTemp, floatTemp);
966 storeDouble(floatTemp, dest);
967 truncateDoubleToInt64(dest, dest, temp);
968
969 load32(HighWord(dest), temp);
970 orl(Imm32(0x80000000), temp);
971 store32(temp, HighWord(dest));
972
973 bind(&done);
974 }
975
976 // ========================================================================
977 // wasm support
978
979 template <class L>
wasmBoundsCheck(Condition cond,Register index,Register boundsCheckLimit,L label)980 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
981 Register boundsCheckLimit, L label) {
982 cmp32(index, boundsCheckLimit);
983 j(cond, label);
984 if (JitOptions.spectreIndexMasking)
985 cmovCCl(cond, Operand(boundsCheckLimit), index);
986 }
987
988 template <class L>
wasmBoundsCheck(Condition cond,Register index,Address boundsCheckLimit,L label)989 void MacroAssembler::wasmBoundsCheck(Condition cond, Register index,
990 Address boundsCheckLimit, L label) {
991 cmp32(index, Operand(boundsCheckLimit));
992 j(cond, label);
993 if (JitOptions.spectreIndexMasking)
994 cmovCCl(cond, Operand(boundsCheckLimit), index);
995 }
996
997 //}}} check_macroassembler_style
998 // ===============================================================
999
1000 // Note: this function clobbers the source register.
convertUInt32ToDouble(Register src,FloatRegister dest)1001 void MacroAssemblerX86::convertUInt32ToDouble(Register src,
1002 FloatRegister dest) {
1003 // src is [0, 2^32-1]
1004 subl(Imm32(0x80000000), src);
1005
1006 // Now src is [-2^31, 2^31-1] - int range, but not the same value.
1007 convertInt32ToDouble(src, dest);
1008
1009 // dest is now a double with the int range.
1010 // correct the double value by adding 0x80000000.
1011 asMasm().addConstantDouble(2147483648.0, dest);
1012 }
1013
1014 // Note: this function clobbers the source register.
convertUInt32ToFloat32(Register src,FloatRegister dest)1015 void MacroAssemblerX86::convertUInt32ToFloat32(Register src,
1016 FloatRegister dest) {
1017 convertUInt32ToDouble(src, dest);
1018 convertDoubleToFloat32(dest, dest);
1019 }
1020
unboxValue(const ValueOperand & src,AnyRegister dest,JSValueType)1021 void MacroAssemblerX86::unboxValue(const ValueOperand& src, AnyRegister dest,
1022 JSValueType) {
1023 if (dest.isFloat()) {
1024 Label notInt32, end;
1025 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
1026 convertInt32ToDouble(src.payloadReg(), dest.fpu());
1027 jump(&end);
1028 bind(¬Int32);
1029 unboxDouble(src, dest.fpu());
1030 bind(&end);
1031 } else {
1032 if (src.payloadReg() != dest.gpr()) movl(src.payloadReg(), dest.gpr());
1033 }
1034 }
1035
1036 template <typename T>
loadInt32OrDouble(const T & src,FloatRegister dest)1037 void MacroAssemblerX86::loadInt32OrDouble(const T& src, FloatRegister dest) {
1038 Label notInt32, end;
1039 asMasm().branchTestInt32(Assembler::NotEqual, src, ¬Int32);
1040 convertInt32ToDouble(ToPayload(src), dest);
1041 jump(&end);
1042 bind(¬Int32);
1043 loadDouble(src, dest);
1044 bind(&end);
1045 }
1046
1047 template <typename T>
loadUnboxedValue(const T & src,MIRType type,AnyRegister dest)1048 void MacroAssemblerX86::loadUnboxedValue(const T& src, MIRType type,
1049 AnyRegister dest) {
1050 if (dest.isFloat())
1051 loadInt32OrDouble(src, dest.fpu());
1052 else
1053 movl(Operand(src), dest.gpr());
1054 }
1055
1056 // If source is a double, load it into dest. If source is int32,
1057 // convert it to double. Else, branch to failure.
ensureDouble(const ValueOperand & source,FloatRegister dest,Label * failure)1058 void MacroAssemblerX86::ensureDouble(const ValueOperand& source,
1059 FloatRegister dest, Label* failure) {
1060 Label isDouble, done;
1061 asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
1062 asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
1063
1064 convertInt32ToDouble(source.payloadReg(), dest);
1065 jump(&done);
1066
1067 bind(&isDouble);
1068 unboxDouble(source, dest);
1069
1070 bind(&done);
1071 }
1072
1073 } // namespace jit
1074 } // namespace js
1075
1076 #endif /* jit_x86_MacroAssembler_x86_inl_h */
1077