1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/x86-shared/CodeGenerator-x86-shared.h"
8
9 #include "mozilla/DebugOnly.h"
10 #include "mozilla/MathAlgorithms.h"
11
12 #include "jit/CodeGenerator.h"
13 #include "jit/InlineScriptTree.h"
14 #include "jit/JitRuntime.h"
15 #include "jit/RangeAnalysis.h"
16 #include "js/ScalarType.h" // js::Scalar::Type
17 #include "util/DifferentialTesting.h"
18
19 #include "jit/MacroAssembler-inl.h"
20 #include "jit/shared/CodeGenerator-shared-inl.h"
21
22 using namespace js;
23 using namespace js::jit;
24
25 using mozilla::Abs;
26 using mozilla::DebugOnly;
27 using mozilla::FloorLog2;
28 using mozilla::NegativeInfinity;
29
30 using JS::GenericNaN;
31
32 namespace js {
33 namespace jit {
34
CodeGeneratorX86Shared(MIRGenerator * gen,LIRGraph * graph,MacroAssembler * masm)35 CodeGeneratorX86Shared::CodeGeneratorX86Shared(MIRGenerator* gen,
36 LIRGraph* graph,
37 MacroAssembler* masm)
38 : CodeGeneratorShared(gen, graph, masm) {}
39
40 #ifdef JS_PUNBOX64
ToOperandOrRegister64(const LInt64Allocation input)41 Operand CodeGeneratorX86Shared::ToOperandOrRegister64(
42 const LInt64Allocation input) {
43 return ToOperand(input.value());
44 }
45 #else
ToOperandOrRegister64(const LInt64Allocation input)46 Register64 CodeGeneratorX86Shared::ToOperandOrRegister64(
47 const LInt64Allocation input) {
48 return ToRegister64(input);
49 }
50 #endif
51
accept(CodeGeneratorX86Shared * codegen)52 void OutOfLineBailout::accept(CodeGeneratorX86Shared* codegen) {
53 codegen->visitOutOfLineBailout(this);
54 }
55
emitBranch(Assembler::Condition cond,MBasicBlock * mirTrue,MBasicBlock * mirFalse,Assembler::NaNCond ifNaN)56 void CodeGeneratorX86Shared::emitBranch(Assembler::Condition cond,
57 MBasicBlock* mirTrue,
58 MBasicBlock* mirFalse,
59 Assembler::NaNCond ifNaN) {
60 if (ifNaN == Assembler::NaN_IsFalse) {
61 jumpToBlock(mirFalse, Assembler::Parity);
62 } else if (ifNaN == Assembler::NaN_IsTrue) {
63 jumpToBlock(mirTrue, Assembler::Parity);
64 }
65
66 if (isNextBlock(mirFalse->lir())) {
67 jumpToBlock(mirTrue, cond);
68 } else {
69 jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
70 jumpToBlock(mirTrue);
71 }
72 }
73
visitDouble(LDouble * ins)74 void CodeGenerator::visitDouble(LDouble* ins) {
75 const LDefinition* out = ins->getDef(0);
76 masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
77 }
78
visitFloat32(LFloat32 * ins)79 void CodeGenerator::visitFloat32(LFloat32* ins) {
80 const LDefinition* out = ins->getDef(0);
81 masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
82 }
83
visitTestIAndBranch(LTestIAndBranch * test)84 void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
85 Register input = ToRegister(test->input());
86 masm.test32(input, input);
87 emitBranch(Assembler::NonZero, test->ifTrue(), test->ifFalse());
88 }
89
visitTestDAndBranch(LTestDAndBranch * test)90 void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
91 const LAllocation* opd = test->input();
92
93 // vucomisd flags:
94 // Z P C
95 // ---------
96 // NaN 1 1 1
97 // > 0 0 0
98 // < 0 0 1
99 // = 1 0 0
100 //
101 // NaN is falsey, so comparing against 0 and then using the Z flag is
102 // enough to determine which branch to take.
103 ScratchDoubleScope scratch(masm);
104 masm.zeroDouble(scratch);
105 masm.vucomisd(scratch, ToFloatRegister(opd));
106 emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
107 }
108
visitTestFAndBranch(LTestFAndBranch * test)109 void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
110 const LAllocation* opd = test->input();
111 // vucomiss flags are the same as doubles; see comment above
112 {
113 ScratchFloat32Scope scratch(masm);
114 masm.zeroFloat32(scratch);
115 masm.vucomiss(scratch, ToFloatRegister(opd));
116 }
117 emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
118 }
119
emitCompare(MCompare::CompareType type,const LAllocation * left,const LAllocation * right)120 void CodeGeneratorX86Shared::emitCompare(MCompare::CompareType type,
121 const LAllocation* left,
122 const LAllocation* right) {
123 #ifdef JS_CODEGEN_X64
124 if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol ||
125 type == MCompare::Compare_UIntPtr) {
126 if (right->isConstant()) {
127 MOZ_ASSERT(type == MCompare::Compare_UIntPtr);
128 masm.cmpPtr(ToRegister(left), Imm32(ToInt32(right)));
129 } else {
130 masm.cmpPtr(ToRegister(left), ToOperand(right));
131 }
132 return;
133 }
134 #endif
135
136 if (right->isConstant()) {
137 masm.cmp32(ToRegister(left), Imm32(ToInt32(right)));
138 } else {
139 masm.cmp32(ToRegister(left), ToOperand(right));
140 }
141 }
142
visitCompare(LCompare * comp)143 void CodeGenerator::visitCompare(LCompare* comp) {
144 MCompare* mir = comp->mir();
145 emitCompare(mir->compareType(), comp->left(), comp->right());
146 masm.emitSet(JSOpToCondition(mir->compareType(), comp->jsop()),
147 ToRegister(comp->output()));
148 }
149
visitCompareAndBranch(LCompareAndBranch * comp)150 void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
151 MCompare* mir = comp->cmpMir();
152 emitCompare(mir->compareType(), comp->left(), comp->right());
153 Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
154 emitBranch(cond, comp->ifTrue(), comp->ifFalse());
155 }
156
visitCompareD(LCompareD * comp)157 void CodeGenerator::visitCompareD(LCompareD* comp) {
158 FloatRegister lhs = ToFloatRegister(comp->left());
159 FloatRegister rhs = ToFloatRegister(comp->right());
160
161 Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
162
163 Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
164 if (comp->mir()->operandsAreNeverNaN()) {
165 nanCond = Assembler::NaN_HandledByCond;
166 }
167
168 masm.compareDouble(cond, lhs, rhs);
169 masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
170 ToRegister(comp->output()), nanCond);
171 }
172
visitCompareF(LCompareF * comp)173 void CodeGenerator::visitCompareF(LCompareF* comp) {
174 FloatRegister lhs = ToFloatRegister(comp->left());
175 FloatRegister rhs = ToFloatRegister(comp->right());
176
177 Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
178
179 Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
180 if (comp->mir()->operandsAreNeverNaN()) {
181 nanCond = Assembler::NaN_HandledByCond;
182 }
183
184 masm.compareFloat(cond, lhs, rhs);
185 masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
186 ToRegister(comp->output()), nanCond);
187 }
188
visitNotI(LNotI * ins)189 void CodeGenerator::visitNotI(LNotI* ins) {
190 masm.cmp32(ToRegister(ins->input()), Imm32(0));
191 masm.emitSet(Assembler::Equal, ToRegister(ins->output()));
192 }
193
visitNotD(LNotD * ins)194 void CodeGenerator::visitNotD(LNotD* ins) {
195 FloatRegister opd = ToFloatRegister(ins->input());
196
197 // Not returns true if the input is a NaN. We don't have to worry about
198 // it if we know the input is never NaN though.
199 Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
200 if (ins->mir()->operandIsNeverNaN()) {
201 nanCond = Assembler::NaN_HandledByCond;
202 }
203
204 ScratchDoubleScope scratch(masm);
205 masm.zeroDouble(scratch);
206 masm.compareDouble(Assembler::DoubleEqualOrUnordered, opd, scratch);
207 masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
208 }
209
visitNotF(LNotF * ins)210 void CodeGenerator::visitNotF(LNotF* ins) {
211 FloatRegister opd = ToFloatRegister(ins->input());
212
213 // Not returns true if the input is a NaN. We don't have to worry about
214 // it if we know the input is never NaN though.
215 Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
216 if (ins->mir()->operandIsNeverNaN()) {
217 nanCond = Assembler::NaN_HandledByCond;
218 }
219
220 ScratchFloat32Scope scratch(masm);
221 masm.zeroFloat32(scratch);
222 masm.compareFloat(Assembler::DoubleEqualOrUnordered, opd, scratch);
223 masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
224 }
225
visitCompareDAndBranch(LCompareDAndBranch * comp)226 void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
227 FloatRegister lhs = ToFloatRegister(comp->left());
228 FloatRegister rhs = ToFloatRegister(comp->right());
229
230 Assembler::DoubleCondition cond =
231 JSOpToDoubleCondition(comp->cmpMir()->jsop());
232
233 Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
234 if (comp->cmpMir()->operandsAreNeverNaN()) {
235 nanCond = Assembler::NaN_HandledByCond;
236 }
237
238 masm.compareDouble(cond, lhs, rhs);
239 emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
240 comp->ifFalse(), nanCond);
241 }
242
visitCompareFAndBranch(LCompareFAndBranch * comp)243 void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
244 FloatRegister lhs = ToFloatRegister(comp->left());
245 FloatRegister rhs = ToFloatRegister(comp->right());
246
247 Assembler::DoubleCondition cond =
248 JSOpToDoubleCondition(comp->cmpMir()->jsop());
249
250 Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
251 if (comp->cmpMir()->operandsAreNeverNaN()) {
252 nanCond = Assembler::NaN_HandledByCond;
253 }
254
255 masm.compareFloat(cond, lhs, rhs);
256 emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
257 comp->ifFalse(), nanCond);
258 }
259
visitWasmStackArg(LWasmStackArg * ins)260 void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
261 const MWasmStackArg* mir = ins->mir();
262 Address dst(StackPointer, mir->spOffset());
263 if (ins->arg()->isConstant()) {
264 masm.storePtr(ImmWord(ToInt32(ins->arg())), dst);
265 } else if (ins->arg()->isGeneralReg()) {
266 masm.storePtr(ToRegister(ins->arg()), dst);
267 } else {
268 switch (mir->input()->type()) {
269 case MIRType::Double:
270 masm.storeDouble(ToFloatRegister(ins->arg()), dst);
271 return;
272 case MIRType::Float32:
273 masm.storeFloat32(ToFloatRegister(ins->arg()), dst);
274 return;
275 #ifdef ENABLE_WASM_SIMD
276 case MIRType::Simd128:
277 masm.storeUnalignedSimd128(ToFloatRegister(ins->arg()), dst);
278 return;
279 #endif
280 default:
281 break;
282 }
283 MOZ_CRASH("unexpected mir type in WasmStackArg");
284 }
285 }
286
visitWasmStackArgI64(LWasmStackArgI64 * ins)287 void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
288 const MWasmStackArg* mir = ins->mir();
289 Address dst(StackPointer, mir->spOffset());
290 if (IsConstant(ins->arg())) {
291 masm.store64(Imm64(ToInt64(ins->arg())), dst);
292 } else {
293 masm.store64(ToRegister64(ins->arg()), dst);
294 }
295 }
296
visitWasmSelect(LWasmSelect * ins)297 void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
298 MIRType mirType = ins->mir()->type();
299
300 Register cond = ToRegister(ins->condExpr());
301 Operand falseExpr = ToOperand(ins->falseExpr());
302
303 masm.test32(cond, cond);
304
305 if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) {
306 Register out = ToRegister(ins->output());
307 MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
308 "true expr input is reused for output");
309 if (mirType == MIRType::Int32) {
310 masm.cmovz32(falseExpr, out);
311 } else {
312 masm.cmovzPtr(falseExpr, out);
313 }
314 return;
315 }
316
317 FloatRegister out = ToFloatRegister(ins->output());
318 MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
319 "true expr input is reused for output");
320
321 Label done;
322 masm.j(Assembler::NonZero, &done);
323
324 if (mirType == MIRType::Float32) {
325 if (falseExpr.kind() == Operand::FPREG) {
326 masm.moveFloat32(ToFloatRegister(ins->falseExpr()), out);
327 } else {
328 masm.loadFloat32(falseExpr, out);
329 }
330 } else if (mirType == MIRType::Double) {
331 if (falseExpr.kind() == Operand::FPREG) {
332 masm.moveDouble(ToFloatRegister(ins->falseExpr()), out);
333 } else {
334 masm.loadDouble(falseExpr, out);
335 }
336 } else if (mirType == MIRType::Simd128) {
337 if (falseExpr.kind() == Operand::FPREG) {
338 masm.moveSimd128(ToFloatRegister(ins->falseExpr()), out);
339 } else {
340 masm.loadUnalignedSimd128(falseExpr, out);
341 }
342 } else {
343 MOZ_CRASH("unhandled type in visitWasmSelect!");
344 }
345
346 masm.bind(&done);
347 }
348
visitWasmReinterpret(LWasmReinterpret * lir)349 void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
350 MOZ_ASSERT(gen->compilingWasm());
351 MWasmReinterpret* ins = lir->mir();
352
353 MIRType to = ins->type();
354 #ifdef DEBUG
355 MIRType from = ins->input()->type();
356 #endif
357
358 switch (to) {
359 case MIRType::Int32:
360 MOZ_ASSERT(from == MIRType::Float32);
361 masm.vmovd(ToFloatRegister(lir->input()), ToRegister(lir->output()));
362 break;
363 case MIRType::Float32:
364 MOZ_ASSERT(from == MIRType::Int32);
365 masm.vmovd(ToRegister(lir->input()), ToFloatRegister(lir->output()));
366 break;
367 case MIRType::Double:
368 case MIRType::Int64:
369 MOZ_CRASH("not handled by this LIR opcode");
370 default:
371 MOZ_CRASH("unexpected WasmReinterpret");
372 }
373 }
374
visitAsmJSLoadHeap(LAsmJSLoadHeap * ins)375 void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
376 const MAsmJSLoadHeap* mir = ins->mir();
377 MOZ_ASSERT(mir->access().offset() == 0);
378
379 const LAllocation* ptr = ins->ptr();
380 const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
381 AnyRegister out = ToAnyRegister(ins->output());
382
383 Scalar::Type accessType = mir->accessType();
384
385 OutOfLineLoadTypedArrayOutOfBounds* ool = nullptr;
386 if (mir->needsBoundsCheck()) {
387 ool = new (alloc()) OutOfLineLoadTypedArrayOutOfBounds(out, accessType);
388 addOutOfLineCode(ool, mir);
389
390 masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ToRegister(ptr),
391 ToRegister(boundsCheckLimit), ool->entry());
392 }
393
394 Operand srcAddr = toMemoryAccessOperand(ins, 0);
395 masm.wasmLoad(mir->access(), srcAddr, out);
396
397 if (ool) {
398 masm.bind(ool->rejoin());
399 }
400 }
401
visitOutOfLineLoadTypedArrayOutOfBounds(OutOfLineLoadTypedArrayOutOfBounds * ool)402 void CodeGeneratorX86Shared::visitOutOfLineLoadTypedArrayOutOfBounds(
403 OutOfLineLoadTypedArrayOutOfBounds* ool) {
404 switch (ool->viewType()) {
405 case Scalar::Int64:
406 case Scalar::BigInt64:
407 case Scalar::BigUint64:
408 case Scalar::Simd128:
409 case Scalar::MaxTypedArrayViewType:
410 MOZ_CRASH("unexpected array type");
411 case Scalar::Float32:
412 masm.loadConstantFloat32(float(GenericNaN()), ool->dest().fpu());
413 break;
414 case Scalar::Float64:
415 masm.loadConstantDouble(GenericNaN(), ool->dest().fpu());
416 break;
417 case Scalar::Int8:
418 case Scalar::Uint8:
419 case Scalar::Int16:
420 case Scalar::Uint16:
421 case Scalar::Int32:
422 case Scalar::Uint32:
423 case Scalar::Uint8Clamped:
424 Register destReg = ool->dest().gpr();
425 masm.mov(ImmWord(0), destReg);
426 break;
427 }
428 masm.jmp(ool->rejoin());
429 }
430
visitAsmJSStoreHeap(LAsmJSStoreHeap * ins)431 void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
432 const MAsmJSStoreHeap* mir = ins->mir();
433
434 const LAllocation* ptr = ins->ptr();
435 const LAllocation* value = ins->value();
436 const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
437
438 Scalar::Type accessType = mir->accessType();
439 canonicalizeIfDeterministic(accessType, value);
440
441 Label rejoin;
442 if (mir->needsBoundsCheck()) {
443 masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ToRegister(ptr),
444 ToRegister(boundsCheckLimit), &rejoin);
445 }
446
447 Operand dstAddr = toMemoryAccessOperand(ins, 0);
448 masm.wasmStore(mir->access(), ToAnyRegister(value), dstAddr);
449
450 if (rejoin.used()) {
451 masm.bind(&rejoin);
452 }
453 }
454
visitWasmAddOffset(LWasmAddOffset * lir)455 void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
456 MWasmAddOffset* mir = lir->mir();
457 Register base = ToRegister(lir->base());
458 Register out = ToRegister(lir->output());
459
460 if (base != out) {
461 masm.move32(base, out);
462 }
463 masm.add32(Imm32(mir->offset()), out);
464
465 Label ok;
466 masm.j(Assembler::CarryClear, &ok);
467 masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
468 masm.bind(&ok);
469 }
470
visitWasmAddOffset64(LWasmAddOffset64 * lir)471 void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
472 MWasmAddOffset* mir = lir->mir();
473 Register64 base = ToRegister64(lir->base());
474 Register64 out = ToOutRegister64(lir);
475
476 if (base != out) {
477 masm.move64(base, out);
478 }
479 masm.add64(Imm64(mir->offset()), out);
480
481 Label ok;
482 masm.j(Assembler::CarryClear, &ok);
483 masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
484 masm.bind(&ok);
485 }
486
visitWasmTruncateToInt32(LWasmTruncateToInt32 * lir)487 void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
488 FloatRegister input = ToFloatRegister(lir->input());
489 Register output = ToRegister(lir->output());
490
491 MWasmTruncateToInt32* mir = lir->mir();
492 MIRType inputType = mir->input()->type();
493
494 MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
495
496 auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
497 addOutOfLineCode(ool, mir);
498
499 Label* oolEntry = ool->entry();
500 if (mir->isUnsigned()) {
501 if (inputType == MIRType::Double) {
502 masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(),
503 oolEntry);
504 } else if (inputType == MIRType::Float32) {
505 masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(),
506 oolEntry);
507 } else {
508 MOZ_CRASH("unexpected type");
509 }
510 if (mir->isSaturating()) {
511 masm.bind(ool->rejoin());
512 }
513 return;
514 }
515
516 if (inputType == MIRType::Double) {
517 masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(),
518 oolEntry);
519 } else if (inputType == MIRType::Float32) {
520 masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(),
521 oolEntry);
522 } else {
523 MOZ_CRASH("unexpected type");
524 }
525
526 masm.bind(ool->rejoin());
527 }
528
generateOutOfLineCode()529 bool CodeGeneratorX86Shared::generateOutOfLineCode() {
530 if (!CodeGeneratorShared::generateOutOfLineCode()) {
531 return false;
532 }
533
534 if (deoptLabel_.used()) {
535 // All non-table-based bailouts will go here.
536 masm.bind(&deoptLabel_);
537
538 // Push the frame size, so the handler can recover the IonScript.
539 masm.push(Imm32(frameSize()));
540
541 TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
542 masm.jump(handler);
543 }
544
545 return !masm.oom();
546 }
547
548 class BailoutJump {
549 Assembler::Condition cond_;
550
551 public:
BailoutJump(Assembler::Condition cond)552 explicit BailoutJump(Assembler::Condition cond) : cond_(cond) {}
553 #ifdef JS_CODEGEN_X86
operator ()(MacroAssembler & masm,uint8_t * code) const554 void operator()(MacroAssembler& masm, uint8_t* code) const {
555 masm.j(cond_, ImmPtr(code), RelocationKind::HARDCODED);
556 }
557 #endif
operator ()(MacroAssembler & masm,Label * label) const558 void operator()(MacroAssembler& masm, Label* label) const {
559 masm.j(cond_, label);
560 }
561 };
562
563 class BailoutLabel {
564 Label* label_;
565
566 public:
BailoutLabel(Label * label)567 explicit BailoutLabel(Label* label) : label_(label) {}
568 #ifdef JS_CODEGEN_X86
operator ()(MacroAssembler & masm,uint8_t * code) const569 void operator()(MacroAssembler& masm, uint8_t* code) const {
570 masm.retarget(label_, ImmPtr(code), RelocationKind::HARDCODED);
571 }
572 #endif
operator ()(MacroAssembler & masm,Label * label) const573 void operator()(MacroAssembler& masm, Label* label) const {
574 masm.retarget(label_, label);
575 }
576 };
577
578 template <typename T>
bailout(const T & binder,LSnapshot * snapshot)579 void CodeGeneratorX86Shared::bailout(const T& binder, LSnapshot* snapshot) {
580 encode(snapshot);
581
582 // Though the assembler doesn't track all frame pushes, at least make sure
583 // the known value makes sense. We can't use bailout tables if the stack
584 // isn't properly aligned to the static frame size.
585 MOZ_ASSERT_IF(frameClass_ != FrameSizeClass::None() && deoptTable_,
586 frameClass_.frameSize() == masm.framePushed());
587
588 #ifdef JS_CODEGEN_X86
589 // On x64, bailout tables are pointless, because 16 extra bytes are
590 // reserved per external jump, whereas it takes only 10 bytes to encode a
591 // a non-table based bailout.
592 if (assignBailoutId(snapshot)) {
593 binder(masm, deoptTable_->value +
594 snapshot->bailoutId() * BAILOUT_TABLE_ENTRY_SIZE);
595 return;
596 }
597 #endif
598
599 // We could not use a jump table, either because all bailout IDs were
600 // reserved, or a jump table is not optimal for this frame size or
601 // platform. Whatever, we will generate a lazy bailout.
602 //
603 // All bailout code is associated with the bytecodeSite of the block we are
604 // bailing out from.
605 InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
606 OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot);
607 addOutOfLineCode(ool,
608 new (alloc()) BytecodeSite(tree, tree->script()->code()));
609
610 binder(masm, ool->entry());
611 }
612
bailoutIf(Assembler::Condition condition,LSnapshot * snapshot)613 void CodeGeneratorX86Shared::bailoutIf(Assembler::Condition condition,
614 LSnapshot* snapshot) {
615 bailout(BailoutJump(condition), snapshot);
616 }
617
bailoutIf(Assembler::DoubleCondition condition,LSnapshot * snapshot)618 void CodeGeneratorX86Shared::bailoutIf(Assembler::DoubleCondition condition,
619 LSnapshot* snapshot) {
620 MOZ_ASSERT(Assembler::NaNCondFromDoubleCondition(condition) ==
621 Assembler::NaN_HandledByCond);
622 bailoutIf(Assembler::ConditionFromDoubleCondition(condition), snapshot);
623 }
624
bailoutFrom(Label * label,LSnapshot * snapshot)625 void CodeGeneratorX86Shared::bailoutFrom(Label* label, LSnapshot* snapshot) {
626 MOZ_ASSERT_IF(!masm.oom(), label->used() && !label->bound());
627 bailout(BailoutLabel(label), snapshot);
628 }
629
bailout(LSnapshot * snapshot)630 void CodeGeneratorX86Shared::bailout(LSnapshot* snapshot) {
631 Label label;
632 masm.jump(&label);
633 bailoutFrom(&label, snapshot);
634 }
635
visitOutOfLineBailout(OutOfLineBailout * ool)636 void CodeGeneratorX86Shared::visitOutOfLineBailout(OutOfLineBailout* ool) {
637 masm.push(Imm32(ool->snapshot()->snapshotOffset()));
638 masm.jmp(&deoptLabel_);
639 }
640
visitMinMaxD(LMinMaxD * ins)641 void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
642 FloatRegister first = ToFloatRegister(ins->first());
643 FloatRegister second = ToFloatRegister(ins->second());
644 #ifdef DEBUG
645 FloatRegister output = ToFloatRegister(ins->output());
646 MOZ_ASSERT(first == output);
647 #endif
648
649 bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
650
651 if (ins->mir()->isMax()) {
652 masm.maxDouble(second, first, handleNaN);
653 } else {
654 masm.minDouble(second, first, handleNaN);
655 }
656 }
657
visitMinMaxF(LMinMaxF * ins)658 void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
659 FloatRegister first = ToFloatRegister(ins->first());
660 FloatRegister second = ToFloatRegister(ins->second());
661 #ifdef DEBUG
662 FloatRegister output = ToFloatRegister(ins->output());
663 MOZ_ASSERT(first == output);
664 #endif
665
666 bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
667
668 if (ins->mir()->isMax()) {
669 masm.maxFloat32(second, first, handleNaN);
670 } else {
671 masm.minFloat32(second, first, handleNaN);
672 }
673 }
674
visitClzI(LClzI * ins)675 void CodeGenerator::visitClzI(LClzI* ins) {
676 Register input = ToRegister(ins->input());
677 Register output = ToRegister(ins->output());
678 bool knownNotZero = ins->mir()->operandIsNeverZero();
679
680 masm.clz32(input, output, knownNotZero);
681 }
682
visitCtzI(LCtzI * ins)683 void CodeGenerator::visitCtzI(LCtzI* ins) {
684 Register input = ToRegister(ins->input());
685 Register output = ToRegister(ins->output());
686 bool knownNotZero = ins->mir()->operandIsNeverZero();
687
688 masm.ctz32(input, output, knownNotZero);
689 }
690
visitPopcntI(LPopcntI * ins)691 void CodeGenerator::visitPopcntI(LPopcntI* ins) {
692 Register input = ToRegister(ins->input());
693 Register output = ToRegister(ins->output());
694 Register temp =
695 ins->temp0()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp0());
696
697 masm.popcnt32(input, output, temp);
698 }
699
visitPowHalfD(LPowHalfD * ins)700 void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
701 FloatRegister input = ToFloatRegister(ins->input());
702 FloatRegister output = ToFloatRegister(ins->output());
703
704 ScratchDoubleScope scratch(masm);
705
706 Label done, sqrt;
707
708 if (!ins->mir()->operandIsNeverNegativeInfinity()) {
709 // Branch if not -Infinity.
710 masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
711
712 Assembler::DoubleCondition cond = Assembler::DoubleNotEqualOrUnordered;
713 if (ins->mir()->operandIsNeverNaN()) {
714 cond = Assembler::DoubleNotEqual;
715 }
716 masm.branchDouble(cond, input, scratch, &sqrt);
717
718 // Math.pow(-Infinity, 0.5) == Infinity.
719 masm.zeroDouble(output);
720 masm.subDouble(scratch, output);
721 masm.jump(&done);
722
723 masm.bind(&sqrt);
724 }
725
726 if (!ins->mir()->operandIsNeverNegativeZero()) {
727 // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
728 // Adding 0 converts any -0 to 0.
729 masm.zeroDouble(scratch);
730 masm.addDouble(input, scratch);
731 masm.vsqrtsd(scratch, output, output);
732 } else {
733 masm.vsqrtsd(input, output, output);
734 }
735
736 masm.bind(&done);
737 }
738
739 class OutOfLineUndoALUOperation
740 : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
741 LInstruction* ins_;
742
743 public:
OutOfLineUndoALUOperation(LInstruction * ins)744 explicit OutOfLineUndoALUOperation(LInstruction* ins) : ins_(ins) {}
745
accept(CodeGeneratorX86Shared * codegen)746 virtual void accept(CodeGeneratorX86Shared* codegen) override {
747 codegen->visitOutOfLineUndoALUOperation(this);
748 }
ins() const749 LInstruction* ins() const { return ins_; }
750 };
751
visitAddI(LAddI * ins)752 void CodeGenerator::visitAddI(LAddI* ins) {
753 if (ins->rhs()->isConstant()) {
754 masm.addl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
755 } else {
756 masm.addl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
757 }
758
759 if (ins->snapshot()) {
760 if (ins->recoversInput()) {
761 OutOfLineUndoALUOperation* ool =
762 new (alloc()) OutOfLineUndoALUOperation(ins);
763 addOutOfLineCode(ool, ins->mir());
764 masm.j(Assembler::Overflow, ool->entry());
765 } else {
766 bailoutIf(Assembler::Overflow, ins->snapshot());
767 }
768 }
769 }
770
visitAddI64(LAddI64 * lir)771 void CodeGenerator::visitAddI64(LAddI64* lir) {
772 const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
773 const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
774
775 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
776
777 if (IsConstant(rhs)) {
778 masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
779 return;
780 }
781
782 masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
783 }
784
visitSubI(LSubI * ins)785 void CodeGenerator::visitSubI(LSubI* ins) {
786 if (ins->rhs()->isConstant()) {
787 masm.subl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
788 } else {
789 masm.subl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
790 }
791
792 if (ins->snapshot()) {
793 if (ins->recoversInput()) {
794 OutOfLineUndoALUOperation* ool =
795 new (alloc()) OutOfLineUndoALUOperation(ins);
796 addOutOfLineCode(ool, ins->mir());
797 masm.j(Assembler::Overflow, ool->entry());
798 } else {
799 bailoutIf(Assembler::Overflow, ins->snapshot());
800 }
801 }
802 }
803
visitSubI64(LSubI64 * lir)804 void CodeGenerator::visitSubI64(LSubI64* lir) {
805 const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
806 const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
807
808 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
809
810 if (IsConstant(rhs)) {
811 masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
812 return;
813 }
814
815 masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
816 }
817
visitOutOfLineUndoALUOperation(OutOfLineUndoALUOperation * ool)818 void CodeGeneratorX86Shared::visitOutOfLineUndoALUOperation(
819 OutOfLineUndoALUOperation* ool) {
820 LInstruction* ins = ool->ins();
821 Register reg = ToRegister(ins->getDef(0));
822
823 DebugOnly<LAllocation*> lhs = ins->getOperand(0);
824 LAllocation* rhs = ins->getOperand(1);
825
826 MOZ_ASSERT(reg == ToRegister(lhs));
827 MOZ_ASSERT_IF(rhs->isGeneralReg(), reg != ToRegister(rhs));
828
829 // Undo the effect of the ALU operation, which was performed on the output
830 // register and overflowed. Writing to the output register clobbered an
831 // input reg, and the original value of the input needs to be recovered
832 // to satisfy the constraint imposed by any RECOVERED_INPUT operands to
833 // the bailout snapshot.
834
835 if (rhs->isConstant()) {
836 Imm32 constant(ToInt32(rhs));
837 if (ins->isAddI()) {
838 masm.subl(constant, reg);
839 } else {
840 masm.addl(constant, reg);
841 }
842 } else {
843 if (ins->isAddI()) {
844 masm.subl(ToOperand(rhs), reg);
845 } else {
846 masm.addl(ToOperand(rhs), reg);
847 }
848 }
849
850 bailout(ool->ins()->snapshot());
851 }
852
853 class MulNegativeZeroCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
854 LMulI* ins_;
855
856 public:
MulNegativeZeroCheck(LMulI * ins)857 explicit MulNegativeZeroCheck(LMulI* ins) : ins_(ins) {}
858
accept(CodeGeneratorX86Shared * codegen)859 virtual void accept(CodeGeneratorX86Shared* codegen) override {
860 codegen->visitMulNegativeZeroCheck(this);
861 }
ins() const862 LMulI* ins() const { return ins_; }
863 };
864
visitMulI(LMulI * ins)865 void CodeGenerator::visitMulI(LMulI* ins) {
866 const LAllocation* lhs = ins->lhs();
867 const LAllocation* rhs = ins->rhs();
868 MMul* mul = ins->mir();
869 MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
870 !mul->canBeNegativeZero() && !mul->canOverflow());
871
872 if (rhs->isConstant()) {
873 // Bailout on -0.0
874 int32_t constant = ToInt32(rhs);
875 if (mul->canBeNegativeZero() && constant <= 0) {
876 Assembler::Condition bailoutCond =
877 (constant == 0) ? Assembler::Signed : Assembler::Equal;
878 masm.test32(ToRegister(lhs), ToRegister(lhs));
879 bailoutIf(bailoutCond, ins->snapshot());
880 }
881
882 switch (constant) {
883 case -1:
884 masm.negl(ToOperand(lhs));
885 break;
886 case 0:
887 masm.xorl(ToOperand(lhs), ToRegister(lhs));
888 return; // escape overflow check;
889 case 1:
890 // nop
891 return; // escape overflow check;
892 case 2:
893 masm.addl(ToOperand(lhs), ToRegister(lhs));
894 break;
895 default:
896 if (!mul->canOverflow() && constant > 0) {
897 // Use shift if cannot overflow and constant is power of 2
898 int32_t shift = FloorLog2(constant);
899 if ((1 << shift) == constant) {
900 masm.shll(Imm32(shift), ToRegister(lhs));
901 return;
902 }
903 }
904 masm.imull(Imm32(ToInt32(rhs)), ToRegister(lhs));
905 }
906
907 // Bailout on overflow
908 if (mul->canOverflow()) {
909 bailoutIf(Assembler::Overflow, ins->snapshot());
910 }
911 } else {
912 masm.imull(ToOperand(rhs), ToRegister(lhs));
913
914 // Bailout on overflow
915 if (mul->canOverflow()) {
916 bailoutIf(Assembler::Overflow, ins->snapshot());
917 }
918
919 if (mul->canBeNegativeZero()) {
920 // Jump to an OOL path if the result is 0.
921 MulNegativeZeroCheck* ool = new (alloc()) MulNegativeZeroCheck(ins);
922 addOutOfLineCode(ool, mul);
923
924 masm.test32(ToRegister(lhs), ToRegister(lhs));
925 masm.j(Assembler::Zero, ool->entry());
926 masm.bind(ool->rejoin());
927 }
928 }
929 }
930
visitMulI64(LMulI64 * lir)931 void CodeGenerator::visitMulI64(LMulI64* lir) {
932 const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
933 const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
934
935 MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));
936
937 if (IsConstant(rhs)) {
938 int64_t constant = ToInt64(rhs);
939 switch (constant) {
940 case -1:
941 masm.neg64(ToRegister64(lhs));
942 return;
943 case 0:
944 masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
945 return;
946 case 1:
947 // nop
948 return;
949 case 2:
950 masm.add64(ToRegister64(lhs), ToRegister64(lhs));
951 return;
952 default:
953 if (constant > 0) {
954 // Use shift if constant is power of 2.
955 int32_t shift = mozilla::FloorLog2(constant);
956 if (int64_t(1) << shift == constant) {
957 masm.lshift64(Imm32(shift), ToRegister64(lhs));
958 return;
959 }
960 }
961 Register temp = ToTempRegisterOrInvalid(lir->temp());
962 masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
963 }
964 } else {
965 Register temp = ToTempRegisterOrInvalid(lir->temp());
966 masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
967 }
968 }
969
970 class ReturnZero : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
971 Register reg_;
972
973 public:
ReturnZero(Register reg)974 explicit ReturnZero(Register reg) : reg_(reg) {}
975
accept(CodeGeneratorX86Shared * codegen)976 virtual void accept(CodeGeneratorX86Shared* codegen) override {
977 codegen->visitReturnZero(this);
978 }
reg() const979 Register reg() const { return reg_; }
980 };
981
visitReturnZero(ReturnZero * ool)982 void CodeGeneratorX86Shared::visitReturnZero(ReturnZero* ool) {
983 masm.mov(ImmWord(0), ool->reg());
984 masm.jmp(ool->rejoin());
985 }
986
visitUDivOrMod(LUDivOrMod * ins)987 void CodeGenerator::visitUDivOrMod(LUDivOrMod* ins) {
988 Register lhs = ToRegister(ins->lhs());
989 Register rhs = ToRegister(ins->rhs());
990 Register output = ToRegister(ins->output());
991
992 MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
993 MOZ_ASSERT(rhs != edx);
994 MOZ_ASSERT_IF(output == eax, ToRegister(ins->remainder()) == edx);
995
996 ReturnZero* ool = nullptr;
997
998 // Put the lhs in eax.
999 if (lhs != eax) {
1000 masm.mov(lhs, eax);
1001 }
1002
1003 // Prevent divide by zero.
1004 if (ins->canBeDivideByZero()) {
1005 masm.test32(rhs, rhs);
1006 if (ins->mir()->isTruncated()) {
1007 if (ins->trapOnError()) {
1008 Label nonZero;
1009 masm.j(Assembler::NonZero, &nonZero);
1010 masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
1011 masm.bind(&nonZero);
1012 } else {
1013 ool = new (alloc()) ReturnZero(output);
1014 masm.j(Assembler::Zero, ool->entry());
1015 }
1016 } else {
1017 bailoutIf(Assembler::Zero, ins->snapshot());
1018 }
1019 }
1020
1021 // Zero extend the lhs into edx to make (edx:eax), since udiv is 64-bit.
1022 masm.mov(ImmWord(0), edx);
1023 masm.udiv(rhs);
1024
1025 // If the remainder is > 0, bailout since this must be a double.
1026 if (ins->mir()->isDiv() && !ins->mir()->toDiv()->canTruncateRemainder()) {
1027 Register remainder = ToRegister(ins->remainder());
1028 masm.test32(remainder, remainder);
1029 bailoutIf(Assembler::NonZero, ins->snapshot());
1030 }
1031
1032 // Unsigned div or mod can return a value that's not a signed int32.
1033 // If our users aren't expecting that, bail.
1034 if (!ins->mir()->isTruncated()) {
1035 masm.test32(output, output);
1036 bailoutIf(Assembler::Signed, ins->snapshot());
1037 }
1038
1039 if (ool) {
1040 addOutOfLineCode(ool, ins->mir());
1041 masm.bind(ool->rejoin());
1042 }
1043 }
1044
visitUDivOrModConstant(LUDivOrModConstant * ins)1045 void CodeGenerator::visitUDivOrModConstant(LUDivOrModConstant* ins) {
1046 Register lhs = ToRegister(ins->numerator());
1047 Register output = ToRegister(ins->output());
1048 uint32_t d = ins->denominator();
1049
1050 // This emits the division answer into edx or the modulus answer into eax.
1051 MOZ_ASSERT(output == eax || output == edx);
1052 MOZ_ASSERT(lhs != eax && lhs != edx);
1053 bool isDiv = (output == edx);
1054
1055 if (d == 0) {
1056 if (ins->mir()->isTruncated()) {
1057 if (ins->trapOnError()) {
1058 masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
1059 } else {
1060 masm.xorl(output, output);
1061 }
1062 } else {
1063 bailout(ins->snapshot());
1064 }
1065 return;
1066 }
1067
1068 // The denominator isn't a power of 2 (see LDivPowTwoI and LModPowTwoI).
1069 MOZ_ASSERT((d & (d - 1)) != 0);
1070
1071 ReciprocalMulConstants rmc = computeDivisionConstants(d, /* maxLog = */ 32);
1072
1073 // We first compute (M * n) >> 32, where M = rmc.multiplier.
1074 masm.movl(Imm32(rmc.multiplier), eax);
1075 masm.umull(lhs);
1076 if (rmc.multiplier > UINT32_MAX) {
1077 // M >= 2^32 and shift == 0 is impossible, as d >= 2 implies that
1078 // ((M * n) >> (32 + shift)) >= n > floor(n/d) whenever n >= d,
1079 // contradicting the proof of correctness in computeDivisionConstants.
1080 MOZ_ASSERT(rmc.shiftAmount > 0);
1081 MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 33));
1082
1083 // We actually computed edx = ((uint32_t(M) * n) >> 32) instead. Since
1084 // (M * n) >> (32 + shift) is the same as (edx + n) >> shift, we can
1085 // correct for the overflow. This case is a bit trickier than the signed
1086 // case, though, as the (edx + n) addition itself can overflow; however,
1087 // note that (edx + n) >> shift == (((n - edx) >> 1) + edx) >> (shift - 1),
1088 // which is overflow-free. See Hacker's Delight, section 10-8 for details.
1089
1090 // Compute (n - edx) >> 1 into eax.
1091 masm.movl(lhs, eax);
1092 masm.subl(edx, eax);
1093 masm.shrl(Imm32(1), eax);
1094
1095 // Finish the computation.
1096 masm.addl(eax, edx);
1097 masm.shrl(Imm32(rmc.shiftAmount - 1), edx);
1098 } else {
1099 masm.shrl(Imm32(rmc.shiftAmount), edx);
1100 }
1101
1102 // We now have the truncated division value in edx. If we're
1103 // computing a modulus or checking whether the division resulted
1104 // in an integer, we need to multiply the obtained value by d and
1105 // finish the computation/check.
1106 if (!isDiv) {
1107 masm.imull(Imm32(d), edx, edx);
1108 masm.movl(lhs, eax);
1109 masm.subl(edx, eax);
1110
1111 // The final result of the modulus op, just computed above by the
1112 // sub instruction, can be a number in the range [2^31, 2^32). If
1113 // this is the case and the modulus is not truncated, we must bail
1114 // out.
1115 if (!ins->mir()->isTruncated()) {
1116 bailoutIf(Assembler::Signed, ins->snapshot());
1117 }
1118 } else if (!ins->mir()->isTruncated()) {
1119 masm.imull(Imm32(d), edx, eax);
1120 masm.cmpl(lhs, eax);
1121 bailoutIf(Assembler::NotEqual, ins->snapshot());
1122 }
1123 }
1124
visitMulNegativeZeroCheck(MulNegativeZeroCheck * ool)1125 void CodeGeneratorX86Shared::visitMulNegativeZeroCheck(
1126 MulNegativeZeroCheck* ool) {
1127 LMulI* ins = ool->ins();
1128 Register result = ToRegister(ins->output());
1129 Operand lhsCopy = ToOperand(ins->lhsCopy());
1130 Operand rhs = ToOperand(ins->rhs());
1131 MOZ_ASSERT_IF(lhsCopy.kind() == Operand::REG, lhsCopy.reg() != result.code());
1132
1133 // Result is -0 if lhs or rhs is negative.
1134 masm.movl(lhsCopy, result);
1135 masm.orl(rhs, result);
1136 bailoutIf(Assembler::Signed, ins->snapshot());
1137
1138 masm.mov(ImmWord(0), result);
1139 masm.jmp(ool->rejoin());
1140 }
1141
visitDivPowTwoI(LDivPowTwoI * ins)1142 void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
1143 Register lhs = ToRegister(ins->numerator());
1144 DebugOnly<Register> output = ToRegister(ins->output());
1145
1146 int32_t shift = ins->shift();
1147 bool negativeDivisor = ins->negativeDivisor();
1148 MDiv* mir = ins->mir();
1149
1150 // We use defineReuseInput so these should always be the same, which is
1151 // convenient since all of our instructions here are two-address.
1152 MOZ_ASSERT(lhs == output);
1153
1154 if (!mir->isTruncated() && negativeDivisor) {
1155 // 0 divided by a negative number must return a double.
1156 masm.test32(lhs, lhs);
1157 bailoutIf(Assembler::Zero, ins->snapshot());
1158 }
1159
1160 if (shift) {
1161 if (!mir->isTruncated()) {
1162 // If the remainder is != 0, bailout since this must be a double.
1163 masm.test32(lhs, Imm32(UINT32_MAX >> (32 - shift)));
1164 bailoutIf(Assembler::NonZero, ins->snapshot());
1165 }
1166
1167 if (mir->isUnsigned()) {
1168 masm.shrl(Imm32(shift), lhs);
1169 } else {
1170 // Adjust the value so that shifting produces a correctly
1171 // rounded result when the numerator is negative. See 10-1
1172 // "Signed Division by a Known Power of 2" in Henry
1173 // S. Warren, Jr.'s Hacker's Delight.
1174 if (mir->canBeNegativeDividend() && mir->isTruncated()) {
1175 // Note: There is no need to execute this code, which handles how to
1176 // round the signed integer division towards 0, if we previously bailed
1177 // due to a non-zero remainder.
1178 Register lhsCopy = ToRegister(ins->numeratorCopy());
1179 MOZ_ASSERT(lhsCopy != lhs);
1180 if (shift > 1) {
1181 // Copy the sign bit of the numerator. (= (2^32 - 1) or 0)
1182 masm.sarl(Imm32(31), lhs);
1183 }
1184 // Divide by 2^(32 - shift)
1185 // i.e. (= (2^32 - 1) / 2^(32 - shift) or 0)
1186 // i.e. (= (2^shift - 1) or 0)
1187 masm.shrl(Imm32(32 - shift), lhs);
1188 // If signed, make any 1 bit below the shifted bits to bubble up, such
1189 // that once shifted the value would be rounded towards 0.
1190 masm.addl(lhsCopy, lhs);
1191 }
1192 masm.sarl(Imm32(shift), lhs);
1193
1194 if (negativeDivisor) {
1195 masm.negl(lhs);
1196 }
1197 }
1198 return;
1199 }
1200
1201 if (negativeDivisor) {
1202 // INT32_MIN / -1 overflows.
1203 masm.negl(lhs);
1204 if (!mir->isTruncated()) {
1205 bailoutIf(Assembler::Overflow, ins->snapshot());
1206 } else if (mir->trapOnError()) {
1207 Label ok;
1208 masm.j(Assembler::NoOverflow, &ok);
1209 masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
1210 masm.bind(&ok);
1211 }
1212 } else if (mir->isUnsigned() && !mir->isTruncated()) {
1213 // Unsigned division by 1 can overflow if output is not
1214 // truncated.
1215 masm.test32(lhs, lhs);
1216 bailoutIf(Assembler::Signed, ins->snapshot());
1217 }
1218 }
1219
visitDivOrModConstantI(LDivOrModConstantI * ins)1220 void CodeGenerator::visitDivOrModConstantI(LDivOrModConstantI* ins) {
1221 Register lhs = ToRegister(ins->numerator());
1222 Register output = ToRegister(ins->output());
1223 int32_t d = ins->denominator();
1224
1225 // This emits the division answer into edx or the modulus answer into eax.
1226 MOZ_ASSERT(output == eax || output == edx);
1227 MOZ_ASSERT(lhs != eax && lhs != edx);
1228 bool isDiv = (output == edx);
1229
1230 // The absolute value of the denominator isn't a power of 2 (see LDivPowTwoI
1231 // and LModPowTwoI).
1232 MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0);
1233
1234 // We will first divide by Abs(d), and negate the answer if d is negative.
1235 // If desired, this can be avoided by generalizing computeDivisionConstants.
1236 ReciprocalMulConstants rmc =
1237 computeDivisionConstants(Abs(d), /* maxLog = */ 31);
1238
1239 // We first compute (M * n) >> 32, where M = rmc.multiplier.
1240 masm.movl(Imm32(rmc.multiplier), eax);
1241 masm.imull(lhs);
1242 if (rmc.multiplier > INT32_MAX) {
1243 MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 32));
1244
1245 // We actually computed edx = ((int32_t(M) * n) >> 32) instead. Since
1246 // (M * n) >> 32 is the same as (edx + n), we can correct for the overflow.
1247 // (edx + n) can't overflow, as n and edx have opposite signs because
1248 // int32_t(M) is negative.
1249 masm.addl(lhs, edx);
1250 }
1251 // (M * n) >> (32 + shift) is the truncated division answer if n is
1252 // non-negative, as proved in the comments of computeDivisionConstants. We
1253 // must add 1 later if n is negative to get the right answer in all cases.
1254 masm.sarl(Imm32(rmc.shiftAmount), edx);
1255
1256 // We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be
1257 // computed with just a sign-extending shift of 31 bits.
1258 if (ins->canBeNegativeDividend()) {
1259 masm.movl(lhs, eax);
1260 masm.sarl(Imm32(31), eax);
1261 masm.subl(eax, edx);
1262 }
1263
1264 // After this, edx contains the correct truncated division result.
1265 if (d < 0) {
1266 masm.negl(edx);
1267 }
1268
1269 if (!isDiv) {
1270 masm.imull(Imm32(-d), edx, eax);
1271 masm.addl(lhs, eax);
1272 }
1273
1274 if (!ins->mir()->isTruncated()) {
1275 if (isDiv) {
1276 // This is a division op. Multiply the obtained value by d to check if
1277 // the correct answer is an integer. This cannot overflow, since |d| > 1.
1278 masm.imull(Imm32(d), edx, eax);
1279 masm.cmp32(lhs, eax);
1280 bailoutIf(Assembler::NotEqual, ins->snapshot());
1281
1282 // If lhs is zero and the divisor is negative, the answer should have
1283 // been -0.
1284 if (d < 0) {
1285 masm.test32(lhs, lhs);
1286 bailoutIf(Assembler::Zero, ins->snapshot());
1287 }
1288 } else if (ins->canBeNegativeDividend()) {
1289 // This is a mod op. If the computed value is zero and lhs
1290 // is negative, the answer should have been -0.
1291 Label done;
1292
1293 masm.cmp32(lhs, Imm32(0));
1294 masm.j(Assembler::GreaterThanOrEqual, &done);
1295
1296 masm.test32(eax, eax);
1297 bailoutIf(Assembler::Zero, ins->snapshot());
1298
1299 masm.bind(&done);
1300 }
1301 }
1302 }
1303
visitDivI(LDivI * ins)1304 void CodeGenerator::visitDivI(LDivI* ins) {
1305 Register remainder = ToRegister(ins->remainder());
1306 Register lhs = ToRegister(ins->lhs());
1307 Register rhs = ToRegister(ins->rhs());
1308 Register output = ToRegister(ins->output());
1309
1310 MDiv* mir = ins->mir();
1311
1312 MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
1313 MOZ_ASSERT(rhs != edx);
1314 MOZ_ASSERT(remainder == edx);
1315 MOZ_ASSERT(output == eax);
1316
1317 Label done;
1318 ReturnZero* ool = nullptr;
1319
1320 // Put the lhs in eax, for either the negative overflow case or the regular
1321 // divide case.
1322 if (lhs != eax) {
1323 masm.mov(lhs, eax);
1324 }
1325
1326 // Handle divide by zero.
1327 if (mir->canBeDivideByZero()) {
1328 masm.test32(rhs, rhs);
1329 if (mir->trapOnError()) {
1330 Label nonZero;
1331 masm.j(Assembler::NonZero, &nonZero);
1332 masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
1333 masm.bind(&nonZero);
1334 } else if (mir->canTruncateInfinities()) {
1335 // Truncated division by zero is zero (Infinity|0 == 0)
1336 if (!ool) {
1337 ool = new (alloc()) ReturnZero(output);
1338 }
1339 masm.j(Assembler::Zero, ool->entry());
1340 } else {
1341 MOZ_ASSERT(mir->fallible());
1342 bailoutIf(Assembler::Zero, ins->snapshot());
1343 }
1344 }
1345
1346 // Handle an integer overflow exception from -2147483648 / -1.
1347 if (mir->canBeNegativeOverflow()) {
1348 Label notOverflow;
1349 masm.cmp32(lhs, Imm32(INT32_MIN));
1350 masm.j(Assembler::NotEqual, ¬Overflow);
1351 masm.cmp32(rhs, Imm32(-1));
1352 if (mir->trapOnError()) {
1353 masm.j(Assembler::NotEqual, ¬Overflow);
1354 masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
1355 } else if (mir->canTruncateOverflow()) {
1356 // (-INT32_MIN)|0 == INT32_MIN and INT32_MIN is already in the
1357 // output register (lhs == eax).
1358 masm.j(Assembler::Equal, &done);
1359 } else {
1360 MOZ_ASSERT(mir->fallible());
1361 bailoutIf(Assembler::Equal, ins->snapshot());
1362 }
1363 masm.bind(¬Overflow);
1364 }
1365
1366 // Handle negative 0.
1367 if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
1368 Label nonzero;
1369 masm.test32(lhs, lhs);
1370 masm.j(Assembler::NonZero, &nonzero);
1371 masm.cmp32(rhs, Imm32(0));
1372 bailoutIf(Assembler::LessThan, ins->snapshot());
1373 masm.bind(&nonzero);
1374 }
1375
1376 // Sign extend the lhs into edx to make (edx:eax), since idiv is 64-bit.
1377 if (lhs != eax) {
1378 masm.mov(lhs, eax);
1379 }
1380 masm.cdq();
1381 masm.idiv(rhs);
1382
1383 if (!mir->canTruncateRemainder()) {
1384 // If the remainder is > 0, bailout since this must be a double.
1385 masm.test32(remainder, remainder);
1386 bailoutIf(Assembler::NonZero, ins->snapshot());
1387 }
1388
1389 masm.bind(&done);
1390
1391 if (ool) {
1392 addOutOfLineCode(ool, mir);
1393 masm.bind(ool->rejoin());
1394 }
1395 }
1396
visitModPowTwoI(LModPowTwoI * ins)1397 void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
1398 Register lhs = ToRegister(ins->getOperand(0));
1399 int32_t shift = ins->shift();
1400
1401 Label negative;
1402
1403 if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
1404 // Switch based on sign of the lhs.
1405 // Positive numbers are just a bitmask
1406 masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
1407 }
1408
1409 masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
1410
1411 if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
1412 Label done;
1413 masm.jump(&done);
1414
1415 // Negative numbers need a negate, bitmask, negate
1416 masm.bind(&negative);
1417
1418 // Unlike in the visitModI case, we are not computing the mod by means of a
1419 // division. Therefore, the divisor = -1 case isn't problematic (the andl
1420 // always returns 0, which is what we expect).
1421 //
1422 // The negl instruction overflows if lhs == INT32_MIN, but this is also not
1423 // a problem: shift is at most 31, and so the andl also always returns 0.
1424 masm.negl(lhs);
1425 masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
1426 masm.negl(lhs);
1427
1428 // Since a%b has the same sign as b, and a is negative in this branch,
1429 // an answer of 0 means the correct result is actually -0. Bail out.
1430 if (!ins->mir()->isTruncated()) {
1431 bailoutIf(Assembler::Zero, ins->snapshot());
1432 }
1433 masm.bind(&done);
1434 }
1435 }
1436
1437 class ModOverflowCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
1438 Label done_;
1439 LModI* ins_;
1440 Register rhs_;
1441
1442 public:
ModOverflowCheck(LModI * ins,Register rhs)1443 explicit ModOverflowCheck(LModI* ins, Register rhs) : ins_(ins), rhs_(rhs) {}
1444
accept(CodeGeneratorX86Shared * codegen)1445 virtual void accept(CodeGeneratorX86Shared* codegen) override {
1446 codegen->visitModOverflowCheck(this);
1447 }
done()1448 Label* done() { return &done_; }
ins() const1449 LModI* ins() const { return ins_; }
rhs() const1450 Register rhs() const { return rhs_; }
1451 };
1452
visitModOverflowCheck(ModOverflowCheck * ool)1453 void CodeGeneratorX86Shared::visitModOverflowCheck(ModOverflowCheck* ool) {
1454 masm.cmp32(ool->rhs(), Imm32(-1));
1455 if (ool->ins()->mir()->isTruncated()) {
1456 masm.j(Assembler::NotEqual, ool->rejoin());
1457 masm.mov(ImmWord(0), edx);
1458 masm.jmp(ool->done());
1459 } else {
1460 bailoutIf(Assembler::Equal, ool->ins()->snapshot());
1461 masm.jmp(ool->rejoin());
1462 }
1463 }
1464
visitModI(LModI * ins)1465 void CodeGenerator::visitModI(LModI* ins) {
1466 Register remainder = ToRegister(ins->remainder());
1467 Register lhs = ToRegister(ins->lhs());
1468 Register rhs = ToRegister(ins->rhs());
1469
1470 // Required to use idiv.
1471 MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
1472 MOZ_ASSERT(rhs != edx);
1473 MOZ_ASSERT(remainder == edx);
1474 MOZ_ASSERT(ToRegister(ins->getTemp(0)) == eax);
1475
1476 Label done;
1477 ReturnZero* ool = nullptr;
1478 ModOverflowCheck* overflow = nullptr;
1479
1480 // Set up eax in preparation for doing a div.
1481 if (lhs != eax) {
1482 masm.mov(lhs, eax);
1483 }
1484
1485 MMod* mir = ins->mir();
1486
1487 // Prevent divide by zero.
1488 if (mir->canBeDivideByZero()) {
1489 masm.test32(rhs, rhs);
1490 if (mir->isTruncated()) {
1491 if (mir->trapOnError()) {
1492 Label nonZero;
1493 masm.j(Assembler::NonZero, &nonZero);
1494 masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
1495 masm.bind(&nonZero);
1496 } else {
1497 if (!ool) {
1498 ool = new (alloc()) ReturnZero(edx);
1499 }
1500 masm.j(Assembler::Zero, ool->entry());
1501 }
1502 } else {
1503 bailoutIf(Assembler::Zero, ins->snapshot());
1504 }
1505 }
1506
1507 Label negative;
1508
1509 // Switch based on sign of the lhs.
1510 if (mir->canBeNegativeDividend()) {
1511 masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
1512 }
1513
1514 // If lhs >= 0 then remainder = lhs % rhs. The remainder must be positive.
1515 {
1516 // Check if rhs is a power-of-two.
1517 if (mir->canBePowerOfTwoDivisor()) {
1518 MOZ_ASSERT(rhs != remainder);
1519
1520 // Rhs y is a power-of-two if (y & (y-1)) == 0. Note that if
1521 // y is any negative number other than INT32_MIN, both y and
1522 // y-1 will have the sign bit set so these are never optimized
1523 // as powers-of-two. If y is INT32_MIN, y-1 will be INT32_MAX
1524 // and because lhs >= 0 at this point, lhs & INT32_MAX returns
1525 // the correct value.
1526 Label notPowerOfTwo;
1527 masm.mov(rhs, remainder);
1528 masm.subl(Imm32(1), remainder);
1529 masm.branchTest32(Assembler::NonZero, remainder, rhs, ¬PowerOfTwo);
1530 {
1531 masm.andl(lhs, remainder);
1532 masm.jmp(&done);
1533 }
1534 masm.bind(¬PowerOfTwo);
1535 }
1536
1537 // Since lhs >= 0, the sign-extension will be 0
1538 masm.mov(ImmWord(0), edx);
1539 masm.idiv(rhs);
1540 }
1541
1542 // Otherwise, we have to beware of two special cases:
1543 if (mir->canBeNegativeDividend()) {
1544 masm.jump(&done);
1545
1546 masm.bind(&negative);
1547
1548 // Prevent an integer overflow exception from -2147483648 % -1
1549 Label notmin;
1550 masm.cmp32(lhs, Imm32(INT32_MIN));
1551 overflow = new (alloc()) ModOverflowCheck(ins, rhs);
1552 masm.j(Assembler::Equal, overflow->entry());
1553 masm.bind(overflow->rejoin());
1554 masm.cdq();
1555 masm.idiv(rhs);
1556
1557 if (!mir->isTruncated()) {
1558 // A remainder of 0 means that the rval must be -0, which is a double.
1559 masm.test32(remainder, remainder);
1560 bailoutIf(Assembler::Zero, ins->snapshot());
1561 }
1562 }
1563
1564 masm.bind(&done);
1565
1566 if (overflow) {
1567 addOutOfLineCode(overflow, mir);
1568 masm.bind(overflow->done());
1569 }
1570
1571 if (ool) {
1572 addOutOfLineCode(ool, mir);
1573 masm.bind(ool->rejoin());
1574 }
1575 }
1576
visitBitNotI(LBitNotI * ins)1577 void CodeGenerator::visitBitNotI(LBitNotI* ins) {
1578 const LAllocation* input = ins->getOperand(0);
1579 MOZ_ASSERT(!input->isConstant());
1580
1581 masm.notl(ToOperand(input));
1582 }
1583
visitBitOpI(LBitOpI * ins)1584 void CodeGenerator::visitBitOpI(LBitOpI* ins) {
1585 const LAllocation* lhs = ins->getOperand(0);
1586 const LAllocation* rhs = ins->getOperand(1);
1587
1588 switch (ins->bitop()) {
1589 case JSOp::BitOr:
1590 if (rhs->isConstant()) {
1591 masm.orl(Imm32(ToInt32(rhs)), ToOperand(lhs));
1592 } else {
1593 masm.orl(ToOperand(rhs), ToRegister(lhs));
1594 }
1595 break;
1596 case JSOp::BitXor:
1597 if (rhs->isConstant()) {
1598 masm.xorl(Imm32(ToInt32(rhs)), ToOperand(lhs));
1599 } else {
1600 masm.xorl(ToOperand(rhs), ToRegister(lhs));
1601 }
1602 break;
1603 case JSOp::BitAnd:
1604 if (rhs->isConstant()) {
1605 masm.andl(Imm32(ToInt32(rhs)), ToOperand(lhs));
1606 } else {
1607 masm.andl(ToOperand(rhs), ToRegister(lhs));
1608 }
1609 break;
1610 default:
1611 MOZ_CRASH("unexpected binary opcode");
1612 }
1613 }
1614
visitBitOpI64(LBitOpI64 * lir)1615 void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
1616 const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
1617 const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
1618
1619 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
1620
1621 switch (lir->bitop()) {
1622 case JSOp::BitOr:
1623 if (IsConstant(rhs)) {
1624 masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
1625 } else {
1626 masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
1627 }
1628 break;
1629 case JSOp::BitXor:
1630 if (IsConstant(rhs)) {
1631 masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
1632 } else {
1633 masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
1634 }
1635 break;
1636 case JSOp::BitAnd:
1637 if (IsConstant(rhs)) {
1638 masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
1639 } else {
1640 masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
1641 }
1642 break;
1643 default:
1644 MOZ_CRASH("unexpected binary opcode");
1645 }
1646 }
1647
visitShiftI(LShiftI * ins)1648 void CodeGenerator::visitShiftI(LShiftI* ins) {
1649 Register lhs = ToRegister(ins->lhs());
1650 const LAllocation* rhs = ins->rhs();
1651
1652 if (rhs->isConstant()) {
1653 int32_t shift = ToInt32(rhs) & 0x1F;
1654 switch (ins->bitop()) {
1655 case JSOp::Lsh:
1656 if (shift) {
1657 masm.lshift32(Imm32(shift), lhs);
1658 }
1659 break;
1660 case JSOp::Rsh:
1661 if (shift) {
1662 masm.rshift32Arithmetic(Imm32(shift), lhs);
1663 }
1664 break;
1665 case JSOp::Ursh:
1666 if (shift) {
1667 masm.rshift32(Imm32(shift), lhs);
1668 } else if (ins->mir()->toUrsh()->fallible()) {
1669 // x >>> 0 can overflow.
1670 masm.test32(lhs, lhs);
1671 bailoutIf(Assembler::Signed, ins->snapshot());
1672 }
1673 break;
1674 default:
1675 MOZ_CRASH("Unexpected shift op");
1676 }
1677 } else {
1678 Register shift = ToRegister(rhs);
1679 switch (ins->bitop()) {
1680 case JSOp::Lsh:
1681 masm.lshift32(shift, lhs);
1682 break;
1683 case JSOp::Rsh:
1684 masm.rshift32Arithmetic(shift, lhs);
1685 break;
1686 case JSOp::Ursh:
1687 masm.rshift32(shift, lhs);
1688 if (ins->mir()->toUrsh()->fallible()) {
1689 // x >>> 0 can overflow.
1690 masm.test32(lhs, lhs);
1691 bailoutIf(Assembler::Signed, ins->snapshot());
1692 }
1693 break;
1694 default:
1695 MOZ_CRASH("Unexpected shift op");
1696 }
1697 }
1698 }
1699
visitShiftI64(LShiftI64 * lir)1700 void CodeGenerator::visitShiftI64(LShiftI64* lir) {
1701 const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
1702 LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
1703
1704 MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
1705
1706 if (rhs->isConstant()) {
1707 int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
1708 switch (lir->bitop()) {
1709 case JSOp::Lsh:
1710 if (shift) {
1711 masm.lshift64(Imm32(shift), ToRegister64(lhs));
1712 }
1713 break;
1714 case JSOp::Rsh:
1715 if (shift) {
1716 masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
1717 }
1718 break;
1719 case JSOp::Ursh:
1720 if (shift) {
1721 masm.rshift64(Imm32(shift), ToRegister64(lhs));
1722 }
1723 break;
1724 default:
1725 MOZ_CRASH("Unexpected shift op");
1726 }
1727 return;
1728 }
1729
1730 Register shift = ToRegister(rhs);
1731 #ifdef JS_CODEGEN_X86
1732 MOZ_ASSERT(shift == ecx);
1733 #endif
1734 switch (lir->bitop()) {
1735 case JSOp::Lsh:
1736 masm.lshift64(shift, ToRegister64(lhs));
1737 break;
1738 case JSOp::Rsh:
1739 masm.rshift64Arithmetic(shift, ToRegister64(lhs));
1740 break;
1741 case JSOp::Ursh:
1742 masm.rshift64(shift, ToRegister64(lhs));
1743 break;
1744 default:
1745 MOZ_CRASH("Unexpected shift op");
1746 }
1747 }
1748
visitUrshD(LUrshD * ins)1749 void CodeGenerator::visitUrshD(LUrshD* ins) {
1750 Register lhs = ToRegister(ins->lhs());
1751 MOZ_ASSERT(ToRegister(ins->temp()) == lhs);
1752
1753 const LAllocation* rhs = ins->rhs();
1754 FloatRegister out = ToFloatRegister(ins->output());
1755
1756 if (rhs->isConstant()) {
1757 int32_t shift = ToInt32(rhs) & 0x1F;
1758 if (shift) {
1759 masm.shrl(Imm32(shift), lhs);
1760 }
1761 } else {
1762 Register shift = ToRegister(rhs);
1763 masm.rshift32(shift, lhs);
1764 }
1765
1766 masm.convertUInt32ToDouble(lhs, out);
1767 }
1768
ToOperand(const LAllocation & a)1769 Operand CodeGeneratorX86Shared::ToOperand(const LAllocation& a) {
1770 if (a.isGeneralReg()) {
1771 return Operand(a.toGeneralReg()->reg());
1772 }
1773 if (a.isFloatReg()) {
1774 return Operand(a.toFloatReg()->reg());
1775 }
1776 return Operand(ToAddress(a));
1777 }
1778
ToOperand(const LAllocation * a)1779 Operand CodeGeneratorX86Shared::ToOperand(const LAllocation* a) {
1780 return ToOperand(*a);
1781 }
1782
ToOperand(const LDefinition * def)1783 Operand CodeGeneratorX86Shared::ToOperand(const LDefinition* def) {
1784 return ToOperand(def->output());
1785 }
1786
toMoveOperand(LAllocation a) const1787 MoveOperand CodeGeneratorX86Shared::toMoveOperand(LAllocation a) const {
1788 if (a.isGeneralReg()) {
1789 return MoveOperand(ToRegister(a));
1790 }
1791 if (a.isFloatReg()) {
1792 return MoveOperand(ToFloatRegister(a));
1793 }
1794 MoveOperand::Kind kind =
1795 a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
1796 return MoveOperand(ToAddress(a), kind);
1797 }
1798
1799 class OutOfLineTableSwitch : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
1800 MTableSwitch* mir_;
1801 CodeLabel jumpLabel_;
1802
accept(CodeGeneratorX86Shared * codegen)1803 void accept(CodeGeneratorX86Shared* codegen) override {
1804 codegen->visitOutOfLineTableSwitch(this);
1805 }
1806
1807 public:
OutOfLineTableSwitch(MTableSwitch * mir)1808 explicit OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {}
1809
mir() const1810 MTableSwitch* mir() const { return mir_; }
1811
jumpLabel()1812 CodeLabel* jumpLabel() { return &jumpLabel_; }
1813 };
1814
visitOutOfLineTableSwitch(OutOfLineTableSwitch * ool)1815 void CodeGeneratorX86Shared::visitOutOfLineTableSwitch(
1816 OutOfLineTableSwitch* ool) {
1817 MTableSwitch* mir = ool->mir();
1818
1819 masm.haltingAlign(sizeof(void*));
1820 masm.bind(ool->jumpLabel());
1821 masm.addCodeLabel(*ool->jumpLabel());
1822
1823 for (size_t i = 0; i < mir->numCases(); i++) {
1824 LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
1825 Label* caseheader = caseblock->label();
1826 uint32_t caseoffset = caseheader->offset();
1827
1828 // The entries of the jump table need to be absolute addresses and thus
1829 // must be patched after codegen is finished.
1830 CodeLabel cl;
1831 masm.writeCodePointer(&cl);
1832 cl.target()->bind(caseoffset);
1833 masm.addCodeLabel(cl);
1834 }
1835 }
1836
emitTableSwitchDispatch(MTableSwitch * mir,Register index,Register base)1837 void CodeGeneratorX86Shared::emitTableSwitchDispatch(MTableSwitch* mir,
1838 Register index,
1839 Register base) {
1840 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
1841
1842 // Lower value with low value
1843 if (mir->low() != 0) {
1844 masm.subl(Imm32(mir->low()), index);
1845 }
1846
1847 // Jump to default case if input is out of range
1848 int32_t cases = mir->numCases();
1849 masm.cmp32(index, Imm32(cases));
1850 masm.j(AssemblerX86Shared::AboveOrEqual, defaultcase);
1851
1852 // To fill in the CodeLabels for the case entries, we need to first
1853 // generate the case entries (we don't yet know their offsets in the
1854 // instruction stream).
1855 OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir);
1856 addOutOfLineCode(ool, mir);
1857
1858 // Compute the position where a pointer to the right case stands.
1859 masm.mov(ool->jumpLabel(), base);
1860 BaseIndex pointer(base, index, ScalePointer);
1861
1862 // Jump to the right case
1863 masm.branchToComputedAddress(pointer);
1864 }
1865
visitMathD(LMathD * math)1866 void CodeGenerator::visitMathD(LMathD* math) {
1867 FloatRegister lhs = ToFloatRegister(math->lhs());
1868 Operand rhs = ToOperand(math->rhs());
1869 FloatRegister output = ToFloatRegister(math->output());
1870
1871 switch (math->jsop()) {
1872 case JSOp::Add:
1873 masm.vaddsd(rhs, lhs, output);
1874 break;
1875 case JSOp::Sub:
1876 masm.vsubsd(rhs, lhs, output);
1877 break;
1878 case JSOp::Mul:
1879 masm.vmulsd(rhs, lhs, output);
1880 break;
1881 case JSOp::Div:
1882 masm.vdivsd(rhs, lhs, output);
1883 break;
1884 default:
1885 MOZ_CRASH("unexpected opcode");
1886 }
1887 }
1888
visitMathF(LMathF * math)1889 void CodeGenerator::visitMathF(LMathF* math) {
1890 FloatRegister lhs = ToFloatRegister(math->lhs());
1891 Operand rhs = ToOperand(math->rhs());
1892 FloatRegister output = ToFloatRegister(math->output());
1893
1894 switch (math->jsop()) {
1895 case JSOp::Add:
1896 masm.vaddss(rhs, lhs, output);
1897 break;
1898 case JSOp::Sub:
1899 masm.vsubss(rhs, lhs, output);
1900 break;
1901 case JSOp::Mul:
1902 masm.vmulss(rhs, lhs, output);
1903 break;
1904 case JSOp::Div:
1905 masm.vdivss(rhs, lhs, output);
1906 break;
1907 default:
1908 MOZ_CRASH("unexpected opcode");
1909 }
1910 }
1911
visitNearbyInt(LNearbyInt * lir)1912 void CodeGenerator::visitNearbyInt(LNearbyInt* lir) {
1913 FloatRegister input = ToFloatRegister(lir->input());
1914 FloatRegister output = ToFloatRegister(lir->output());
1915
1916 RoundingMode roundingMode = lir->mir()->roundingMode();
1917 masm.nearbyIntDouble(roundingMode, input, output);
1918 }
1919
visitNearbyIntF(LNearbyIntF * lir)1920 void CodeGenerator::visitNearbyIntF(LNearbyIntF* lir) {
1921 FloatRegister input = ToFloatRegister(lir->input());
1922 FloatRegister output = ToFloatRegister(lir->output());
1923
1924 RoundingMode roundingMode = lir->mir()->roundingMode();
1925 masm.nearbyIntFloat32(roundingMode, input, output);
1926 }
1927
visitEffectiveAddress(LEffectiveAddress * ins)1928 void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
1929 const MEffectiveAddress* mir = ins->mir();
1930 Register base = ToRegister(ins->base());
1931 Register index = ToRegister(ins->index());
1932 Register output = ToRegister(ins->output());
1933 masm.leal(Operand(base, index, mir->scale(), mir->displacement()), output);
1934 }
1935
generateInvalidateEpilogue()1936 void CodeGeneratorX86Shared::generateInvalidateEpilogue() {
1937 // Ensure that there is enough space in the buffer for the OsiPoint
1938 // patching to occur. Otherwise, we could overwrite the invalidation
1939 // epilogue.
1940 for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
1941 masm.nop();
1942 }
1943
1944 masm.bind(&invalidate_);
1945
1946 // Push the Ion script onto the stack (when we determine what that pointer
1947 // is).
1948 invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
1949
1950 // Jump to the invalidator which will replace the current frame.
1951 TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
1952 masm.jump(thunk);
1953 }
1954
visitNegI(LNegI * ins)1955 void CodeGenerator::visitNegI(LNegI* ins) {
1956 Register input = ToRegister(ins->input());
1957 MOZ_ASSERT(input == ToRegister(ins->output()));
1958
1959 masm.neg32(input);
1960 }
1961
visitNegI64(LNegI64 * ins)1962 void CodeGenerator::visitNegI64(LNegI64* ins) {
1963 Register64 input = ToRegister64(ins->getInt64Operand(0));
1964 MOZ_ASSERT(input == ToOutRegister64(ins));
1965 masm.neg64(input);
1966 }
1967
visitNegD(LNegD * ins)1968 void CodeGenerator::visitNegD(LNegD* ins) {
1969 FloatRegister input = ToFloatRegister(ins->input());
1970 MOZ_ASSERT(input == ToFloatRegister(ins->output()));
1971
1972 masm.negateDouble(input);
1973 }
1974
visitNegF(LNegF * ins)1975 void CodeGenerator::visitNegF(LNegF* ins) {
1976 FloatRegister input = ToFloatRegister(ins->input());
1977 MOZ_ASSERT(input == ToFloatRegister(ins->output()));
1978
1979 masm.negateFloat(input);
1980 }
1981
visitCompareExchangeTypedArrayElement(LCompareExchangeTypedArrayElement * lir)1982 void CodeGenerator::visitCompareExchangeTypedArrayElement(
1983 LCompareExchangeTypedArrayElement* lir) {
1984 Register elements = ToRegister(lir->elements());
1985 AnyRegister output = ToAnyRegister(lir->output());
1986 Register temp =
1987 lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
1988
1989 Register oldval = ToRegister(lir->oldval());
1990 Register newval = ToRegister(lir->newval());
1991
1992 Scalar::Type arrayType = lir->mir()->arrayType();
1993
1994 if (lir->index()->isConstant()) {
1995 Address dest = ToAddress(elements, lir->index(), arrayType);
1996 masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
1997 newval, temp, output);
1998 } else {
1999 BaseIndex dest(elements, ToRegister(lir->index()),
2000 ScaleFromScalarType(arrayType));
2001 masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
2002 newval, temp, output);
2003 }
2004 }
2005
visitAtomicExchangeTypedArrayElement(LAtomicExchangeTypedArrayElement * lir)2006 void CodeGenerator::visitAtomicExchangeTypedArrayElement(
2007 LAtomicExchangeTypedArrayElement* lir) {
2008 Register elements = ToRegister(lir->elements());
2009 AnyRegister output = ToAnyRegister(lir->output());
2010 Register temp =
2011 lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
2012
2013 Register value = ToRegister(lir->value());
2014
2015 Scalar::Type arrayType = lir->mir()->arrayType();
2016
2017 if (lir->index()->isConstant()) {
2018 Address dest = ToAddress(elements, lir->index(), arrayType);
2019 masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
2020 output);
2021 } else {
2022 BaseIndex dest(elements, ToRegister(lir->index()),
2023 ScaleFromScalarType(arrayType));
2024 masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
2025 output);
2026 }
2027 }
2028
2029 template <typename T>
AtomicBinopToTypedArray(MacroAssembler & masm,AtomicOp op,Scalar::Type arrayType,const LAllocation * value,const T & mem,Register temp1,Register temp2,AnyRegister output)2030 static inline void AtomicBinopToTypedArray(MacroAssembler& masm, AtomicOp op,
2031 Scalar::Type arrayType,
2032 const LAllocation* value,
2033 const T& mem, Register temp1,
2034 Register temp2, AnyRegister output) {
2035 if (value->isConstant()) {
2036 masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op,
2037 Imm32(ToInt32(value)), mem, temp1, temp2, output);
2038 } else {
2039 masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op,
2040 ToRegister(value), mem, temp1, temp2, output);
2041 }
2042 }
2043
visitAtomicTypedArrayElementBinop(LAtomicTypedArrayElementBinop * lir)2044 void CodeGenerator::visitAtomicTypedArrayElementBinop(
2045 LAtomicTypedArrayElementBinop* lir) {
2046 MOZ_ASSERT(!lir->mir()->isForEffect());
2047
2048 AnyRegister output = ToAnyRegister(lir->output());
2049 Register elements = ToRegister(lir->elements());
2050 Register temp1 =
2051 lir->temp1()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp1());
2052 Register temp2 =
2053 lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2());
2054 const LAllocation* value = lir->value();
2055
2056 Scalar::Type arrayType = lir->mir()->arrayType();
2057
2058 if (lir->index()->isConstant()) {
2059 Address mem = ToAddress(elements, lir->index(), arrayType);
2060 AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value,
2061 mem, temp1, temp2, output);
2062 } else {
2063 BaseIndex mem(elements, ToRegister(lir->index()),
2064 ScaleFromScalarType(arrayType));
2065 AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value,
2066 mem, temp1, temp2, output);
2067 }
2068 }
2069
2070 template <typename T>
AtomicBinopToTypedArray(MacroAssembler & masm,Scalar::Type arrayType,AtomicOp op,const LAllocation * value,const T & mem)2071 static inline void AtomicBinopToTypedArray(MacroAssembler& masm,
2072 Scalar::Type arrayType, AtomicOp op,
2073 const LAllocation* value,
2074 const T& mem) {
2075 if (value->isConstant()) {
2076 masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op,
2077 Imm32(ToInt32(value)), mem, InvalidReg);
2078 } else {
2079 masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op,
2080 ToRegister(value), mem, InvalidReg);
2081 }
2082 }
2083
visitAtomicTypedArrayElementBinopForEffect(LAtomicTypedArrayElementBinopForEffect * lir)2084 void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
2085 LAtomicTypedArrayElementBinopForEffect* lir) {
2086 MOZ_ASSERT(lir->mir()->isForEffect());
2087
2088 Register elements = ToRegister(lir->elements());
2089 const LAllocation* value = lir->value();
2090 Scalar::Type arrayType = lir->mir()->arrayType();
2091
2092 if (lir->index()->isConstant()) {
2093 Address mem = ToAddress(elements, lir->index(), arrayType);
2094 AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value,
2095 mem);
2096 } else {
2097 BaseIndex mem(elements, ToRegister(lir->index()),
2098 ScaleFromScalarType(arrayType));
2099 AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value,
2100 mem);
2101 }
2102 }
2103
visitMemoryBarrier(LMemoryBarrier * ins)2104 void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
2105 if (ins->type() & MembarStoreLoad) {
2106 masm.storeLoadFence();
2107 }
2108 }
2109
visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck * ool)2110 void CodeGeneratorX86Shared::visitOutOfLineWasmTruncateCheck(
2111 OutOfLineWasmTruncateCheck* ool) {
2112 FloatRegister input = ool->input();
2113 Register output = ool->output();
2114 Register64 output64 = ool->output64();
2115 MIRType fromType = ool->fromType();
2116 MIRType toType = ool->toType();
2117 Label* oolRejoin = ool->rejoin();
2118 TruncFlags flags = ool->flags();
2119 wasm::BytecodeOffset off = ool->bytecodeOffset();
2120
2121 if (fromType == MIRType::Float32) {
2122 if (toType == MIRType::Int32) {
2123 masm.oolWasmTruncateCheckF32ToI32(input, output, flags, off, oolRejoin);
2124 } else if (toType == MIRType::Int64) {
2125 masm.oolWasmTruncateCheckF32ToI64(input, output64, flags, off, oolRejoin);
2126 } else {
2127 MOZ_CRASH("unexpected type");
2128 }
2129 } else if (fromType == MIRType::Double) {
2130 if (toType == MIRType::Int32) {
2131 masm.oolWasmTruncateCheckF64ToI32(input, output, flags, off, oolRejoin);
2132 } else if (toType == MIRType::Int64) {
2133 masm.oolWasmTruncateCheckF64ToI64(input, output64, flags, off, oolRejoin);
2134 } else {
2135 MOZ_CRASH("unexpected type");
2136 }
2137 } else {
2138 MOZ_CRASH("unexpected type");
2139 }
2140 }
2141
canonicalizeIfDeterministic(Scalar::Type type,const LAllocation * value)2142 void CodeGeneratorX86Shared::canonicalizeIfDeterministic(
2143 Scalar::Type type, const LAllocation* value) {
2144 #ifdef DEBUG
2145 if (!js::SupportDifferentialTesting()) {
2146 return;
2147 }
2148
2149 switch (type) {
2150 case Scalar::Float32: {
2151 FloatRegister in = ToFloatRegister(value);
2152 masm.canonicalizeFloatIfDeterministic(in);
2153 break;
2154 }
2155 case Scalar::Float64: {
2156 FloatRegister in = ToFloatRegister(value);
2157 masm.canonicalizeDoubleIfDeterministic(in);
2158 break;
2159 }
2160 default: {
2161 // Other types don't need canonicalization.
2162 break;
2163 }
2164 }
2165 #endif // DEBUG
2166 }
2167
2168 template <typename T>
toMemoryAccessOperand(T * lir,int32_t disp)2169 Operand CodeGeneratorX86Shared::toMemoryAccessOperand(T* lir, int32_t disp) {
2170 const LAllocation* ptr = lir->ptr();
2171 #ifdef JS_CODEGEN_X86
2172 const LAllocation* memoryBase = lir->memoryBase();
2173 Operand destAddr = ptr->isBogus() ? Operand(ToRegister(memoryBase), disp)
2174 : Operand(ToRegister(memoryBase),
2175 ToRegister(ptr), TimesOne, disp);
2176 #else
2177 Operand destAddr = ptr->isBogus()
2178 ? Operand(HeapReg, disp)
2179 : Operand(HeapReg, ToRegister(ptr), TimesOne, disp);
2180 #endif
2181 return destAddr;
2182 }
2183
visitCopySignF(LCopySignF * lir)2184 void CodeGenerator::visitCopySignF(LCopySignF* lir) {
2185 FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
2186 FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
2187
2188 FloatRegister out = ToFloatRegister(lir->output());
2189
2190 if (lhs == rhs) {
2191 if (lhs != out) {
2192 masm.moveFloat32(lhs, out);
2193 }
2194 return;
2195 }
2196
2197 masm.copySignFloat32(lhs, rhs, out);
2198 }
2199
visitCopySignD(LCopySignD * lir)2200 void CodeGenerator::visitCopySignD(LCopySignD* lir) {
2201 FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
2202 FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
2203
2204 FloatRegister out = ToFloatRegister(lir->output());
2205
2206 if (lhs == rhs) {
2207 if (lhs != out) {
2208 masm.moveDouble(lhs, out);
2209 }
2210 return;
2211 }
2212
2213 masm.copySignDouble(lhs, rhs, out);
2214 }
2215
visitRotateI64(LRotateI64 * lir)2216 void CodeGenerator::visitRotateI64(LRotateI64* lir) {
2217 MRotate* mir = lir->mir();
2218 LAllocation* count = lir->count();
2219
2220 Register64 input = ToRegister64(lir->input());
2221 Register64 output = ToOutRegister64(lir);
2222 Register temp = ToTempRegisterOrInvalid(lir->temp());
2223
2224 MOZ_ASSERT(input == output);
2225
2226 if (count->isConstant()) {
2227 int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
2228 if (!c) {
2229 return;
2230 }
2231 if (mir->isLeftRotate()) {
2232 masm.rotateLeft64(Imm32(c), input, output, temp);
2233 } else {
2234 masm.rotateRight64(Imm32(c), input, output, temp);
2235 }
2236 } else {
2237 if (mir->isLeftRotate()) {
2238 masm.rotateLeft64(ToRegister(count), input, output, temp);
2239 } else {
2240 masm.rotateRight64(ToRegister(count), input, output, temp);
2241 }
2242 }
2243 }
2244
visitPopcntI64(LPopcntI64 * lir)2245 void CodeGenerator::visitPopcntI64(LPopcntI64* lir) {
2246 Register64 input = ToRegister64(lir->getInt64Operand(0));
2247 Register64 output = ToOutRegister64(lir);
2248 Register temp = InvalidReg;
2249 if (!AssemblerX86Shared::HasPOPCNT()) {
2250 temp = ToRegister(lir->getTemp(0));
2251 }
2252
2253 masm.popcnt64(input, output, temp);
2254 }
2255
visitSimd128(LSimd128 * ins)2256 void CodeGenerator::visitSimd128(LSimd128* ins) {
2257 #ifdef ENABLE_WASM_SIMD
2258 const LDefinition* out = ins->getDef(0);
2259 masm.loadConstantSimd128(ins->simd128(), ToFloatRegister(out));
2260 #else
2261 MOZ_CRASH("No SIMD");
2262 #endif
2263 }
2264
visitWasmTernarySimd128(LWasmTernarySimd128 * ins)2265 void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
2266 #ifdef ENABLE_WASM_SIMD
2267 switch (ins->simdOp()) {
2268 case wasm::SimdOp::V128Bitselect: {
2269 FloatRegister lhsDest = ToFloatRegister(ins->v0());
2270 FloatRegister rhs = ToFloatRegister(ins->v1());
2271 FloatRegister control = ToFloatRegister(ins->v2());
2272 FloatRegister temp = ToFloatRegister(ins->temp());
2273 masm.bitwiseSelectSimd128(control, lhsDest, rhs, lhsDest, temp);
2274 break;
2275 }
2276 case wasm::SimdOp::F32x4RelaxedFma:
2277 masm.fmaFloat32x4(ToFloatRegister(ins->v1()), ToFloatRegister(ins->v2()),
2278 ToFloatRegister(ins->v0()));
2279 break;
2280 case wasm::SimdOp::F32x4RelaxedFms:
2281 masm.fmsFloat32x4(ToFloatRegister(ins->v1()), ToFloatRegister(ins->v2()),
2282 ToFloatRegister(ins->v0()));
2283 break;
2284 case wasm::SimdOp::F64x2RelaxedFma:
2285 masm.fmaFloat64x2(ToFloatRegister(ins->v1()), ToFloatRegister(ins->v2()),
2286 ToFloatRegister(ins->v0()));
2287 break;
2288 case wasm::SimdOp::F64x2RelaxedFms:
2289 masm.fmsFloat64x2(ToFloatRegister(ins->v1()), ToFloatRegister(ins->v2()),
2290 ToFloatRegister(ins->v0()));
2291 break;
2292 case wasm::SimdOp::I8x16LaneSelect:
2293 case wasm::SimdOp::I16x8LaneSelect:
2294 case wasm::SimdOp::I32x4LaneSelect:
2295 case wasm::SimdOp::I64x2LaneSelect: {
2296 FloatRegister lhs = ToFloatRegister(ins->v0());
2297 FloatRegister rhs = ToFloatRegister(ins->v1());
2298 FloatRegister mask = ToFloatRegister(ins->v2());
2299 FloatRegister dest = ToFloatRegister(ins->output());
2300 masm.laneSelectSimd128(mask, lhs, rhs, dest);
2301 break;
2302 }
2303 default:
2304 MOZ_CRASH("NYI");
2305 }
2306 #else
2307 MOZ_CRASH("No SIMD");
2308 #endif
2309 }
2310
visitWasmBinarySimd128(LWasmBinarySimd128 * ins)2311 void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
2312 #ifdef ENABLE_WASM_SIMD
2313 FloatRegister lhs = ToFloatRegister(ins->lhsDest());
2314 FloatRegister lhsDest = lhs;
2315 FloatRegister rhs = ToFloatRegister(ins->rhs());
2316 FloatRegister temp1 = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
2317 FloatRegister temp2 = ToTempFloatRegisterOrInvalid(ins->getTemp(1));
2318 FloatRegister dest = ToFloatRegister(ins->output());
2319
2320 switch (ins->simdOp()) {
2321 case wasm::SimdOp::V128And:
2322 masm.bitwiseAndSimd128(lhs, rhs, dest);
2323 break;
2324 case wasm::SimdOp::V128Or:
2325 masm.bitwiseOrSimd128(lhs, rhs, dest);
2326 break;
2327 case wasm::SimdOp::V128Xor:
2328 masm.bitwiseXorSimd128(lhs, rhs, dest);
2329 break;
2330 case wasm::SimdOp::V128AndNot:
2331 // x86/x64 specific: The CPU provides ~A & B. The operands were swapped
2332 // during lowering, and we'll compute A & ~B here as desired.
2333 masm.bitwiseNotAndSimd128(lhs, rhs, dest);
2334 break;
2335 case wasm::SimdOp::I8x16AvgrU:
2336 masm.unsignedAverageInt8x16(rhs, lhsDest);
2337 break;
2338 case wasm::SimdOp::I16x8AvgrU:
2339 masm.unsignedAverageInt16x8(rhs, lhsDest);
2340 break;
2341 case wasm::SimdOp::I8x16Add:
2342 masm.addInt8x16(rhs, lhsDest);
2343 break;
2344 case wasm::SimdOp::I8x16AddSatS:
2345 masm.addSatInt8x16(rhs, lhsDest);
2346 break;
2347 case wasm::SimdOp::I8x16AddSatU:
2348 masm.unsignedAddSatInt8x16(rhs, lhsDest);
2349 break;
2350 case wasm::SimdOp::I8x16Sub:
2351 masm.subInt8x16(rhs, lhsDest);
2352 break;
2353 case wasm::SimdOp::I8x16SubSatS:
2354 masm.subSatInt8x16(rhs, lhsDest);
2355 break;
2356 case wasm::SimdOp::I8x16SubSatU:
2357 masm.unsignedSubSatInt8x16(rhs, lhsDest);
2358 break;
2359 case wasm::SimdOp::I8x16MinS:
2360 masm.minInt8x16(rhs, lhsDest);
2361 break;
2362 case wasm::SimdOp::I8x16MinU:
2363 masm.unsignedMinInt8x16(rhs, lhsDest);
2364 break;
2365 case wasm::SimdOp::I8x16MaxS:
2366 masm.maxInt8x16(rhs, lhsDest);
2367 break;
2368 case wasm::SimdOp::I8x16MaxU:
2369 masm.unsignedMaxInt8x16(rhs, lhsDest);
2370 break;
2371 case wasm::SimdOp::I16x8Add:
2372 masm.addInt16x8(rhs, lhsDest);
2373 break;
2374 case wasm::SimdOp::I16x8AddSatS:
2375 masm.addSatInt16x8(rhs, lhsDest);
2376 break;
2377 case wasm::SimdOp::I16x8AddSatU:
2378 masm.unsignedAddSatInt16x8(rhs, lhsDest);
2379 break;
2380 case wasm::SimdOp::I16x8Sub:
2381 masm.subInt16x8(rhs, lhsDest);
2382 break;
2383 case wasm::SimdOp::I16x8SubSatS:
2384 masm.subSatInt16x8(rhs, lhsDest);
2385 break;
2386 case wasm::SimdOp::I16x8SubSatU:
2387 masm.unsignedSubSatInt16x8(rhs, lhsDest);
2388 break;
2389 case wasm::SimdOp::I16x8Mul:
2390 masm.mulInt16x8(rhs, lhsDest);
2391 break;
2392 case wasm::SimdOp::I16x8MinS:
2393 masm.minInt16x8(rhs, lhsDest);
2394 break;
2395 case wasm::SimdOp::I16x8MinU:
2396 masm.unsignedMinInt16x8(rhs, lhsDest);
2397 break;
2398 case wasm::SimdOp::I16x8MaxS:
2399 masm.maxInt16x8(rhs, lhsDest);
2400 break;
2401 case wasm::SimdOp::I16x8MaxU:
2402 masm.unsignedMaxInt16x8(rhs, lhsDest);
2403 break;
2404 case wasm::SimdOp::I32x4Add:
2405 masm.addInt32x4(lhs, rhs, dest);
2406 break;
2407 case wasm::SimdOp::I32x4Sub:
2408 masm.subInt32x4(lhs, rhs, dest);
2409 break;
2410 case wasm::SimdOp::I32x4Mul:
2411 masm.mulInt32x4(lhs, rhs, dest);
2412 break;
2413 case wasm::SimdOp::I32x4MinS:
2414 masm.minInt32x4(rhs, lhsDest);
2415 break;
2416 case wasm::SimdOp::I32x4MinU:
2417 masm.unsignedMinInt32x4(rhs, lhsDest);
2418 break;
2419 case wasm::SimdOp::I32x4MaxS:
2420 masm.maxInt32x4(rhs, lhsDest);
2421 break;
2422 case wasm::SimdOp::I32x4MaxU:
2423 masm.unsignedMaxInt32x4(rhs, lhsDest);
2424 break;
2425 case wasm::SimdOp::I64x2Add:
2426 masm.addInt64x2(rhs, lhsDest);
2427 break;
2428 case wasm::SimdOp::I64x2Sub:
2429 masm.subInt64x2(rhs, lhsDest);
2430 break;
2431 case wasm::SimdOp::I64x2Mul:
2432 masm.mulInt64x2(lhsDest, rhs, lhsDest, temp1);
2433 break;
2434 case wasm::SimdOp::F32x4Add:
2435 masm.addFloat32x4(lhs, rhs, dest);
2436 break;
2437 case wasm::SimdOp::F32x4Sub:
2438 masm.subFloat32x4(lhs, rhs, dest);
2439 break;
2440 case wasm::SimdOp::F32x4Mul:
2441 masm.mulFloat32x4(lhs, rhs, dest);
2442 break;
2443 case wasm::SimdOp::F32x4Div:
2444 masm.divFloat32x4(lhs, rhs, dest);
2445 break;
2446 case wasm::SimdOp::F32x4Min:
2447 masm.minFloat32x4(lhs, rhs, dest, temp1, temp2);
2448 break;
2449 case wasm::SimdOp::F32x4Max:
2450 masm.maxFloat32x4(lhs, rhs, dest, temp1, temp2);
2451 break;
2452 case wasm::SimdOp::F64x2Add:
2453 masm.addFloat64x2(rhs, lhsDest);
2454 break;
2455 case wasm::SimdOp::F64x2Sub:
2456 masm.subFloat64x2(rhs, lhsDest);
2457 break;
2458 case wasm::SimdOp::F64x2Mul:
2459 masm.mulFloat64x2(rhs, lhsDest);
2460 break;
2461 case wasm::SimdOp::F64x2Div:
2462 masm.divFloat64x2(rhs, lhsDest);
2463 break;
2464 case wasm::SimdOp::F64x2Min:
2465 masm.minFloat64x2(lhs, rhs, dest, temp1, temp2);
2466 break;
2467 case wasm::SimdOp::F64x2Max:
2468 masm.maxFloat64x2(lhs, rhs, dest, temp1, temp2);
2469 break;
2470 case wasm::SimdOp::I8x16Swizzle:
2471 masm.swizzleInt8x16(rhs, lhsDest);
2472 break;
2473 case wasm::SimdOp::V8x16RelaxedSwizzle:
2474 masm.swizzleInt8x16Relaxed(rhs, lhsDest);
2475 break;
2476 case wasm::SimdOp::I8x16NarrowI16x8S:
2477 masm.narrowInt16x8(rhs, lhsDest);
2478 break;
2479 case wasm::SimdOp::I8x16NarrowI16x8U:
2480 masm.unsignedNarrowInt16x8(rhs, lhsDest);
2481 break;
2482 case wasm::SimdOp::I16x8NarrowI32x4S:
2483 masm.narrowInt32x4(rhs, lhsDest);
2484 break;
2485 case wasm::SimdOp::I16x8NarrowI32x4U:
2486 masm.unsignedNarrowInt32x4(rhs, lhsDest);
2487 break;
2488 case wasm::SimdOp::I8x16Eq:
2489 masm.compareInt8x16(Assembler::Equal, rhs, lhsDest);
2490 break;
2491 case wasm::SimdOp::I8x16Ne:
2492 masm.compareInt8x16(Assembler::NotEqual, rhs, lhsDest);
2493 break;
2494 case wasm::SimdOp::I8x16LtS:
2495 masm.compareInt8x16(Assembler::LessThan, rhs, lhsDest);
2496 break;
2497 case wasm::SimdOp::I8x16GtS:
2498 masm.compareInt8x16(Assembler::GreaterThan, rhs, lhsDest);
2499 break;
2500 case wasm::SimdOp::I8x16LeS:
2501 masm.compareInt8x16(Assembler::LessThanOrEqual, rhs, lhsDest);
2502 break;
2503 case wasm::SimdOp::I8x16GeS:
2504 masm.compareInt8x16(Assembler::GreaterThanOrEqual, rhs, lhsDest);
2505 break;
2506 case wasm::SimdOp::I8x16LtU:
2507 masm.compareInt8x16(Assembler::Below, rhs, lhsDest);
2508 break;
2509 case wasm::SimdOp::I8x16GtU:
2510 masm.compareInt8x16(Assembler::Above, rhs, lhsDest);
2511 break;
2512 case wasm::SimdOp::I8x16LeU:
2513 masm.compareInt8x16(Assembler::BelowOrEqual, rhs, lhsDest);
2514 break;
2515 case wasm::SimdOp::I8x16GeU:
2516 masm.compareInt8x16(Assembler::AboveOrEqual, rhs, lhsDest);
2517 break;
2518 case wasm::SimdOp::I16x8Eq:
2519 masm.compareInt16x8(Assembler::Equal, rhs, lhsDest);
2520 break;
2521 case wasm::SimdOp::I16x8Ne:
2522 masm.compareInt16x8(Assembler::NotEqual, rhs, lhsDest);
2523 break;
2524 case wasm::SimdOp::I16x8LtS:
2525 masm.compareInt16x8(Assembler::LessThan, rhs, lhsDest);
2526 break;
2527 case wasm::SimdOp::I16x8GtS:
2528 masm.compareInt16x8(Assembler::GreaterThan, rhs, lhsDest);
2529 break;
2530 case wasm::SimdOp::I16x8LeS:
2531 masm.compareInt16x8(Assembler::LessThanOrEqual, rhs, lhsDest);
2532 break;
2533 case wasm::SimdOp::I16x8GeS:
2534 masm.compareInt16x8(Assembler::GreaterThanOrEqual, rhs, lhsDest);
2535 break;
2536 case wasm::SimdOp::I16x8LtU:
2537 masm.compareInt16x8(Assembler::Below, rhs, lhsDest);
2538 break;
2539 case wasm::SimdOp::I16x8GtU:
2540 masm.compareInt16x8(Assembler::Above, rhs, lhsDest);
2541 break;
2542 case wasm::SimdOp::I16x8LeU:
2543 masm.compareInt16x8(Assembler::BelowOrEqual, rhs, lhsDest);
2544 break;
2545 case wasm::SimdOp::I16x8GeU:
2546 masm.compareInt16x8(Assembler::AboveOrEqual, rhs, lhsDest);
2547 break;
2548 case wasm::SimdOp::I32x4Eq:
2549 masm.compareInt32x4(Assembler::Equal, lhs, rhs, dest);
2550 break;
2551 case wasm::SimdOp::I32x4Ne:
2552 masm.compareInt32x4(Assembler::NotEqual, lhs, rhs, dest);
2553 break;
2554 case wasm::SimdOp::I32x4LtS:
2555 masm.compareInt32x4(Assembler::LessThan, lhs, rhs, dest);
2556 break;
2557 case wasm::SimdOp::I32x4GtS:
2558 masm.compareInt32x4(Assembler::GreaterThan, lhs, rhs, dest);
2559 break;
2560 case wasm::SimdOp::I32x4LeS:
2561 masm.compareInt32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
2562 break;
2563 case wasm::SimdOp::I32x4GeS:
2564 masm.compareInt32x4(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
2565 break;
2566 case wasm::SimdOp::I32x4LtU:
2567 masm.compareInt32x4(Assembler::Below, lhs, rhs, dest);
2568 break;
2569 case wasm::SimdOp::I32x4GtU:
2570 masm.compareInt32x4(Assembler::Above, lhs, rhs, dest);
2571 break;
2572 case wasm::SimdOp::I32x4LeU:
2573 masm.compareInt32x4(Assembler::BelowOrEqual, lhs, rhs, dest);
2574 break;
2575 case wasm::SimdOp::I32x4GeU:
2576 masm.compareInt32x4(Assembler::AboveOrEqual, lhs, rhs, dest);
2577 break;
2578 case wasm::SimdOp::I64x2Eq:
2579 masm.compareForEqualityInt64x2(Assembler::Equal, rhs, lhsDest);
2580 break;
2581 case wasm::SimdOp::I64x2Ne:
2582 masm.compareForEqualityInt64x2(Assembler::NotEqual, rhs, lhsDest);
2583 break;
2584 case wasm::SimdOp::I64x2LtS:
2585 masm.compareForOrderingInt64x2(Assembler::LessThan, rhs, lhsDest, temp1,
2586 temp2);
2587 break;
2588 case wasm::SimdOp::I64x2GtS:
2589 masm.compareForOrderingInt64x2(Assembler::GreaterThan, rhs, lhsDest,
2590 temp1, temp2);
2591 break;
2592 case wasm::SimdOp::I64x2LeS:
2593 masm.compareForOrderingInt64x2(Assembler::LessThanOrEqual, rhs, lhsDest,
2594 temp1, temp2);
2595 break;
2596 case wasm::SimdOp::I64x2GeS:
2597 masm.compareForOrderingInt64x2(Assembler::GreaterThanOrEqual, rhs,
2598 lhsDest, temp1, temp2);
2599 break;
2600 case wasm::SimdOp::F32x4Eq:
2601 masm.compareFloat32x4(Assembler::Equal, lhs, rhs, dest);
2602 break;
2603 case wasm::SimdOp::F32x4Ne:
2604 masm.compareFloat32x4(Assembler::NotEqual, lhs, rhs, dest);
2605 break;
2606 case wasm::SimdOp::F32x4Lt:
2607 masm.compareFloat32x4(Assembler::LessThan, lhs, rhs, dest);
2608 break;
2609 case wasm::SimdOp::F32x4Le:
2610 masm.compareFloat32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
2611 break;
2612 case wasm::SimdOp::F64x2Eq:
2613 masm.compareFloat64x2(Assembler::Equal, rhs, lhsDest);
2614 break;
2615 case wasm::SimdOp::F64x2Ne:
2616 masm.compareFloat64x2(Assembler::NotEqual, rhs, lhsDest);
2617 break;
2618 case wasm::SimdOp::F64x2Lt:
2619 masm.compareFloat64x2(Assembler::LessThan, rhs, lhsDest);
2620 break;
2621 case wasm::SimdOp::F64x2Le:
2622 masm.compareFloat64x2(Assembler::LessThanOrEqual, rhs, lhsDest);
2623 break;
2624 case wasm::SimdOp::F32x4PMax:
2625 // `lhsDest` is actually rhsDest, and `rhs` is actually lhs
2626 masm.pseudoMaxFloat32x4(lhsDest, rhs);
2627 break;
2628 case wasm::SimdOp::F32x4PMin:
2629 // `lhsDest` is actually rhsDest, and `rhs` is actually lhs
2630 masm.pseudoMinFloat32x4(lhsDest, rhs);
2631 break;
2632 case wasm::SimdOp::F64x2PMax:
2633 // `lhsDest` is actually rhsDest, and `rhs` is actually lhs
2634 masm.pseudoMaxFloat64x2(lhsDest, rhs);
2635 break;
2636 case wasm::SimdOp::F64x2PMin:
2637 // `lhsDest` is actually rhsDest, and `rhs` is actually lhs
2638 masm.pseudoMinFloat64x2(lhsDest, rhs);
2639 break;
2640 case wasm::SimdOp::I32x4DotI16x8S:
2641 masm.widenDotInt16x8(rhs, lhsDest);
2642 break;
2643 case wasm::SimdOp::I16x8ExtmulLowI8x16S:
2644 masm.extMulLowInt8x16(rhs, lhsDest);
2645 break;
2646 case wasm::SimdOp::I16x8ExtmulHighI8x16S:
2647 masm.extMulHighInt8x16(rhs, lhsDest);
2648 break;
2649 case wasm::SimdOp::I16x8ExtmulLowI8x16U:
2650 masm.unsignedExtMulLowInt8x16(rhs, lhsDest);
2651 break;
2652 case wasm::SimdOp::I16x8ExtmulHighI8x16U:
2653 masm.unsignedExtMulHighInt8x16(rhs, lhsDest);
2654 break;
2655 case wasm::SimdOp::I32x4ExtmulLowI16x8S:
2656 masm.extMulLowInt16x8(rhs, lhsDest);
2657 break;
2658 case wasm::SimdOp::I32x4ExtmulHighI16x8S:
2659 masm.extMulHighInt16x8(rhs, lhsDest);
2660 break;
2661 case wasm::SimdOp::I32x4ExtmulLowI16x8U:
2662 masm.unsignedExtMulLowInt16x8(rhs, lhsDest);
2663 break;
2664 case wasm::SimdOp::I32x4ExtmulHighI16x8U:
2665 masm.unsignedExtMulHighInt16x8(rhs, lhsDest);
2666 break;
2667 case wasm::SimdOp::I64x2ExtmulLowI32x4S:
2668 masm.extMulLowInt32x4(rhs, lhsDest);
2669 break;
2670 case wasm::SimdOp::I64x2ExtmulHighI32x4S:
2671 masm.extMulHighInt32x4(rhs, lhsDest);
2672 break;
2673 case wasm::SimdOp::I64x2ExtmulLowI32x4U:
2674 masm.unsignedExtMulLowInt32x4(rhs, lhsDest);
2675 break;
2676 case wasm::SimdOp::I64x2ExtmulHighI32x4U:
2677 masm.unsignedExtMulHighInt32x4(rhs, lhsDest);
2678 break;
2679 case wasm::SimdOp::I16x8Q15MulrSatS:
2680 masm.q15MulrSatInt16x8(rhs, lhsDest);
2681 break;
2682 case wasm::SimdOp::F32x4RelaxedMin:
2683 masm.minFloat32x4Relaxed(rhs, lhsDest);
2684 break;
2685 case wasm::SimdOp::F32x4RelaxedMax:
2686 masm.maxFloat32x4Relaxed(rhs, lhsDest);
2687 break;
2688 case wasm::SimdOp::F64x2RelaxedMin:
2689 masm.minFloat64x2Relaxed(rhs, lhsDest);
2690 break;
2691 case wasm::SimdOp::F64x2RelaxedMax:
2692 masm.maxFloat64x2Relaxed(rhs, lhsDest);
2693 break;
2694 # ifdef ENABLE_WASM_SIMD_WORMHOLE
2695 case wasm::SimdOp::MozWHSELFTEST:
2696 masm.loadConstantSimd128(wasm::WormholeSignature(), lhsDest);
2697 break;
2698 case wasm::SimdOp::MozWHPMADDUBSW:
2699 masm.vpmaddubsw(rhs, lhsDest, lhsDest);
2700 break;
2701 case wasm::SimdOp::MozWHPMADDWD:
2702 masm.vpmaddwd(Operand(rhs), lhsDest, lhsDest);
2703 break;
2704 # endif
2705 default:
2706 MOZ_CRASH("Binary SimdOp not implemented");
2707 }
2708 #else
2709 MOZ_CRASH("No SIMD");
2710 #endif
2711 }
2712
visitWasmBinarySimd128WithConstant(LWasmBinarySimd128WithConstant * ins)2713 void CodeGenerator::visitWasmBinarySimd128WithConstant(
2714 LWasmBinarySimd128WithConstant* ins) {
2715 #ifdef ENABLE_WASM_SIMD
2716 FloatRegister lhs = ToFloatRegister(ins->lhsDest());
2717 const SimdConstant& rhs = ins->rhs();
2718 FloatRegister dest = ToFloatRegister(ins->output());
2719
2720 switch (ins->simdOp()) {
2721 case wasm::SimdOp::I8x16Add:
2722 masm.addInt8x16(lhs, rhs, dest);
2723 break;
2724 case wasm::SimdOp::I16x8Add:
2725 masm.addInt16x8(lhs, rhs, dest);
2726 break;
2727 case wasm::SimdOp::I32x4Add:
2728 masm.addInt32x4(lhs, rhs, dest);
2729 break;
2730 case wasm::SimdOp::I64x2Add:
2731 masm.addInt64x2(lhs, rhs, dest);
2732 break;
2733 case wasm::SimdOp::I8x16Sub:
2734 masm.subInt8x16(lhs, rhs, dest);
2735 break;
2736 case wasm::SimdOp::I16x8Sub:
2737 masm.subInt16x8(lhs, rhs, dest);
2738 break;
2739 case wasm::SimdOp::I32x4Sub:
2740 masm.subInt32x4(lhs, rhs, dest);
2741 break;
2742 case wasm::SimdOp::I64x2Sub:
2743 masm.subInt64x2(lhs, rhs, dest);
2744 break;
2745 case wasm::SimdOp::I16x8Mul:
2746 masm.mulInt16x8(lhs, rhs, dest);
2747 break;
2748 case wasm::SimdOp::I32x4Mul:
2749 masm.mulInt32x4(lhs, rhs, dest);
2750 break;
2751 case wasm::SimdOp::I8x16AddSatS:
2752 masm.addSatInt8x16(lhs, rhs, dest);
2753 break;
2754 case wasm::SimdOp::I8x16AddSatU:
2755 masm.unsignedAddSatInt8x16(lhs, rhs, dest);
2756 break;
2757 case wasm::SimdOp::I16x8AddSatS:
2758 masm.addSatInt16x8(lhs, rhs, dest);
2759 break;
2760 case wasm::SimdOp::I16x8AddSatU:
2761 masm.unsignedAddSatInt16x8(lhs, rhs, dest);
2762 break;
2763 case wasm::SimdOp::I8x16SubSatS:
2764 masm.subSatInt8x16(lhs, rhs, dest);
2765 break;
2766 case wasm::SimdOp::I8x16SubSatU:
2767 masm.unsignedSubSatInt8x16(lhs, rhs, dest);
2768 break;
2769 case wasm::SimdOp::I16x8SubSatS:
2770 masm.subSatInt16x8(lhs, rhs, dest);
2771 break;
2772 case wasm::SimdOp::I16x8SubSatU:
2773 masm.unsignedSubSatInt16x8(lhs, rhs, dest);
2774 break;
2775 case wasm::SimdOp::I8x16MinS:
2776 masm.minInt8x16(lhs, rhs, dest);
2777 break;
2778 case wasm::SimdOp::I8x16MinU:
2779 masm.unsignedMinInt8x16(lhs, rhs, dest);
2780 break;
2781 case wasm::SimdOp::I16x8MinS:
2782 masm.minInt16x8(lhs, rhs, dest);
2783 break;
2784 case wasm::SimdOp::I16x8MinU:
2785 masm.unsignedMinInt16x8(lhs, rhs, dest);
2786 break;
2787 case wasm::SimdOp::I32x4MinS:
2788 masm.minInt32x4(lhs, rhs, dest);
2789 break;
2790 case wasm::SimdOp::I32x4MinU:
2791 masm.unsignedMinInt32x4(lhs, rhs, dest);
2792 break;
2793 case wasm::SimdOp::I8x16MaxS:
2794 masm.maxInt8x16(lhs, rhs, dest);
2795 break;
2796 case wasm::SimdOp::I8x16MaxU:
2797 masm.unsignedMaxInt8x16(lhs, rhs, dest);
2798 break;
2799 case wasm::SimdOp::I16x8MaxS:
2800 masm.maxInt16x8(lhs, rhs, dest);
2801 break;
2802 case wasm::SimdOp::I16x8MaxU:
2803 masm.unsignedMaxInt16x8(lhs, rhs, dest);
2804 break;
2805 case wasm::SimdOp::I32x4MaxS:
2806 masm.maxInt32x4(lhs, rhs, dest);
2807 break;
2808 case wasm::SimdOp::I32x4MaxU:
2809 masm.unsignedMaxInt32x4(lhs, rhs, dest);
2810 break;
2811 case wasm::SimdOp::V128And:
2812 masm.bitwiseAndSimd128(lhs, rhs, dest);
2813 break;
2814 case wasm::SimdOp::V128Or:
2815 masm.bitwiseOrSimd128(lhs, rhs, dest);
2816 break;
2817 case wasm::SimdOp::V128Xor:
2818 masm.bitwiseXorSimd128(lhs, rhs, dest);
2819 break;
2820 case wasm::SimdOp::I8x16Eq:
2821 masm.compareInt8x16(Assembler::Equal, lhs, rhs, dest);
2822 break;
2823 case wasm::SimdOp::I8x16Ne:
2824 masm.compareInt8x16(Assembler::NotEqual, lhs, rhs, dest);
2825 break;
2826 case wasm::SimdOp::I8x16GtS:
2827 masm.compareInt8x16(Assembler::GreaterThan, lhs, rhs, dest);
2828 break;
2829 case wasm::SimdOp::I8x16LeS:
2830 masm.compareInt8x16(Assembler::LessThanOrEqual, lhs, rhs, dest);
2831 break;
2832 case wasm::SimdOp::I16x8Eq:
2833 masm.compareInt16x8(Assembler::Equal, lhs, rhs, dest);
2834 break;
2835 case wasm::SimdOp::I16x8Ne:
2836 masm.compareInt16x8(Assembler::NotEqual, lhs, rhs, dest);
2837 break;
2838 case wasm::SimdOp::I16x8GtS:
2839 masm.compareInt16x8(Assembler::GreaterThan, lhs, rhs, dest);
2840 break;
2841 case wasm::SimdOp::I16x8LeS:
2842 masm.compareInt16x8(Assembler::LessThanOrEqual, lhs, rhs, dest);
2843 break;
2844 case wasm::SimdOp::I32x4Eq:
2845 masm.compareInt32x4(Assembler::Equal, lhs, rhs, dest);
2846 break;
2847 case wasm::SimdOp::I32x4Ne:
2848 masm.compareInt32x4(Assembler::NotEqual, lhs, rhs, dest);
2849 break;
2850 case wasm::SimdOp::I32x4GtS:
2851 masm.compareInt32x4(Assembler::GreaterThan, lhs, rhs, dest);
2852 break;
2853 case wasm::SimdOp::I32x4LeS:
2854 masm.compareInt32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
2855 break;
2856 case wasm::SimdOp::F32x4Eq:
2857 masm.compareFloat32x4(Assembler::Equal, lhs, rhs, dest);
2858 break;
2859 case wasm::SimdOp::F32x4Ne:
2860 masm.compareFloat32x4(Assembler::NotEqual, lhs, rhs, dest);
2861 break;
2862 case wasm::SimdOp::F32x4Lt:
2863 masm.compareFloat32x4(Assembler::LessThan, lhs, rhs, dest);
2864 break;
2865 case wasm::SimdOp::F32x4Le:
2866 masm.compareFloat32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
2867 break;
2868 case wasm::SimdOp::F64x2Eq:
2869 masm.compareFloat64x2(Assembler::Equal, lhs, rhs, dest);
2870 break;
2871 case wasm::SimdOp::F64x2Ne:
2872 masm.compareFloat64x2(Assembler::NotEqual, lhs, rhs, dest);
2873 break;
2874 case wasm::SimdOp::F64x2Lt:
2875 masm.compareFloat64x2(Assembler::LessThan, lhs, rhs, dest);
2876 break;
2877 case wasm::SimdOp::F64x2Le:
2878 masm.compareFloat64x2(Assembler::LessThanOrEqual, lhs, rhs, dest);
2879 break;
2880 case wasm::SimdOp::I32x4DotI16x8S:
2881 masm.widenDotInt16x8(lhs, rhs, dest);
2882 break;
2883 case wasm::SimdOp::F32x4Add:
2884 masm.addFloat32x4(lhs, rhs, dest);
2885 break;
2886 case wasm::SimdOp::F64x2Add:
2887 masm.addFloat64x2(lhs, rhs, dest);
2888 break;
2889 case wasm::SimdOp::F32x4Sub:
2890 masm.subFloat32x4(lhs, rhs, dest);
2891 break;
2892 case wasm::SimdOp::F64x2Sub:
2893 masm.subFloat64x2(lhs, rhs, dest);
2894 break;
2895 case wasm::SimdOp::F32x4Div:
2896 masm.divFloat32x4(lhs, rhs, dest);
2897 break;
2898 case wasm::SimdOp::F64x2Div:
2899 masm.divFloat64x2(lhs, rhs, dest);
2900 break;
2901 case wasm::SimdOp::F32x4Mul:
2902 masm.mulFloat32x4(lhs, rhs, dest);
2903 break;
2904 case wasm::SimdOp::F64x2Mul:
2905 masm.mulFloat64x2(lhs, rhs, dest);
2906 break;
2907 case wasm::SimdOp::I8x16NarrowI16x8S:
2908 masm.narrowInt16x8(lhs, rhs, dest);
2909 break;
2910 case wasm::SimdOp::I8x16NarrowI16x8U:
2911 masm.unsignedNarrowInt16x8(lhs, rhs, dest);
2912 break;
2913 case wasm::SimdOp::I16x8NarrowI32x4S:
2914 masm.narrowInt32x4(lhs, rhs, dest);
2915 break;
2916 case wasm::SimdOp::I16x8NarrowI32x4U:
2917 masm.unsignedNarrowInt32x4(lhs, rhs, dest);
2918 break;
2919 default:
2920 MOZ_CRASH("Binary SimdOp with constant not implemented");
2921 }
2922 #else
2923 MOZ_CRASH("No SIMD");
2924 #endif
2925 }
2926
visitWasmVariableShiftSimd128(LWasmVariableShiftSimd128 * ins)2927 void CodeGenerator::visitWasmVariableShiftSimd128(
2928 LWasmVariableShiftSimd128* ins) {
2929 #ifdef ENABLE_WASM_SIMD
2930 FloatRegister lhsDest = ToFloatRegister(ins->lhsDest());
2931 Register rhs = ToRegister(ins->rhs());
2932 FloatRegister temp = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
2933
2934 MOZ_ASSERT(ToFloatRegister(ins->output()) == lhsDest);
2935
2936 switch (ins->simdOp()) {
2937 case wasm::SimdOp::I8x16Shl:
2938 masm.leftShiftInt8x16(rhs, lhsDest, temp);
2939 break;
2940 case wasm::SimdOp::I8x16ShrS:
2941 masm.rightShiftInt8x16(rhs, lhsDest, temp);
2942 break;
2943 case wasm::SimdOp::I8x16ShrU:
2944 masm.unsignedRightShiftInt8x16(rhs, lhsDest, temp);
2945 break;
2946 case wasm::SimdOp::I16x8Shl:
2947 masm.leftShiftInt16x8(rhs, lhsDest);
2948 break;
2949 case wasm::SimdOp::I16x8ShrS:
2950 masm.rightShiftInt16x8(rhs, lhsDest);
2951 break;
2952 case wasm::SimdOp::I16x8ShrU:
2953 masm.unsignedRightShiftInt16x8(rhs, lhsDest);
2954 break;
2955 case wasm::SimdOp::I32x4Shl:
2956 masm.leftShiftInt32x4(rhs, lhsDest);
2957 break;
2958 case wasm::SimdOp::I32x4ShrS:
2959 masm.rightShiftInt32x4(rhs, lhsDest);
2960 break;
2961 case wasm::SimdOp::I32x4ShrU:
2962 masm.unsignedRightShiftInt32x4(rhs, lhsDest);
2963 break;
2964 case wasm::SimdOp::I64x2Shl:
2965 masm.leftShiftInt64x2(rhs, lhsDest);
2966 break;
2967 case wasm::SimdOp::I64x2ShrS:
2968 masm.rightShiftInt64x2(rhs, lhsDest, temp);
2969 break;
2970 case wasm::SimdOp::I64x2ShrU:
2971 masm.unsignedRightShiftInt64x2(rhs, lhsDest);
2972 break;
2973 default:
2974 MOZ_CRASH("Shift SimdOp not implemented");
2975 }
2976 #else
2977 MOZ_CRASH("No SIMD");
2978 #endif
2979 }
2980
visitWasmConstantShiftSimd128(LWasmConstantShiftSimd128 * ins)2981 void CodeGenerator::visitWasmConstantShiftSimd128(
2982 LWasmConstantShiftSimd128* ins) {
2983 #ifdef ENABLE_WASM_SIMD
2984 FloatRegister src = ToFloatRegister(ins->src());
2985 FloatRegister dest = ToFloatRegister(ins->output());
2986 int32_t shift = ins->shift();
2987
2988 if (shift == 0) {
2989 masm.moveSimd128(src, dest);
2990 return;
2991 }
2992
2993 switch (ins->simdOp()) {
2994 case wasm::SimdOp::I8x16Shl:
2995 masm.leftShiftInt8x16(Imm32(shift), src, dest);
2996 break;
2997 case wasm::SimdOp::I8x16ShrS:
2998 masm.rightShiftInt8x16(Imm32(shift), src, dest);
2999 break;
3000 case wasm::SimdOp::I8x16ShrU:
3001 masm.unsignedRightShiftInt8x16(Imm32(shift), src, dest);
3002 break;
3003 case wasm::SimdOp::I16x8Shl:
3004 masm.leftShiftInt16x8(Imm32(shift), src, dest);
3005 break;
3006 case wasm::SimdOp::I16x8ShrS:
3007 masm.rightShiftInt16x8(Imm32(shift), src, dest);
3008 break;
3009 case wasm::SimdOp::I16x8ShrU:
3010 masm.unsignedRightShiftInt16x8(Imm32(shift), src, dest);
3011 break;
3012 case wasm::SimdOp::I32x4Shl:
3013 masm.leftShiftInt32x4(Imm32(shift), src, dest);
3014 break;
3015 case wasm::SimdOp::I32x4ShrS:
3016 masm.rightShiftInt32x4(Imm32(shift), src, dest);
3017 break;
3018 case wasm::SimdOp::I32x4ShrU:
3019 masm.unsignedRightShiftInt32x4(Imm32(shift), src, dest);
3020 break;
3021 case wasm::SimdOp::I64x2Shl:
3022 masm.leftShiftInt64x2(Imm32(shift), src, dest);
3023 break;
3024 case wasm::SimdOp::I64x2ShrS:
3025 masm.rightShiftInt64x2(Imm32(shift), src, dest);
3026 break;
3027 case wasm::SimdOp::I64x2ShrU:
3028 masm.unsignedRightShiftInt64x2(Imm32(shift), src, dest);
3029 break;
3030 default:
3031 MOZ_CRASH("Shift SimdOp not implemented");
3032 }
3033 #else
3034 MOZ_CRASH("No SIMD");
3035 #endif
3036 }
3037
visitWasmSignReplicationSimd128(LWasmSignReplicationSimd128 * ins)3038 void CodeGenerator::visitWasmSignReplicationSimd128(
3039 LWasmSignReplicationSimd128* ins) {
3040 #ifdef ENABLE_WASM_SIMD
3041 FloatRegister src = ToFloatRegister(ins->src());
3042 FloatRegister dest = ToFloatRegister(ins->output());
3043
3044 switch (ins->simdOp()) {
3045 case wasm::SimdOp::I8x16ShrS:
3046 masm.signReplicationInt8x16(src, dest);
3047 break;
3048 case wasm::SimdOp::I16x8ShrS:
3049 masm.signReplicationInt16x8(src, dest);
3050 break;
3051 case wasm::SimdOp::I32x4ShrS:
3052 masm.signReplicationInt32x4(src, dest);
3053 break;
3054 case wasm::SimdOp::I64x2ShrS:
3055 masm.signReplicationInt64x2(src, dest);
3056 break;
3057 default:
3058 MOZ_CRASH("Shift SimdOp unsupported sign replication optimization");
3059 }
3060 #else
3061 MOZ_CRASH("No SIMD");
3062 #endif
3063 }
3064
visitWasmShuffleSimd128(LWasmShuffleSimd128 * ins)3065 void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
3066 #ifdef ENABLE_WASM_SIMD
3067 FloatRegister lhsDest = ToFloatRegister(ins->lhsDest());
3068 FloatRegister rhs = ToFloatRegister(ins->rhs());
3069 SimdConstant control = ins->control();
3070 FloatRegister output = ToFloatRegister(ins->output());
3071 switch (ins->op()) {
3072 case SimdShuffleOp::BLEND_8x16: {
3073 masm.blendInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
3074 lhsDest, rhs, output, ToFloatRegister(ins->temp()));
3075 break;
3076 }
3077 case SimdShuffleOp::BLEND_16x8: {
3078 MOZ_ASSERT(ins->temp()->isBogusTemp());
3079 masm.blendInt16x8(reinterpret_cast<const uint16_t*>(control.asInt16x8()),
3080 lhsDest, rhs, output);
3081 break;
3082 }
3083 case SimdShuffleOp::CONCAT_RIGHT_SHIFT_8x16: {
3084 MOZ_ASSERT(ins->temp()->isBogusTemp());
3085 int8_t count = 16 - control.asInt8x16()[0];
3086 MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
3087 masm.concatAndRightShiftSimd128(lhsDest, rhs, output, count);
3088 break;
3089 }
3090 case SimdShuffleOp::INTERLEAVE_HIGH_8x16: {
3091 MOZ_ASSERT(ins->temp()->isBogusTemp());
3092 masm.interleaveHighInt8x16(lhsDest, rhs, output);
3093 break;
3094 }
3095 case SimdShuffleOp::INTERLEAVE_HIGH_16x8: {
3096 MOZ_ASSERT(ins->temp()->isBogusTemp());
3097 masm.interleaveHighInt16x8(lhsDest, rhs, output);
3098 break;
3099 }
3100 case SimdShuffleOp::INTERLEAVE_HIGH_32x4: {
3101 MOZ_ASSERT(ins->temp()->isBogusTemp());
3102 masm.interleaveHighInt32x4(lhsDest, rhs, output);
3103 break;
3104 }
3105 case SimdShuffleOp::INTERLEAVE_HIGH_64x2: {
3106 MOZ_ASSERT(ins->temp()->isBogusTemp());
3107 masm.interleaveHighInt64x2(lhsDest, rhs, output);
3108 break;
3109 }
3110 case SimdShuffleOp::INTERLEAVE_LOW_8x16: {
3111 MOZ_ASSERT(ins->temp()->isBogusTemp());
3112 masm.interleaveLowInt8x16(lhsDest, rhs, output);
3113 break;
3114 }
3115 case SimdShuffleOp::INTERLEAVE_LOW_16x8: {
3116 MOZ_ASSERT(ins->temp()->isBogusTemp());
3117 masm.interleaveLowInt16x8(lhsDest, rhs, output);
3118 break;
3119 }
3120 case SimdShuffleOp::INTERLEAVE_LOW_32x4: {
3121 MOZ_ASSERT(ins->temp()->isBogusTemp());
3122 masm.interleaveLowInt32x4(lhsDest, rhs, output);
3123 break;
3124 }
3125 case SimdShuffleOp::INTERLEAVE_LOW_64x2: {
3126 MOZ_ASSERT(ins->temp()->isBogusTemp());
3127 masm.interleaveLowInt64x2(lhsDest, rhs, output);
3128 break;
3129 }
3130 case SimdShuffleOp::SHUFFLE_BLEND_8x16: {
3131 masm.shuffleInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
3132 lhsDest, rhs, output);
3133 break;
3134 }
3135 default: {
3136 MOZ_CRASH("Unsupported SIMD shuffle operation");
3137 }
3138 }
3139 #else
3140 MOZ_CRASH("No SIMD");
3141 #endif
3142 }
3143
3144 #ifdef ENABLE_WASM_SIMD
3145
3146 enum PermuteX64I16x8Action : uint16_t {
3147 UNAVAILABLE = 0,
3148 SWAP_QWORDS = 1, // Swap qwords first
3149 PERM_LOW = 2, // Permute low qword by control_[0..3]
3150 PERM_HIGH = 4 // Permute high qword by control_[4..7]
3151 };
3152
3153 // Skip lanes that equal v starting at i, returning the index just beyond the
3154 // last of those. There is no requirement that the initial lanes[i] == v.
3155 template <typename T>
ScanConstant(const T * lanes,int v,int i)3156 static int ScanConstant(const T* lanes, int v, int i) {
3157 int len = int(16 / sizeof(T));
3158 MOZ_ASSERT(i <= len);
3159 while (i < len && lanes[i] == v) {
3160 i++;
3161 }
3162 return i;
3163 }
3164
3165 // Apply a transformation to each lane value.
3166 template <typename T>
MapLanes(T * result,const T * input,int (* f)(int))3167 static void MapLanes(T* result, const T* input, int (*f)(int)) {
3168 int len = int(16 / sizeof(T));
3169 for (int i = 0; i < len; i++) {
3170 result[i] = f(input[i]);
3171 }
3172 }
3173
3174 // Recognize part of an identity permutation starting at start, with
3175 // the first value of the permutation expected to be bias.
3176 template <typename T>
IsIdentity(const T * lanes,int start,int len,int bias)3177 static bool IsIdentity(const T* lanes, int start, int len, int bias) {
3178 if (lanes[start] != bias) {
3179 return false;
3180 }
3181 for (int i = start + 1; i < start + len; i++) {
3182 if (lanes[i] != lanes[i - 1] + 1) {
3183 return false;
3184 }
3185 }
3186 return true;
3187 }
3188
3189 // We can permute by words if the mask is reducible to a word mask, but the x64
3190 // lowering is only efficient if we can permute the high and low quadwords
3191 // separately, possibly after swapping quadwords.
CalculateX64Permute16x8(SimdConstant * control)3192 static PermuteX64I16x8Action CalculateX64Permute16x8(SimdConstant* control) {
3193 const SimdConstant::I16x8& lanes = control->asInt16x8();
3194 SimdConstant::I16x8 mapped;
3195 MapLanes(mapped, lanes, [](int x) -> int { return x < 4 ? 0 : 1; });
3196 int i = ScanConstant(mapped, mapped[0], 0);
3197 if (i != 4) {
3198 return PermuteX64I16x8Action::UNAVAILABLE;
3199 }
3200 i = ScanConstant(mapped, mapped[4], 4);
3201 if (i != 8) {
3202 return PermuteX64I16x8Action::UNAVAILABLE;
3203 }
3204 // Now compute the operation bits. `mapped` holds the adjusted lane mask.
3205 memcpy(mapped, lanes, sizeof(mapped));
3206 uint16_t op = 0;
3207 if (mapped[0] > mapped[4]) {
3208 op |= PermuteX64I16x8Action::SWAP_QWORDS;
3209 }
3210 for (auto& m : mapped) {
3211 m &= 3;
3212 }
3213 if (!IsIdentity(mapped, 0, 4, 0)) {
3214 op |= PermuteX64I16x8Action::PERM_LOW;
3215 }
3216 if (!IsIdentity(mapped, 4, 4, 0)) {
3217 op |= PermuteX64I16x8Action::PERM_HIGH;
3218 }
3219 MOZ_ASSERT(op != PermuteX64I16x8Action::UNAVAILABLE);
3220 *control = SimdConstant::CreateX8(mapped);
3221 return (PermuteX64I16x8Action)op;
3222 }
3223
3224 #endif
3225
visitWasmPermuteSimd128(LWasmPermuteSimd128 * ins)3226 void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
3227 #ifdef ENABLE_WASM_SIMD
3228 FloatRegister src = ToFloatRegister(ins->src());
3229 FloatRegister dest = ToFloatRegister(ins->output());
3230 SimdConstant control = ins->control();
3231 switch (ins->op()) {
3232 // For broadcast, would MOVDDUP be better than PSHUFD for the last step?
3233 case SimdPermuteOp::BROADCAST_8x16: {
3234 const SimdConstant::I8x16& mask = control.asInt8x16();
3235 int8_t source = mask[0];
3236 if (source == 0 && Assembler::HasAVX2()) {
3237 masm.vbroadcastb(Operand(src), dest);
3238 break;
3239 }
3240 MOZ_ASSERT_IF(!Assembler::HasAVX(), src == dest);
3241 if (source < 8) {
3242 masm.interleaveLowInt8x16(src, src, dest);
3243 } else {
3244 masm.interleaveHighInt8x16(src, src, dest);
3245 source -= 8;
3246 }
3247 uint16_t v = uint16_t(source & 3);
3248 uint16_t wordMask[4] = {v, v, v, v};
3249 if (source < 4) {
3250 masm.permuteLowInt16x8(wordMask, dest, dest);
3251 uint32_t dwordMask[4] = {0, 0, 0, 0};
3252 masm.permuteInt32x4(dwordMask, dest, dest);
3253 } else {
3254 masm.permuteHighInt16x8(wordMask, dest, dest);
3255 uint32_t dwordMask[4] = {2, 2, 2, 2};
3256 masm.permuteInt32x4(dwordMask, dest, dest);
3257 }
3258 break;
3259 }
3260 case SimdPermuteOp::BROADCAST_16x8: {
3261 const SimdConstant::I16x8& mask = control.asInt16x8();
3262 int16_t source = mask[0];
3263 if (source == 0 && Assembler::HasAVX2()) {
3264 masm.vbroadcastw(Operand(src), dest);
3265 break;
3266 }
3267 uint16_t v = uint16_t(source & 3);
3268 uint16_t wordMask[4] = {v, v, v, v};
3269 if (source < 4) {
3270 masm.permuteLowInt16x8(wordMask, src, dest);
3271 uint32_t dwordMask[4] = {0, 0, 0, 0};
3272 masm.permuteInt32x4(dwordMask, dest, dest);
3273 } else {
3274 masm.permuteHighInt16x8(wordMask, src, dest);
3275 uint32_t dwordMask[4] = {2, 2, 2, 2};
3276 masm.permuteInt32x4(dwordMask, dest, dest);
3277 }
3278 break;
3279 }
3280 case SimdPermuteOp::MOVE: {
3281 masm.moveSimd128(src, dest);
3282 break;
3283 }
3284 case SimdPermuteOp::PERMUTE_8x16: {
3285 const SimdConstant::I8x16& mask = control.asInt8x16();
3286 # ifdef DEBUG
3287 DebugOnly<int> i;
3288 for (i = 0; i < 16 && mask[i] == i; i++) {
3289 }
3290 MOZ_ASSERT(i < 16, "Should have been a MOVE operation");
3291 # endif
3292 masm.permuteInt8x16(reinterpret_cast<const uint8_t*>(mask), src, dest);
3293 break;
3294 }
3295 case SimdPermuteOp::PERMUTE_16x8: {
3296 # ifdef DEBUG
3297 const SimdConstant::I16x8& mask = control.asInt16x8();
3298 DebugOnly<int> i;
3299 for (i = 0; i < 8 && mask[i] == i; i++) {
3300 }
3301 MOZ_ASSERT(i < 8, "Should have been a MOVE operation");
3302 # endif
3303 PermuteX64I16x8Action op = CalculateX64Permute16x8(&control);
3304 if (op != PermuteX64I16x8Action::UNAVAILABLE) {
3305 const SimdConstant::I16x8& mask = control.asInt16x8();
3306 if (op & PermuteX64I16x8Action::SWAP_QWORDS) {
3307 uint32_t dwordMask[4] = {2, 3, 0, 1};
3308 masm.permuteInt32x4(dwordMask, src, dest);
3309 src = dest;
3310 }
3311 if (op & PermuteX64I16x8Action::PERM_LOW) {
3312 masm.permuteLowInt16x8(reinterpret_cast<const uint16_t*>(mask) + 0,
3313 src, dest);
3314 src = dest;
3315 }
3316 if (op & PermuteX64I16x8Action::PERM_HIGH) {
3317 masm.permuteHighInt16x8(reinterpret_cast<const uint16_t*>(mask) + 4,
3318 src, dest);
3319 src = dest;
3320 }
3321 } else {
3322 const SimdConstant::I16x8& wmask = control.asInt16x8();
3323 uint8_t mask[16];
3324 for (unsigned i = 0; i < 16; i += 2) {
3325 mask[i] = wmask[i / 2] * 2;
3326 mask[i + 1] = wmask[i / 2] * 2 + 1;
3327 }
3328 masm.permuteInt8x16(mask, src, dest);
3329 }
3330 break;
3331 }
3332 case SimdPermuteOp::PERMUTE_32x4: {
3333 const SimdConstant::I32x4& mask = control.asInt32x4();
3334 if (Assembler::HasAVX2() && mask[0] == 0 && mask[1] == 0 &&
3335 mask[2] == 0 && mask[3] == 0) {
3336 masm.vbroadcastd(Operand(src), dest);
3337 break;
3338 }
3339 # ifdef DEBUG
3340 DebugOnly<int> i;
3341 for (i = 0; i < 4 && mask[i] == i; i++) {
3342 }
3343 MOZ_ASSERT(i < 4, "Should have been a MOVE operation");
3344 # endif
3345 masm.permuteInt32x4(reinterpret_cast<const uint32_t*>(mask), src, dest);
3346 break;
3347 }
3348 case SimdPermuteOp::ROTATE_RIGHT_8x16: {
3349 MOZ_ASSERT_IF(!Assembler::HasAVX(), src == dest);
3350 int8_t count = control.asInt8x16()[0];
3351 MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
3352 masm.concatAndRightShiftSimd128(src, src, dest, count);
3353 break;
3354 }
3355 case SimdPermuteOp::SHIFT_LEFT_8x16: {
3356 int8_t count = control.asInt8x16()[0];
3357 MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
3358 masm.leftShiftSimd128(Imm32(count), src, dest);
3359 break;
3360 }
3361 case SimdPermuteOp::SHIFT_RIGHT_8x16: {
3362 int8_t count = control.asInt8x16()[0];
3363 MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
3364 masm.rightShiftSimd128(Imm32(count), src, dest);
3365 break;
3366 }
3367 case SimdPermuteOp::REVERSE_16x8:
3368 masm.reverseInt16x8(src, dest);
3369 break;
3370 case SimdPermuteOp::REVERSE_32x4:
3371 masm.reverseInt32x4(src, dest);
3372 break;
3373 case SimdPermuteOp::REVERSE_64x2:
3374 masm.reverseInt64x2(src, dest);
3375 break;
3376 default: {
3377 MOZ_CRASH("Unsupported SIMD permutation operation");
3378 }
3379 }
3380 #else
3381 MOZ_CRASH("No SIMD");
3382 #endif
3383 }
3384
visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128 * ins)3385 void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
3386 #ifdef ENABLE_WASM_SIMD
3387 FloatRegister lhsDest = ToFloatRegister(ins->lhsDest());
3388 const LAllocation* rhs = ins->rhs();
3389 uint32_t laneIndex = ins->laneIndex();
3390
3391 switch (ins->simdOp()) {
3392 case wasm::SimdOp::I8x16ReplaceLane:
3393 masm.replaceLaneInt8x16(laneIndex, ToRegister(rhs), lhsDest);
3394 break;
3395 case wasm::SimdOp::I16x8ReplaceLane:
3396 masm.replaceLaneInt16x8(laneIndex, ToRegister(rhs), lhsDest);
3397 break;
3398 case wasm::SimdOp::I32x4ReplaceLane:
3399 masm.replaceLaneInt32x4(laneIndex, ToRegister(rhs), lhsDest);
3400 break;
3401 case wasm::SimdOp::F32x4ReplaceLane:
3402 masm.replaceLaneFloat32x4(laneIndex, ToFloatRegister(rhs), lhsDest);
3403 break;
3404 case wasm::SimdOp::F64x2ReplaceLane:
3405 masm.replaceLaneFloat64x2(laneIndex, ToFloatRegister(rhs), lhsDest);
3406 break;
3407 default:
3408 MOZ_CRASH("ReplaceLane SimdOp not implemented");
3409 }
3410 #else
3411 MOZ_CRASH("No SIMD");
3412 #endif
3413 }
3414
visitWasmReplaceInt64LaneSimd128(LWasmReplaceInt64LaneSimd128 * ins)3415 void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
3416 LWasmReplaceInt64LaneSimd128* ins) {
3417 #ifdef ENABLE_WASM_SIMD
3418 MOZ_RELEASE_ASSERT(ins->simdOp() == wasm::SimdOp::I64x2ReplaceLane);
3419 masm.replaceLaneInt64x2(ins->laneIndex(), ToFloatRegister(ins->lhs()),
3420 ToRegister64(ins->rhs()),
3421 ToFloatRegister(ins->output()));
3422 #else
3423 MOZ_CRASH("No SIMD");
3424 #endif
3425 }
3426
visitWasmScalarToSimd128(LWasmScalarToSimd128 * ins)3427 void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
3428 #ifdef ENABLE_WASM_SIMD
3429 FloatRegister dest = ToFloatRegister(ins->output());
3430
3431 switch (ins->simdOp()) {
3432 case wasm::SimdOp::I8x16Splat:
3433 masm.splatX16(ToRegister(ins->src()), dest);
3434 break;
3435 case wasm::SimdOp::I16x8Splat:
3436 masm.splatX8(ToRegister(ins->src()), dest);
3437 break;
3438 case wasm::SimdOp::I32x4Splat:
3439 masm.splatX4(ToRegister(ins->src()), dest);
3440 break;
3441 case wasm::SimdOp::F32x4Splat:
3442 masm.splatX4(ToFloatRegister(ins->src()), dest);
3443 break;
3444 case wasm::SimdOp::F64x2Splat:
3445 masm.splatX2(ToFloatRegister(ins->src()), dest);
3446 break;
3447 default:
3448 MOZ_CRASH("ScalarToSimd128 SimdOp not implemented");
3449 }
3450 #else
3451 MOZ_CRASH("No SIMD");
3452 #endif
3453 }
3454
visitWasmInt64ToSimd128(LWasmInt64ToSimd128 * ins)3455 void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
3456 #ifdef ENABLE_WASM_SIMD
3457 Register64 src = ToRegister64(ins->src());
3458 FloatRegister dest = ToFloatRegister(ins->output());
3459
3460 switch (ins->simdOp()) {
3461 case wasm::SimdOp::I64x2Splat:
3462 masm.splatX2(src, dest);
3463 break;
3464 case wasm::SimdOp::V128Load8x8S:
3465 masm.moveGPR64ToDouble(src, dest);
3466 masm.widenLowInt8x16(dest, dest);
3467 break;
3468 case wasm::SimdOp::V128Load8x8U:
3469 masm.moveGPR64ToDouble(src, dest);
3470 masm.unsignedWidenLowInt8x16(dest, dest);
3471 break;
3472 case wasm::SimdOp::V128Load16x4S:
3473 masm.moveGPR64ToDouble(src, dest);
3474 masm.widenLowInt16x8(dest, dest);
3475 break;
3476 case wasm::SimdOp::V128Load16x4U:
3477 masm.moveGPR64ToDouble(src, dest);
3478 masm.unsignedWidenLowInt16x8(dest, dest);
3479 break;
3480 case wasm::SimdOp::V128Load32x2S:
3481 masm.moveGPR64ToDouble(src, dest);
3482 masm.widenLowInt32x4(dest, dest);
3483 break;
3484 case wasm::SimdOp::V128Load32x2U:
3485 masm.moveGPR64ToDouble(src, dest);
3486 masm.unsignedWidenLowInt32x4(dest, dest);
3487 break;
3488 default:
3489 MOZ_CRASH("Int64ToSimd128 SimdOp not implemented");
3490 }
3491 #else
3492 MOZ_CRASH("No SIMD");
3493 #endif
3494 }
3495
visitWasmUnarySimd128(LWasmUnarySimd128 * ins)3496 void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
3497 #ifdef ENABLE_WASM_SIMD
3498 FloatRegister src = ToFloatRegister(ins->src());
3499 FloatRegister dest = ToFloatRegister(ins->output());
3500
3501 switch (ins->simdOp()) {
3502 case wasm::SimdOp::I8x16Neg:
3503 masm.negInt8x16(src, dest);
3504 break;
3505 case wasm::SimdOp::I16x8Neg:
3506 masm.negInt16x8(src, dest);
3507 break;
3508 case wasm::SimdOp::I16x8ExtendLowI8x16S:
3509 masm.widenLowInt8x16(src, dest);
3510 break;
3511 case wasm::SimdOp::I16x8ExtendHighI8x16S:
3512 masm.widenHighInt8x16(src, dest);
3513 break;
3514 case wasm::SimdOp::I16x8ExtendLowI8x16U:
3515 masm.unsignedWidenLowInt8x16(src, dest);
3516 break;
3517 case wasm::SimdOp::I16x8ExtendHighI8x16U:
3518 masm.unsignedWidenHighInt8x16(src, dest);
3519 break;
3520 case wasm::SimdOp::I32x4Neg:
3521 masm.negInt32x4(src, dest);
3522 break;
3523 case wasm::SimdOp::I32x4ExtendLowI16x8S:
3524 masm.widenLowInt16x8(src, dest);
3525 break;
3526 case wasm::SimdOp::I32x4ExtendHighI16x8S:
3527 masm.widenHighInt16x8(src, dest);
3528 break;
3529 case wasm::SimdOp::I32x4ExtendLowI16x8U:
3530 masm.unsignedWidenLowInt16x8(src, dest);
3531 break;
3532 case wasm::SimdOp::I32x4ExtendHighI16x8U:
3533 masm.unsignedWidenHighInt16x8(src, dest);
3534 break;
3535 case wasm::SimdOp::I32x4TruncSatF32x4S:
3536 masm.truncSatFloat32x4ToInt32x4(src, dest);
3537 break;
3538 case wasm::SimdOp::I32x4TruncSatF32x4U:
3539 masm.unsignedTruncSatFloat32x4ToInt32x4(src, dest,
3540 ToFloatRegister(ins->temp()));
3541 break;
3542 case wasm::SimdOp::I64x2Neg:
3543 masm.negInt64x2(src, dest);
3544 break;
3545 case wasm::SimdOp::I64x2ExtendLowI32x4S:
3546 masm.widenLowInt32x4(src, dest);
3547 break;
3548 case wasm::SimdOp::I64x2ExtendHighI32x4S:
3549 masm.widenHighInt32x4(src, dest);
3550 break;
3551 case wasm::SimdOp::I64x2ExtendLowI32x4U:
3552 masm.unsignedWidenLowInt32x4(src, dest);
3553 break;
3554 case wasm::SimdOp::I64x2ExtendHighI32x4U:
3555 masm.unsignedWidenHighInt32x4(src, dest);
3556 break;
3557 case wasm::SimdOp::F32x4Abs:
3558 masm.absFloat32x4(src, dest);
3559 break;
3560 case wasm::SimdOp::F32x4Neg:
3561 masm.negFloat32x4(src, dest);
3562 break;
3563 case wasm::SimdOp::F32x4Sqrt:
3564 masm.sqrtFloat32x4(src, dest);
3565 break;
3566 case wasm::SimdOp::F32x4ConvertI32x4S:
3567 masm.convertInt32x4ToFloat32x4(src, dest);
3568 break;
3569 case wasm::SimdOp::F32x4ConvertI32x4U:
3570 masm.unsignedConvertInt32x4ToFloat32x4(src, dest);
3571 break;
3572 case wasm::SimdOp::F64x2Abs:
3573 masm.absFloat64x2(src, dest);
3574 break;
3575 case wasm::SimdOp::F64x2Neg:
3576 masm.negFloat64x2(src, dest);
3577 break;
3578 case wasm::SimdOp::F64x2Sqrt:
3579 masm.sqrtFloat64x2(src, dest);
3580 break;
3581 case wasm::SimdOp::V128Not:
3582 masm.bitwiseNotSimd128(src, dest);
3583 break;
3584 case wasm::SimdOp::I8x16Popcnt:
3585 masm.popcntInt8x16(src, dest, ToFloatRegister(ins->temp()));
3586 break;
3587 case wasm::SimdOp::I8x16Abs:
3588 masm.absInt8x16(src, dest);
3589 break;
3590 case wasm::SimdOp::I16x8Abs:
3591 masm.absInt16x8(src, dest);
3592 break;
3593 case wasm::SimdOp::I32x4Abs:
3594 masm.absInt32x4(src, dest);
3595 break;
3596 case wasm::SimdOp::I64x2Abs:
3597 masm.absInt64x2(src, dest);
3598 break;
3599 case wasm::SimdOp::F32x4Ceil:
3600 masm.ceilFloat32x4(src, dest);
3601 break;
3602 case wasm::SimdOp::F32x4Floor:
3603 masm.floorFloat32x4(src, dest);
3604 break;
3605 case wasm::SimdOp::F32x4Trunc:
3606 masm.truncFloat32x4(src, dest);
3607 break;
3608 case wasm::SimdOp::F32x4Nearest:
3609 masm.nearestFloat32x4(src, dest);
3610 break;
3611 case wasm::SimdOp::F64x2Ceil:
3612 masm.ceilFloat64x2(src, dest);
3613 break;
3614 case wasm::SimdOp::F64x2Floor:
3615 masm.floorFloat64x2(src, dest);
3616 break;
3617 case wasm::SimdOp::F64x2Trunc:
3618 masm.truncFloat64x2(src, dest);
3619 break;
3620 case wasm::SimdOp::F64x2Nearest:
3621 masm.nearestFloat64x2(src, dest);
3622 break;
3623 case wasm::SimdOp::F32x4DemoteF64x2Zero:
3624 masm.convertFloat64x2ToFloat32x4(src, dest);
3625 break;
3626 case wasm::SimdOp::F64x2PromoteLowF32x4:
3627 masm.convertFloat32x4ToFloat64x2(src, dest);
3628 break;
3629 case wasm::SimdOp::F64x2ConvertLowI32x4S:
3630 masm.convertInt32x4ToFloat64x2(src, dest);
3631 break;
3632 case wasm::SimdOp::F64x2ConvertLowI32x4U:
3633 masm.unsignedConvertInt32x4ToFloat64x2(src, dest);
3634 break;
3635 case wasm::SimdOp::I32x4TruncSatF64x2SZero:
3636 masm.truncSatFloat64x2ToInt32x4(src, dest, ToFloatRegister(ins->temp()));
3637 break;
3638 case wasm::SimdOp::I32x4TruncSatF64x2UZero:
3639 masm.unsignedTruncSatFloat64x2ToInt32x4(src, dest,
3640 ToFloatRegister(ins->temp()));
3641 break;
3642 case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
3643 masm.extAddPairwiseInt8x16(src, dest);
3644 break;
3645 case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
3646 masm.unsignedExtAddPairwiseInt8x16(src, dest);
3647 break;
3648 case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
3649 masm.extAddPairwiseInt16x8(src, dest);
3650 break;
3651 case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
3652 masm.unsignedExtAddPairwiseInt16x8(src, dest);
3653 break;
3654 case wasm::SimdOp::I32x4RelaxedTruncSSatF32x4:
3655 masm.truncSatFloat32x4ToInt32x4Relaxed(src, dest);
3656 break;
3657 case wasm::SimdOp::I32x4RelaxedTruncUSatF32x4:
3658 masm.unsignedTruncSatFloat32x4ToInt32x4Relaxed(src, dest);
3659 break;
3660 case wasm::SimdOp::I32x4RelaxedTruncSatF64x2SZero:
3661 masm.truncSatFloat64x2ToInt32x4Relaxed(src, dest);
3662 break;
3663 case wasm::SimdOp::I32x4RelaxedTruncSatF64x2UZero:
3664 masm.unsignedTruncSatFloat64x2ToInt32x4Relaxed(src, dest);
3665 break;
3666 default:
3667 MOZ_CRASH("Unary SimdOp not implemented");
3668 }
3669 #else
3670 MOZ_CRASH("No SIMD");
3671 #endif
3672 }
3673
visitWasmReduceSimd128(LWasmReduceSimd128 * ins)3674 void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
3675 #ifdef ENABLE_WASM_SIMD
3676 FloatRegister src = ToFloatRegister(ins->src());
3677 const LDefinition* dest = ins->output();
3678 uint32_t imm = ins->imm();
3679
3680 switch (ins->simdOp()) {
3681 case wasm::SimdOp::V128AnyTrue:
3682 masm.anyTrueSimd128(src, ToRegister(dest));
3683 break;
3684 case wasm::SimdOp::I8x16AllTrue:
3685 masm.allTrueInt8x16(src, ToRegister(dest));
3686 break;
3687 case wasm::SimdOp::I16x8AllTrue:
3688 masm.allTrueInt16x8(src, ToRegister(dest));
3689 break;
3690 case wasm::SimdOp::I32x4AllTrue:
3691 masm.allTrueInt32x4(src, ToRegister(dest));
3692 break;
3693 case wasm::SimdOp::I64x2AllTrue:
3694 masm.allTrueInt64x2(src, ToRegister(dest));
3695 break;
3696 case wasm::SimdOp::I8x16Bitmask:
3697 masm.bitmaskInt8x16(src, ToRegister(dest));
3698 break;
3699 case wasm::SimdOp::I16x8Bitmask:
3700 masm.bitmaskInt16x8(src, ToRegister(dest));
3701 break;
3702 case wasm::SimdOp::I32x4Bitmask:
3703 masm.bitmaskInt32x4(src, ToRegister(dest));
3704 break;
3705 case wasm::SimdOp::I64x2Bitmask:
3706 masm.bitmaskInt64x2(src, ToRegister(dest));
3707 break;
3708 case wasm::SimdOp::I8x16ExtractLaneS:
3709 masm.extractLaneInt8x16(imm, src, ToRegister(dest));
3710 break;
3711 case wasm::SimdOp::I8x16ExtractLaneU:
3712 masm.unsignedExtractLaneInt8x16(imm, src, ToRegister(dest));
3713 break;
3714 case wasm::SimdOp::I16x8ExtractLaneS:
3715 masm.extractLaneInt16x8(imm, src, ToRegister(dest));
3716 break;
3717 case wasm::SimdOp::I16x8ExtractLaneU:
3718 masm.unsignedExtractLaneInt16x8(imm, src, ToRegister(dest));
3719 break;
3720 case wasm::SimdOp::I32x4ExtractLane:
3721 masm.extractLaneInt32x4(imm, src, ToRegister(dest));
3722 break;
3723 case wasm::SimdOp::F32x4ExtractLane:
3724 masm.extractLaneFloat32x4(imm, src, ToFloatRegister(dest));
3725 break;
3726 case wasm::SimdOp::F64x2ExtractLane:
3727 masm.extractLaneFloat64x2(imm, src, ToFloatRegister(dest));
3728 break;
3729 default:
3730 MOZ_CRASH("Reduce SimdOp not implemented");
3731 }
3732 #else
3733 MOZ_CRASH("No SIMD");
3734 #endif
3735 }
3736
visitWasmReduceAndBranchSimd128(LWasmReduceAndBranchSimd128 * ins)3737 void CodeGenerator::visitWasmReduceAndBranchSimd128(
3738 LWasmReduceAndBranchSimd128* ins) {
3739 #ifdef ENABLE_WASM_SIMD
3740 FloatRegister src = ToFloatRegister(ins->src());
3741
3742 switch (ins->simdOp()) {
3743 case wasm::SimdOp::V128AnyTrue:
3744 // Set the zero flag if all of the lanes are zero, and branch on that.
3745 masm.vptest(src, src);
3746 emitBranch(Assembler::NotEqual, ins->ifTrue(), ins->ifFalse());
3747 break;
3748 case wasm::SimdOp::I8x16AllTrue:
3749 case wasm::SimdOp::I16x8AllTrue:
3750 case wasm::SimdOp::I32x4AllTrue:
3751 case wasm::SimdOp::I64x2AllTrue: {
3752 // Compare all lanes to zero, set the zero flag if none of the lanes are
3753 // zero, and branch on that.
3754 ScratchSimd128Scope tmp(masm);
3755 masm.vpxor(tmp, tmp, tmp);
3756 switch (ins->simdOp()) {
3757 case wasm::SimdOp::I8x16AllTrue:
3758 masm.vpcmpeqb(Operand(src), tmp, tmp);
3759 break;
3760 case wasm::SimdOp::I16x8AllTrue:
3761 masm.vpcmpeqw(Operand(src), tmp, tmp);
3762 break;
3763 case wasm::SimdOp::I32x4AllTrue:
3764 masm.vpcmpeqd(Operand(src), tmp, tmp);
3765 break;
3766 case wasm::SimdOp::I64x2AllTrue:
3767 masm.vpcmpeqq(Operand(src), tmp, tmp);
3768 break;
3769 default:
3770 MOZ_CRASH();
3771 }
3772 masm.vptest(tmp, tmp);
3773 emitBranch(Assembler::Equal, ins->ifTrue(), ins->ifFalse());
3774 break;
3775 }
3776 case wasm::SimdOp::I16x8Bitmask: {
3777 masm.bitwiseTestSimd128(SimdConstant::SplatX8(0x8000), src);
3778 emitBranch(Assembler::NotEqual, ins->ifTrue(), ins->ifFalse());
3779 break;
3780 }
3781 default:
3782 MOZ_CRASH("Reduce-and-branch SimdOp not implemented");
3783 }
3784 #else
3785 MOZ_CRASH("No SIMD");
3786 #endif
3787 }
3788
visitWasmReduceSimd128ToInt64(LWasmReduceSimd128ToInt64 * ins)3789 void CodeGenerator::visitWasmReduceSimd128ToInt64(
3790 LWasmReduceSimd128ToInt64* ins) {
3791 #ifdef ENABLE_WASM_SIMD
3792 FloatRegister src = ToFloatRegister(ins->src());
3793 Register64 dest = ToOutRegister64(ins);
3794 uint32_t imm = ins->imm();
3795
3796 switch (ins->simdOp()) {
3797 case wasm::SimdOp::I64x2ExtractLane:
3798 masm.extractLaneInt64x2(imm, src, dest);
3799 break;
3800 default:
3801 MOZ_CRASH("Reduce SimdOp not implemented");
3802 }
3803 #else
3804 MOZ_CRASH("No SIMD");
3805 #endif
3806 }
3807
visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128 * ins)3808 void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
3809 #ifdef ENABLE_WASM_SIMD
3810 const MWasmLoadLaneSimd128* mir = ins->mir();
3811 const wasm::MemoryAccessDesc& access = mir->access();
3812
3813 uint32_t offset = access.offset();
3814 MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
3815
3816 const LAllocation* value = ins->src();
3817 Operand srcAddr = toMemoryAccessOperand(ins, offset);
3818
3819 masm.append(access, masm.size());
3820 switch (ins->laneSize()) {
3821 case 1: {
3822 masm.vpinsrb(ins->laneIndex(), srcAddr, ToFloatRegister(value),
3823 ToFloatRegister(value));
3824 break;
3825 }
3826 case 2: {
3827 masm.vpinsrw(ins->laneIndex(), srcAddr, ToFloatRegister(value),
3828 ToFloatRegister(value));
3829 break;
3830 }
3831 case 4: {
3832 masm.vinsertps(ins->laneIndex() << 4, srcAddr, ToFloatRegister(value),
3833 ToFloatRegister(value));
3834 break;
3835 }
3836 case 8: {
3837 if (ins->laneIndex() == 0) {
3838 masm.vmovlps(srcAddr, ToFloatRegister(value), ToFloatRegister(value));
3839 } else {
3840 masm.vmovhps(srcAddr, ToFloatRegister(value), ToFloatRegister(value));
3841 }
3842 break;
3843 }
3844 default:
3845 MOZ_CRASH("Unsupported load lane size");
3846 }
3847 #else
3848 MOZ_CRASH("No SIMD");
3849 #endif
3850 }
3851
visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128 * ins)3852 void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
3853 #ifdef ENABLE_WASM_SIMD
3854 const MWasmStoreLaneSimd128* mir = ins->mir();
3855 const wasm::MemoryAccessDesc& access = mir->access();
3856
3857 uint32_t offset = access.offset();
3858 MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
3859
3860 const LAllocation* src = ins->src();
3861 Operand destAddr = toMemoryAccessOperand(ins, offset);
3862
3863 masm.append(access, masm.size());
3864 switch (ins->laneSize()) {
3865 case 1: {
3866 masm.vpextrb(ins->laneIndex(), ToFloatRegister(src), destAddr);
3867 break;
3868 }
3869 case 2: {
3870 masm.vpextrw(ins->laneIndex(), ToFloatRegister(src), destAddr);
3871 break;
3872 }
3873 case 4: {
3874 unsigned lane = ins->laneIndex();
3875 if (lane == 0) {
3876 masm.vmovss(ToFloatRegister(src), destAddr);
3877 } else {
3878 masm.vextractps(lane, ToFloatRegister(src), destAddr);
3879 }
3880 break;
3881 }
3882 case 8: {
3883 if (ins->laneIndex() == 0) {
3884 masm.vmovlps(ToFloatRegister(src), destAddr);
3885 } else {
3886 masm.vmovhps(ToFloatRegister(src), destAddr);
3887 }
3888 break;
3889 }
3890 default:
3891 MOZ_CRASH("Unsupported store lane size");
3892 }
3893 #else
3894 MOZ_CRASH("No SIMD");
3895 #endif
3896 }
3897
3898 } // namespace jit
3899 } // namespace js
3900