1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2  * vim: set ts=8 sts=4 et sw=4 tw=99:
3  *
4  * Copyright 2016 Mozilla Foundation
5  *
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 
19 /* WebAssembly baseline compiler ("RabaldrMonkey")
20  *
21  * General status notes:
22  *
23  * "FIXME" indicates a known or suspected bug.  Always has a bug#.
24  *
25  * "TODO" indicates an opportunity for a general improvement, with an additional
26  * tag to indicate the area of improvement.  Usually has a bug#.
27  *
28  * Unimplemented functionality:
29  *
30  *  - Tiered compilation (bug 1277562)
31  *  - profiler support / devtools (bug 1286948)
32  *  - SIMD
33  *  - Atomics
34  *
35  * There are lots of machine dependencies here but they are pretty well isolated
36  * to a segment of the compiler.  Many dependencies will eventually be factored
37  * into the MacroAssembler layer and shared with other code generators.
38  *
39  *
40  * High-value compiler performance improvements:
41  *
42  * - (Bug 1316802) The specific-register allocator (the needI32(r), needI64(r)
43  *   etc methods) can avoid syncing the value stack if the specific register is
44  *   in use but there is a free register to shuffle the specific register into.
45  *   (This will also improve the generated code.)  The sync happens often enough
46  *   here to show up in profiles, because it is triggered by integer multiply
47  *   and divide.
48  *
49  *
50  * High-value code generation improvements:
51  *
52  * - (Bug 1316803) Opportunities for cheaply folding in a constant rhs to
53  *   arithmetic operations, we do this already for I32 add and shift operators,
54  *   this reduces register pressure and instruction count.
55  *
56  * - (Bug 1286816) Opportunities for cheaply folding in a constant rhs to
57  *   conditionals.
58  *
59  * - (Bug 1286816) Boolean evaluation for control can be optimized by pushing a
60  *   bool-generating operation onto the value stack in the same way that we now
61  *   push latent constants and local lookups, or (easier) by remembering the
62  *   operation in a side location if the next Op will consume it.
63  *
64  * - (Bug 1286816) brIf pessimizes by branching over code that performs stack
65  *   cleanup and a branch.  If no cleanup is needed we can just branch
66  *   conditionally to the target.
67  *
68  * - (Bug 1316804) brTable pessimizes by always dispatching to code that pops
69  *   the stack and then jumps to the code for the target case.  If no cleanup is
70  *   needed we could just branch conditionally to the target; if the same amount
71  *   of cleanup is needed for all cases then the cleanup can be done before the
72  *   dispatch.  Both are highly likely.
73  *
74  * - (Bug 1316806) Register management around calls: At the moment we sync the
75  *   value stack unconditionally (this is simple) but there are probably many
76  *   common cases where we could instead save/restore live caller-saves
77  *   registers and perform parallel assignment into argument registers.  This
78  *   may be important if we keep some locals in registers.
79  *
80  * - (Bug 1316808) Allocate some locals to registers on machines where there are
81  *   enough registers.  This is probably hard to do well in a one-pass compiler
82  *   but it might be that just keeping register arguments and the first few
83  *   locals in registers is a viable strategy; another (more general) strategy
84  *   is caching locals in registers in straight-line code.  Such caching could
85  *   also track constant values in registers, if that is deemed valuable.  A
86  *   combination of techniques may be desirable: parameters and the first few
87  *   locals could be cached on entry to the function but not statically assigned
88  *   to registers throughout.
89  *
90  *   (On a large corpus of code it should be possible to compute, for every
91  *   signature comprising the types of parameters and locals, and using a static
92  *   weight for loops, a list in priority order of which parameters and locals
93  *   that should be assigned to registers.  Or something like that.  Wasm makes
94  *   this simple.  Static assignments are desirable because they are not flushed
95  *   to memory by the pre-block sync() call.)
96  */
97 
98 #include "wasm/WasmBaselineCompile.h"
99 
100 #include "mozilla/MathAlgorithms.h"
101 
102 #include "jit/AtomicOp.h"
103 #include "jit/IonTypes.h"
104 #include "jit/JitAllocPolicy.h"
105 #include "jit/Label.h"
106 #include "jit/MacroAssembler.h"
107 #include "jit/MIR.h"
108 #include "jit/Registers.h"
109 #include "jit/RegisterSets.h"
110 #if defined(JS_CODEGEN_ARM)
111 # include "jit/arm/Assembler-arm.h"
112 #endif
113 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
114 # include "jit/x86-shared/Architecture-x86-shared.h"
115 # include "jit/x86-shared/Assembler-x86-shared.h"
116 #endif
117 
118 #include "wasm/WasmBinaryFormat.h"
119 #include "wasm/WasmBinaryIterator.h"
120 #include "wasm/WasmGenerator.h"
121 #include "wasm/WasmSignalHandlers.h"
122 
123 #include "jit/MacroAssembler-inl.h"
124 
125 using mozilla::DebugOnly;
126 using mozilla::FloatingPoint;
127 using mozilla::IsPowerOfTwo;
128 using mozilla::SpecificNaN;
129 
130 namespace js {
131 namespace wasm {
132 
133 using namespace js::jit;
134 using JS::GenericNaN;
135 
136 struct BaseCompilePolicy : OpIterPolicy
137 {
138     static const bool Output = true;
139 
140     // The baseline compiler tracks values on a stack of its own -- it
141     // needs to scan that stack for spilling -- and thus has no need
142     // for the values maintained by the iterator.
143     //
144     // The baseline compiler tracks control items on a stack of its
145     // own as well.
146     //
147     // TODO / REDUNDANT (Bug 1316814): It would be nice if we could
148     // make use of the iterator's ControlItems and not require our own
149     // stack for that.
150 };
151 
152 typedef OpIter<BaseCompilePolicy> BaseOpIter;
153 
154 typedef bool IsUnsigned;
155 typedef bool IsSigned;
156 typedef bool ZeroOnOverflow;
157 typedef bool IsKnownNotZero;
158 typedef bool HandleNaNSpecially;
159 typedef unsigned ByteSize;
160 typedef unsigned BitSize;
161 
162 // UseABI::Wasm implies that the Tls/Heap/Global registers are nonvolatile,
163 // except when InterModule::True is also set, when they are volatile.
164 //
165 // UseABI::System implies that the Tls/Heap/Global registers are volatile.
166 // Additionally, the parameter passing mechanism may be slightly different from
167 // the UseABI::Wasm convention.
168 //
169 // When the Tls/Heap/Global registers are not volatile, the baseline compiler
170 // will restore the Tls register from its save slot before the call, since the
171 // baseline compiler uses the Tls register for other things.
172 //
173 // When those registers are volatile, the baseline compiler will reload them
174 // after the call (it will restore the Tls register from the save slot and load
175 // the other two from the Tls data).
176 
177 enum class UseABI { Wasm, System };
178 enum class InterModule { False = false, True = true };
179 
180 #ifdef JS_CODEGEN_ARM64
181 // FIXME: This is not correct, indeed for ARM64 there is no reliable
182 // StackPointer and we'll need to change the abstractions that use
183 // SP-relative addressing.  There's a guard in emitFunction() below to
184 // prevent this workaround from having any consequence.  This hack
185 // exists only as a stopgap; there is no ARM64 JIT support yet.
186 static const Register StackPointer = RealStackPointer;
187 #endif
188 
189 #ifdef JS_CODEGEN_X86
190 // The selection of EBX here steps gingerly around: the need for EDX
191 // to be allocatable for multiply/divide; ECX to be allocatable for
192 // shift/rotate; EAX (= ReturnReg) to be allocatable as the joinreg;
193 // EBX not being one of the WasmTableCall registers; and needing a
194 // temp register for load/store that has a single-byte persona.
195 static const Register ScratchRegX86 = ebx;
196 
197 # define INT_DIV_I64_CALLOUT
198 #endif
199 
200 #ifdef JS_CODEGEN_ARM
201 // We need a temp for funcPtrCall.  It can't be any of the
202 // WasmTableCall registers, an argument register, or a scratch
203 // register, and probably should not be ReturnReg.
204 static const Register FuncPtrCallTemp = CallTempReg1;
205 
206 // We use our own scratch register, because the macro assembler uses
207 // the regular scratch register(s) pretty liberally.  We could
208 // work around that in several cases but the mess does not seem
209 // worth it yet.  CallTempReg2 seems safe.
210 static const Register ScratchRegARM = CallTempReg2;
211 
212 # define INT_DIV_I64_CALLOUT
213 # define I64_TO_FLOAT_CALLOUT
214 # define FLOAT_TO_I64_CALLOUT
215 #endif
216 
217 class BaseCompiler
218 {
219     // We define our own ScratchRegister abstractions, deferring to
220     // the platform's when possible.
221 
222 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
223     typedef ScratchDoubleScope ScratchF64;
224 #else
225     class ScratchF64
226     {
227       public:
228         ScratchF64(BaseCompiler& b) {}
229         operator FloatRegister() const {
230             MOZ_CRASH("BaseCompiler platform hook - ScratchF64");
231         }
232     };
233 #endif
234 
235 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
236     typedef ScratchFloat32Scope ScratchF32;
237 #else
238     class ScratchF32
239     {
240       public:
ScratchF32(BaseCompiler & b)241         ScratchF32(BaseCompiler& b) {}
operator FloatRegister() const242         operator FloatRegister() const {
243             MOZ_CRASH("BaseCompiler platform hook - ScratchF32");
244         }
245     };
246 #endif
247 
248 #if defined(JS_CODEGEN_X64)
249     typedef ScratchRegisterScope ScratchI32;
250 #elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
251     class ScratchI32
252     {
253 # ifdef DEBUG
254         BaseCompiler& bc;
255       public:
ScratchI32(BaseCompiler & bc)256         explicit ScratchI32(BaseCompiler& bc) : bc(bc) {
257             MOZ_ASSERT(!bc.scratchRegisterTaken());
258             bc.setScratchRegisterTaken(true);
259         }
~ScratchI32()260         ~ScratchI32() {
261             MOZ_ASSERT(bc.scratchRegisterTaken());
262             bc.setScratchRegisterTaken(false);
263         }
264 # else
265       public:
266         explicit ScratchI32(BaseCompiler& bc) {}
267 # endif
operator Register() const268         operator Register() const {
269 # ifdef JS_CODEGEN_X86
270             return ScratchRegX86;
271 # else
272             return ScratchRegARM;
273 # endif
274         }
275     };
276 #else
277     class ScratchI32
278     {
279       public:
ScratchI32(BaseCompiler & bc)280         ScratchI32(BaseCompiler& bc) {}
operator Register() const281         operator Register() const {
282             MOZ_CRASH("BaseCompiler platform hook - ScratchI32");
283         }
284     };
285 #endif
286 
287     // A Label in the code, allocated out of a temp pool in the
288     // TempAllocator attached to the compilation.
289 
290     struct PooledLabel : public Label, public TempObject, public InlineListNode<PooledLabel>
291     {
PooledLabeljs::wasm::BaseCompiler::PooledLabel292         PooledLabel() : f(nullptr) {}
PooledLabeljs::wasm::BaseCompiler::PooledLabel293         explicit PooledLabel(BaseCompiler* f) : f(f) {}
294         BaseCompiler* f;
295     };
296 
297     typedef Vector<PooledLabel*, 8, SystemAllocPolicy> LabelVector;
298 
299     struct UniquePooledLabelFreePolicy
300     {
operator ()js::wasm::BaseCompiler::UniquePooledLabelFreePolicy301         void operator()(PooledLabel* p) {
302             p->f->freeLabel(p);
303         }
304     };
305 
306     typedef UniquePtr<PooledLabel, UniquePooledLabelFreePolicy> UniquePooledLabel;
307 
308     // The strongly typed register wrappers have saved my bacon a few
309     // times; though they are largely redundant they stay, for now.
310 
311     // TODO / INVESTIGATE (Bug 1316815): Things would probably be
312     // simpler if these inherited from Register, Register64, and
313     // FloatRegister.
314 
315     struct RegI32
316     {
RegI32js::wasm::BaseCompiler::RegI32317         RegI32() : reg(Register::Invalid()) {}
RegI32js::wasm::BaseCompiler::RegI32318         explicit RegI32(Register reg) : reg(reg) {}
319         Register reg;
operator ==js::wasm::BaseCompiler::RegI32320         bool operator==(const RegI32& that) { return reg == that.reg; }
operator !=js::wasm::BaseCompiler::RegI32321         bool operator!=(const RegI32& that) { return reg != that.reg; }
322     };
323 
324     struct RegI64
325     {
RegI64js::wasm::BaseCompiler::RegI64326         RegI64() : reg(Register64::Invalid()) {}
RegI64js::wasm::BaseCompiler::RegI64327         explicit RegI64(Register64 reg) : reg(reg) {}
328         Register64 reg;
operator ==js::wasm::BaseCompiler::RegI64329         bool operator==(const RegI64& that) { return reg == that.reg; }
operator !=js::wasm::BaseCompiler::RegI64330         bool operator!=(const RegI64& that) { return reg != that.reg; }
331     };
332 
333     struct RegF32
334     {
RegF32js::wasm::BaseCompiler::RegF32335         RegF32() {}
RegF32js::wasm::BaseCompiler::RegF32336         explicit RegF32(FloatRegister reg) : reg(reg) {}
337         FloatRegister reg;
operator ==js::wasm::BaseCompiler::RegF32338         bool operator==(const RegF32& that) { return reg == that.reg; }
operator !=js::wasm::BaseCompiler::RegF32339         bool operator!=(const RegF32& that) { return reg != that.reg; }
340     };
341 
342     struct RegF64
343     {
RegF64js::wasm::BaseCompiler::RegF64344         RegF64() {}
RegF64js::wasm::BaseCompiler::RegF64345         explicit RegF64(FloatRegister reg) : reg(reg) {}
346         FloatRegister reg;
operator ==js::wasm::BaseCompiler::RegF64347         bool operator==(const RegF64& that) { return reg == that.reg; }
operator !=js::wasm::BaseCompiler::RegF64348         bool operator!=(const RegF64& that) { return reg != that.reg; }
349     };
350 
351     struct AnyReg
352     {
AnyRegjs::wasm::BaseCompiler::AnyReg353         AnyReg() { tag = NONE; }
AnyRegjs::wasm::BaseCompiler::AnyReg354         explicit AnyReg(RegI32 r) { tag = I32; i32_ = r; }
AnyRegjs::wasm::BaseCompiler::AnyReg355         explicit AnyReg(RegI64 r) { tag = I64; i64_ = r; }
AnyRegjs::wasm::BaseCompiler::AnyReg356         explicit AnyReg(RegF32 r) { tag = F32; f32_ = r; }
AnyRegjs::wasm::BaseCompiler::AnyReg357         explicit AnyReg(RegF64 r) { tag = F64; f64_ = r; }
358 
i32js::wasm::BaseCompiler::AnyReg359         RegI32 i32() {
360             MOZ_ASSERT(tag == I32);
361             return i32_;
362         }
i64js::wasm::BaseCompiler::AnyReg363         RegI64 i64() {
364             MOZ_ASSERT(tag == I64);
365             return i64_;
366         }
f32js::wasm::BaseCompiler::AnyReg367         RegF32 f32() {
368             MOZ_ASSERT(tag == F32);
369             return f32_;
370         }
f64js::wasm::BaseCompiler::AnyReg371         RegF64 f64() {
372             MOZ_ASSERT(tag == F64);
373             return f64_;
374         }
anyjs::wasm::BaseCompiler::AnyReg375         AnyRegister any() {
376             switch (tag) {
377               case F32: return AnyRegister(f32_.reg);
378               case F64: return AnyRegister(f64_.reg);
379               case I32: return AnyRegister(i32_.reg);
380               case I64:
381 #ifdef JS_PUNBOX64
382                 return AnyRegister(i64_.reg.reg);
383 #else
384                 // The compiler is written so that this is never needed: any() is called
385                 // on arbitrary registers for asm.js but asm.js does not have 64-bit ints.
386                 // For wasm, any() is called on arbitrary registers only on 64-bit platforms.
387                 MOZ_CRASH("AnyReg::any() on 32-bit platform");
388 #endif
389               case NONE:
390                 MOZ_CRASH("AnyReg::any() on NONE");
391             }
392             // Work around GCC 5 analysis/warning bug.
393             MOZ_CRASH("AnyReg::any(): impossible case");
394         }
395 
396         union {
397             RegI32 i32_;
398             RegI64 i64_;
399             RegF32 f32_;
400             RegF64 f64_;
401         };
402         enum { NONE, I32, I64, F32, F64 } tag;
403     };
404 
405     struct Local
406     {
Localjs::wasm::BaseCompiler::Local407         Local() : type_(MIRType::None), offs_(UINT32_MAX) {}
Localjs::wasm::BaseCompiler::Local408         Local(MIRType type, uint32_t offs) : type_(type), offs_(offs) {}
409 
initjs::wasm::BaseCompiler::Local410         void init(MIRType type_, uint32_t offs_) {
411             this->type_ = type_;
412             this->offs_ = offs_;
413         }
414 
415         MIRType  type_;              // Type of the value, or MIRType::None
416         uint32_t offs_;              // Zero-based frame offset of value, or UINT32_MAX
417 
typejs::wasm::BaseCompiler::Local418         MIRType type() const { MOZ_ASSERT(type_ != MIRType::None); return type_; }
offsjs::wasm::BaseCompiler::Local419         uint32_t offs() const { MOZ_ASSERT(offs_ != UINT32_MAX); return offs_; }
420     };
421 
422     // Control node, representing labels and stack heights at join points.
423 
424     struct Control
425     {
Controljs::wasm::BaseCompiler::Control426         Control(uint32_t framePushed, uint32_t stackSize)
427             : label(nullptr),
428               otherLabel(nullptr),
429               framePushed(framePushed),
430               stackSize(stackSize),
431               deadOnArrival(false),
432               deadThenBranch(false)
433         {}
434 
435         PooledLabel* label;
436         PooledLabel* otherLabel;        // Used for the "else" branch of if-then-else
437         uint32_t framePushed;           // From masm
438         uint32_t stackSize;             // Value stack height
439         bool deadOnArrival;             // deadCode_ was set on entry to the region
440         bool deadThenBranch;            // deadCode_ was set on exit from "then"
441     };
442 
443     // Volatile registers except ReturnReg.
444 
445     static LiveRegisterSet VolatileReturnGPR;
446 
447     // The baseline compiler will use OOL code more sparingly than
448     // Baldr since our code is not high performance and frills like
449     // code density and branch prediction friendliness will be less
450     // important.
451 
452     class OutOfLineCode : public TempObject
453     {
454       private:
455         Label entry_;
456         Label rejoin_;
457         uint32_t framePushed_;
458 
459       public:
OutOfLineCode()460         OutOfLineCode() : framePushed_(UINT32_MAX) {}
461 
entry()462         Label* entry() { return &entry_; }
rejoin()463         Label* rejoin() { return &rejoin_; }
464 
setFramePushed(uint32_t framePushed)465         void setFramePushed(uint32_t framePushed) {
466             MOZ_ASSERT(framePushed_ == UINT32_MAX);
467             framePushed_ = framePushed;
468         }
469 
bind(MacroAssembler & masm)470         void bind(MacroAssembler& masm) {
471             MOZ_ASSERT(framePushed_ != UINT32_MAX);
472             masm.bind(&entry_);
473             masm.setFramePushed(framePushed_);
474         }
475 
476         // Save volatile registers but not ReturnReg.
477 
saveVolatileReturnGPR(MacroAssembler & masm)478         void saveVolatileReturnGPR(MacroAssembler& masm) {
479             masm.PushRegsInMask(BaseCompiler::VolatileReturnGPR);
480         }
481 
482         // Restore volatile registers but not ReturnReg.
483 
restoreVolatileReturnGPR(MacroAssembler & masm)484         void restoreVolatileReturnGPR(MacroAssembler& masm) {
485             masm.PopRegsInMask(BaseCompiler::VolatileReturnGPR);
486         }
487 
488         // The generate() method must be careful about register use
489         // because it will be invoked when there is a register
490         // assignment in the BaseCompiler that does not correspond
491         // to the available registers when the generated OOL code is
492         // executed.  The register allocator *must not* be called.
493         //
494         // The best strategy is for the creator of the OOL object to
495         // allocate all temps that the OOL code will need.
496         //
497         // Input, output, and temp registers are embedded in the OOL
498         // object and are known to the code generator.
499         //
500         // Scratch registers are available to use in OOL code.
501         //
502         // All other registers must be explicitly saved and restored
503         // by the OOL code before being used.
504 
505         virtual void generate(MacroAssembler& masm) = 0;
506     };
507 
508     const ModuleGeneratorData&  mg_;
509     BaseOpIter                  iter_;
510     const FuncBytes&            func_;
511     size_t                      lastReadCallSite_;
512     TempAllocator&              alloc_;
513     const ValTypeVector&        locals_;         // Types of parameters and locals
514     int32_t                     localSize_;      // Size of local area in bytes (stable after beginFunction)
515     int32_t                     varLow_;         // Low byte offset of local area for true locals (not parameters)
516     int32_t                     varHigh_;        // High byte offset + 1 of local area for true locals
517     int32_t                     maxFramePushed_; // Max value of masm.framePushed() observed
518     bool                        deadCode_;       // Flag indicating we should decode & discard the opcode
519     ValTypeVector               SigI64I64_;
520     ValTypeVector               SigDD_;
521     ValTypeVector               SigD_;
522     ValTypeVector               SigF_;
523     ValTypeVector               SigI_;
524     ValTypeVector               Sig_;
525     Label                       returnLabel_;
526     Label                       outOfLinePrologue_;
527     Label                       bodyLabel_;
528     TrapOffset                  prologueTrapOffset_;
529 
530     FuncCompileResults&         compileResults_;
531     MacroAssembler&             masm;            // No '_' suffix - too tedious...
532 
533     AllocatableGeneralRegisterSet availGPR_;
534     AllocatableFloatRegisterSet availFPU_;
535 #ifdef DEBUG
536     bool                        scratchRegisterTaken_;
537 #endif
538 
539     TempObjectPool<PooledLabel> labelPool_;
540 
541     Vector<Local, 8, SystemAllocPolicy> localInfo_;
542     Vector<OutOfLineCode*, 8, SystemAllocPolicy> outOfLine_;
543 
544     // Index into localInfo_ of the special local used for saving the TLS
545     // pointer. This follows the function's real arguments and locals.
546     uint32_t                    tlsSlot_;
547 
548     // On specific platforms we sometimes need to use specific registers.
549 
550 #ifdef JS_CODEGEN_X64
551     RegI64 specific_rax;
552     RegI64 specific_rcx;
553     RegI64 specific_rdx;
554 #endif
555 
556 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
557     RegI32 specific_eax;
558     RegI32 specific_ecx;
559     RegI32 specific_edx;
560 #endif
561 
562 #if defined(JS_CODEGEN_X86)
563     AllocatableGeneralRegisterSet singleByteRegs_;
564 #endif
565 #if defined(JS_NUNBOX32)
566     RegI64 abiReturnRegI64;
567 #endif
568 
569     // The join registers are used to carry values out of blocks.
570     // JoinRegI32 and joinRegI64 must overlap: emitBrIf and
571     // emitBrTable assume that.
572 
573     RegI32 joinRegI32;
574     RegI64 joinRegI64;
575     RegF32 joinRegF32;
576     RegF64 joinRegF64;
577 
578     // More members: see the stk_ and ctl_ vectors, defined below.
579 
580   public:
581     BaseCompiler(const ModuleGeneratorData& mg,
582                  Decoder& decoder,
583                  const FuncBytes& func,
584                  const ValTypeVector& locals,
585                  FuncCompileResults& compileResults);
586 
587     MOZ_MUST_USE bool init();
588 
589     void finish();
590 
591     MOZ_MUST_USE bool emitFunction();
592 
593     // Used by some of the ScratchRegister implementations.
operator MacroAssembler&() const594     operator MacroAssembler&() const { return masm; }
595 
596 #ifdef DEBUG
scratchRegisterTaken() const597     bool scratchRegisterTaken() const {
598         return scratchRegisterTaken_;
599     }
setScratchRegisterTaken(bool state)600     void setScratchRegisterTaken(bool state) {
601         scratchRegisterTaken_ = state;
602     }
603 #endif
604 
605   private:
606 
607     ////////////////////////////////////////////////////////////
608     //
609     // Out of line code management.
610 
addOutOfLineCode(OutOfLineCode * ool)611     MOZ_MUST_USE OutOfLineCode* addOutOfLineCode(OutOfLineCode* ool) {
612         if (!ool || !outOfLine_.append(ool))
613             return nullptr;
614         ool->setFramePushed(masm.framePushed());
615         return ool;
616     }
617 
generateOutOfLineCode()618     MOZ_MUST_USE bool generateOutOfLineCode() {
619         for (uint32_t i = 0; i < outOfLine_.length(); i++) {
620             OutOfLineCode* ool = outOfLine_[i];
621             ool->bind(masm);
622             ool->generate(masm);
623         }
624 
625         return !masm.oom();
626     }
627 
628     ////////////////////////////////////////////////////////////
629     //
630     // The stack frame.
631 
632     // SP-relative load and store.
633 
localOffsetToSPOffset(int32_t offset)634     int32_t localOffsetToSPOffset(int32_t offset) {
635         return masm.framePushed() - offset;
636     }
637 
storeToFrameI32(Register r,int32_t offset)638     void storeToFrameI32(Register r, int32_t offset) {
639         masm.store32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
640     }
641 
storeToFrameI64(Register64 r,int32_t offset)642     void storeToFrameI64(Register64 r, int32_t offset) {
643         masm.store64(r, Address(StackPointer, localOffsetToSPOffset(offset)));
644     }
645 
storeToFramePtr(Register r,int32_t offset)646     void storeToFramePtr(Register r, int32_t offset) {
647         masm.storePtr(r, Address(StackPointer, localOffsetToSPOffset(offset)));
648     }
649 
storeToFrameF64(FloatRegister r,int32_t offset)650     void storeToFrameF64(FloatRegister r, int32_t offset) {
651         masm.storeDouble(r, Address(StackPointer, localOffsetToSPOffset(offset)));
652     }
653 
storeToFrameF32(FloatRegister r,int32_t offset)654     void storeToFrameF32(FloatRegister r, int32_t offset) {
655         masm.storeFloat32(r, Address(StackPointer, localOffsetToSPOffset(offset)));
656     }
657 
loadFromFrameI32(Register r,int32_t offset)658     void loadFromFrameI32(Register r, int32_t offset) {
659         masm.load32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
660     }
661 
loadFromFrameI64(Register64 r,int32_t offset)662     void loadFromFrameI64(Register64 r, int32_t offset) {
663         masm.load64(Address(StackPointer, localOffsetToSPOffset(offset)), r);
664     }
665 
loadFromFramePtr(Register r,int32_t offset)666     void loadFromFramePtr(Register r, int32_t offset) {
667         masm.loadPtr(Address(StackPointer, localOffsetToSPOffset(offset)), r);
668     }
669 
loadFromFrameF64(FloatRegister r,int32_t offset)670     void loadFromFrameF64(FloatRegister r, int32_t offset) {
671         masm.loadDouble(Address(StackPointer, localOffsetToSPOffset(offset)), r);
672     }
673 
loadFromFrameF32(FloatRegister r,int32_t offset)674     void loadFromFrameF32(FloatRegister r, int32_t offset) {
675         masm.loadFloat32(Address(StackPointer, localOffsetToSPOffset(offset)), r);
676     }
677 
678     // Stack-allocated local slots.
679 
pushLocal(size_t nbytes)680     int32_t pushLocal(size_t nbytes) {
681         if (nbytes == 8)
682             localSize_ = AlignBytes(localSize_, 8u);
683         else if (nbytes == 16)
684             localSize_ = AlignBytes(localSize_, 16u);
685         localSize_ += nbytes;
686         return localSize_;          // Locals grow down so capture base address
687     }
688 
frameOffsetFromSlot(uint32_t slot,MIRType type)689     int32_t frameOffsetFromSlot(uint32_t slot, MIRType type) {
690         MOZ_ASSERT(localInfo_[slot].type() == type);
691         return localInfo_[slot].offs();
692     }
693 
694     ////////////////////////////////////////////////////////////
695     //
696     // Low-level register allocation.
697 
isAvailable(Register r)698     bool isAvailable(Register r) {
699         return availGPR_.has(r);
700     }
701 
hasGPR()702     bool hasGPR() {
703         return !availGPR_.empty();
704     }
705 
allocGPR(Register r)706     void allocGPR(Register r) {
707         MOZ_ASSERT(isAvailable(r));
708         availGPR_.take(r);
709     }
710 
allocGPR()711     Register allocGPR() {
712         MOZ_ASSERT(hasGPR());
713         return availGPR_.takeAny();
714     }
715 
freeGPR(Register r)716     void freeGPR(Register r) {
717         availGPR_.add(r);
718     }
719 
isAvailable(Register64 r)720     bool isAvailable(Register64 r) {
721 #ifdef JS_PUNBOX64
722         return isAvailable(r.reg);
723 #else
724         return isAvailable(r.low) && isAvailable(r.high);
725 #endif
726     }
727 
hasInt64()728     bool hasInt64() {
729 #ifdef JS_PUNBOX64
730         return !availGPR_.empty();
731 #else
732         if (availGPR_.empty())
733             return false;
734         Register r = allocGPR();
735         bool available = !availGPR_.empty();
736         freeGPR(r);
737         return available;
738 #endif
739     }
740 
allocInt64(Register64 r)741     void allocInt64(Register64 r) {
742         MOZ_ASSERT(isAvailable(r));
743 #ifdef JS_PUNBOX64
744         availGPR_.take(r.reg);
745 #else
746         availGPR_.take(r.low);
747         availGPR_.take(r.high);
748 #endif
749     }
750 
allocInt64()751     Register64 allocInt64() {
752         MOZ_ASSERT(hasInt64());
753 #ifdef JS_PUNBOX64
754         return Register64(availGPR_.takeAny());
755 #else
756         Register high = availGPR_.takeAny();
757         Register low = availGPR_.takeAny();
758         return Register64(high, low);
759 #endif
760     }
761 
freeInt64(Register64 r)762     void freeInt64(Register64 r) {
763 #ifdef JS_PUNBOX64
764         availGPR_.add(r.reg);
765 #else
766         availGPR_.add(r.low);
767         availGPR_.add(r.high);
768 #endif
769     }
770 
771     // Notes on float register allocation.
772     //
773     // The general rule in SpiderMonkey is that float registers can
774     // alias double registers, but there are predicates to handle
775     // exceptions to that rule: hasUnaliasedDouble() and
776     // hasMultiAlias().  The way aliasing actually works is platform
777     // dependent and exposed through the aliased(n, &r) predicate,
778     // etc.
779     //
780     //  - hasUnaliasedDouble(): on ARM VFPv3-D32 there are double
781     //    registers that cannot be treated as float.
782     //  - hasMultiAlias(): on ARM and MIPS a double register aliases
783     //    two float registers.
784     //  - notes in Architecture-arm.h indicate that when we use a
785     //    float register that aliases a double register we only use
786     //    the low float register, never the high float register.  I
787     //    think those notes lie, or at least are confusing.
788     //  - notes in Architecture-mips32.h suggest that the MIPS port
789     //    will use both low and high float registers except on the
790     //    Longsoon, which may be the only MIPS that's being tested, so
791     //    who knows what's working.
792     //  - SIMD is not yet implemented on ARM or MIPS so constraints
793     //    may change there.
794     //
795     // On some platforms (x86, x64, ARM64) but not all (ARM)
796     // ScratchFloat32Register is the same as ScratchDoubleRegister.
797     //
798     // It's a basic invariant of the AllocatableRegisterSet that it
799     // deals properly with aliasing of registers: if s0 or s1 are
800     // allocated then d0 is not allocatable; if s0 and s1 are freed
801     // individually then d0 becomes allocatable.
802 
803     template<MIRType t>
maskFromTypeFPU()804     FloatRegisters::SetType maskFromTypeFPU() {
805         static_assert(t == MIRType::Float32 || t == MIRType::Double, "Float mask type");
806         if (t == MIRType::Float32)
807             return FloatRegisters::AllSingleMask;
808         return FloatRegisters::AllDoubleMask;
809     }
810 
811     template<MIRType t>
hasFPU()812     bool hasFPU() {
813         return !!(availFPU_.bits() & maskFromTypeFPU<t>());
814     }
815 
isAvailable(FloatRegister r)816     bool isAvailable(FloatRegister r) {
817         return availFPU_.has(r);
818     }
819 
allocFPU(FloatRegister r)820     void allocFPU(FloatRegister r) {
821         MOZ_ASSERT(isAvailable(r));
822         availFPU_.take(r);
823     }
824 
825     template<MIRType t>
allocFPU()826     FloatRegister allocFPU() {
827         MOZ_ASSERT(hasFPU<t>());
828         FloatRegister r =
829             FloatRegisterSet::Intersect(FloatRegisterSet(availFPU_.bits()),
830                                         FloatRegisterSet(maskFromTypeFPU<t>())).getAny();
831         availFPU_.take(r);
832         return r;
833     }
834 
freeFPU(FloatRegister r)835     void freeFPU(FloatRegister r) {
836         availFPU_.add(r);
837     }
838 
839     ////////////////////////////////////////////////////////////
840     //
841     // Value stack and high-level register allocation.
842     //
843     // The value stack facilitates some on-the-fly register allocation
844     // and immediate-constant use.  It tracks constants, latent
845     // references to locals, register contents, and values on the CPU
846     // stack.
847     //
848     // The stack can be flushed to memory using sync().  This is handy
849     // to avoid problems with control flow and messy register usage
850     // patterns.
851 
852     struct Stk
853     {
854         enum Kind
855         {
856             // The Mem opcodes are all clustered at the beginning to
857             // allow for a quick test within sync().
858             MemI32,               // 32-bit integer stack value ("offs")
859             MemI64,               // 64-bit integer stack value ("offs")
860             MemF32,               // 32-bit floating stack value ("offs")
861             MemF64,               // 64-bit floating stack value ("offs")
862 
863             // The Local opcodes follow the Mem opcodes for a similar
864             // quick test within hasLocal().
865             LocalI32,             // Local int32 var ("slot")
866             LocalI64,             // Local int64 var ("slot")
867             LocalF32,             // Local float32 var ("slot")
868             LocalF64,             // Local double var ("slot")
869 
870             RegisterI32,          // 32-bit integer register ("i32reg")
871             RegisterI64,          // 64-bit integer register ("i64reg")
872             RegisterF32,          // 32-bit floating register ("f32reg")
873             RegisterF64,          // 64-bit floating register ("f64reg")
874 
875             ConstI32,             // 32-bit integer constant ("i32val")
876             ConstI64,             // 64-bit integer constant ("i64val")
877             ConstF32,             // 32-bit floating constant ("f32val")
878             ConstF64,             // 64-bit floating constant ("f64val")
879 
880             None                  // Uninitialized or void
881         };
882 
883         Kind kind_;
884 
885         static const Kind MemLast = MemF64;
886         static const Kind LocalLast = LocalF64;
887 
888         union {
889             RegI32   i32reg_;
890             RegI64   i64reg_;
891             RegF32   f32reg_;
892             RegF64   f64reg_;
893             int32_t  i32val_;
894             int64_t  i64val_;
895             RawF32   f32val_;
896             RawF64   f64val_;
897             uint32_t slot_;
898             uint32_t offs_;
899         };
900 
Stkjs::wasm::BaseCompiler::Stk901         Stk() { kind_ = None; }
902 
kindjs::wasm::BaseCompiler::Stk903         Kind kind() const { return kind_; }
isMemjs::wasm::BaseCompiler::Stk904         bool isMem() const { return kind_ <= MemLast; }
905 
i32regjs::wasm::BaseCompiler::Stk906         RegI32   i32reg() const { MOZ_ASSERT(kind_ == RegisterI32); return i32reg_; }
i64regjs::wasm::BaseCompiler::Stk907         RegI64   i64reg() const { MOZ_ASSERT(kind_ == RegisterI64); return i64reg_; }
f32regjs::wasm::BaseCompiler::Stk908         RegF32   f32reg() const { MOZ_ASSERT(kind_ == RegisterF32); return f32reg_; }
f64regjs::wasm::BaseCompiler::Stk909         RegF64   f64reg() const { MOZ_ASSERT(kind_ == RegisterF64); return f64reg_; }
i32valjs::wasm::BaseCompiler::Stk910         int32_t  i32val() const { MOZ_ASSERT(kind_ == ConstI32); return i32val_; }
i64valjs::wasm::BaseCompiler::Stk911         int64_t  i64val() const { MOZ_ASSERT(kind_ == ConstI64); return i64val_; }
f32valjs::wasm::BaseCompiler::Stk912         RawF32   f32val() const { MOZ_ASSERT(kind_ == ConstF32); return f32val_; }
f64valjs::wasm::BaseCompiler::Stk913         RawF64   f64val() const { MOZ_ASSERT(kind_ == ConstF64); return f64val_; }
slotjs::wasm::BaseCompiler::Stk914         uint32_t slot() const { MOZ_ASSERT(kind_ > MemLast && kind_ <= LocalLast); return slot_; }
offsjs::wasm::BaseCompiler::Stk915         uint32_t offs() const { MOZ_ASSERT(isMem()); return offs_; }
916 
setI32Regjs::wasm::BaseCompiler::Stk917         void setI32Reg(RegI32 r) { kind_ = RegisterI32; i32reg_ = r; }
setI64Regjs::wasm::BaseCompiler::Stk918         void setI64Reg(RegI64 r) { kind_ = RegisterI64; i64reg_ = r; }
setF32Regjs::wasm::BaseCompiler::Stk919         void setF32Reg(RegF32 r) { kind_ = RegisterF32; f32reg_ = r; }
setF64Regjs::wasm::BaseCompiler::Stk920         void setF64Reg(RegF64 r) { kind_ = RegisterF64; f64reg_ = r; }
setI32Valjs::wasm::BaseCompiler::Stk921         void setI32Val(int32_t v) { kind_ = ConstI32; i32val_ = v; }
setI64Valjs::wasm::BaseCompiler::Stk922         void setI64Val(int64_t v) { kind_ = ConstI64; i64val_ = v; }
setF32Valjs::wasm::BaseCompiler::Stk923         void setF32Val(RawF32 v) { kind_ = ConstF32; f32val_ = v; }
setF64Valjs::wasm::BaseCompiler::Stk924         void setF64Val(RawF64 v) { kind_ = ConstF64; f64val_ = v; }
setSlotjs::wasm::BaseCompiler::Stk925         void setSlot(Kind k, uint32_t v) { MOZ_ASSERT(k > MemLast && k <= LocalLast); kind_ = k; slot_ = v; }
setOffsjs::wasm::BaseCompiler::Stk926         void setOffs(Kind k, uint32_t v) { MOZ_ASSERT(k <= MemLast); kind_ = k; offs_ = v; }
927     };
928 
929     Vector<Stk, 8, SystemAllocPolicy> stk_;
930 
push()931     Stk& push() {
932         stk_.infallibleEmplaceBack(Stk());
933         return stk_.back();
934     }
935 
invalidRegister64()936     Register64 invalidRegister64() {
937         return Register64::Invalid();
938     }
939 
invalidI32()940     RegI32 invalidI32() {
941         return RegI32(Register::Invalid());
942     }
943 
invalidI64()944     RegI64 invalidI64() {
945         return RegI64(invalidRegister64());
946     }
947 
invalidF64()948     RegF64 invalidF64() {
949         return RegF64(InvalidFloatReg);
950     }
951 
fromI64(RegI64 r)952     RegI32 fromI64(RegI64 r) {
953         return RegI32(lowPart(r));
954     }
955 
widenI32(RegI32 r)956     RegI64 widenI32(RegI32 r) {
957         MOZ_ASSERT(!isAvailable(r.reg));
958 #ifdef JS_PUNBOX64
959         return RegI64(Register64(r.reg));
960 #else
961         RegI32 high = needI32();
962         return RegI64(Register64(high.reg, r.reg));
963 #endif
964     }
965 
lowPart(RegI64 r)966     Register lowPart(RegI64 r) {
967 #ifdef JS_PUNBOX64
968         return r.reg.reg;
969 #else
970         return r.reg.low;
971 #endif
972     }
973 
maybeHighPart(RegI64 r)974     Register maybeHighPart(RegI64 r) {
975 #ifdef JS_PUNBOX64
976         return Register::Invalid();
977 #else
978         return r.reg.high;
979 #endif
980     }
981 
maybeClearHighPart(RegI64 r)982     void maybeClearHighPart(RegI64 r) {
983 #ifdef JS_NUNBOX32
984         masm.move32(Imm32(0), r.reg.high);
985 #endif
986     }
987 
freeI32(RegI32 r)988     void freeI32(RegI32 r) {
989         freeGPR(r.reg);
990     }
991 
freeI64(RegI64 r)992     void freeI64(RegI64 r) {
993         freeInt64(r.reg);
994     }
995 
freeI64Except(RegI64 r,RegI32 except)996     void freeI64Except(RegI64 r, RegI32 except) {
997 #ifdef JS_PUNBOX64
998         MOZ_ASSERT(r.reg.reg == except.reg);
999 #else
1000         MOZ_ASSERT(r.reg.high == except.reg || r.reg.low == except.reg);
1001         freeI64(r);
1002         needI32(except);
1003 #endif
1004     }
1005 
freeF64(RegF64 r)1006     void freeF64(RegF64 r) {
1007         freeFPU(r.reg);
1008     }
1009 
freeF32(RegF32 r)1010     void freeF32(RegF32 r) {
1011         freeFPU(r.reg);
1012     }
1013 
needI32()1014     MOZ_MUST_USE RegI32 needI32() {
1015         if (!hasGPR())
1016             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1017         return RegI32(allocGPR());
1018     }
1019 
needI32(RegI32 specific)1020     void needI32(RegI32 specific) {
1021         if (!isAvailable(specific.reg))
1022             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1023         allocGPR(specific.reg);
1024     }
1025 
1026     // TODO / OPTIMIZE: need2xI32() can be optimized along with needI32()
1027     // to avoid sync(). (Bug 1316802)
1028 
need2xI32(RegI32 r0,RegI32 r1)1029     void need2xI32(RegI32 r0, RegI32 r1) {
1030         needI32(r0);
1031         needI32(r1);
1032     }
1033 
needI64()1034     MOZ_MUST_USE RegI64 needI64() {
1035         if (!hasInt64())
1036             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1037         return RegI64(allocInt64());
1038     }
1039 
needI64(RegI64 specific)1040     void needI64(RegI64 specific) {
1041         if (!isAvailable(specific.reg))
1042             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1043         allocInt64(specific.reg);
1044     }
1045 
need2xI64(RegI64 r0,RegI64 r1)1046     void need2xI64(RegI64 r0, RegI64 r1) {
1047         needI64(r0);
1048         needI64(r1);
1049     }
1050 
needF32()1051     MOZ_MUST_USE RegF32 needF32() {
1052         if (!hasFPU<MIRType::Float32>())
1053             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1054         return RegF32(allocFPU<MIRType::Float32>());
1055     }
1056 
needF32(RegF32 specific)1057     void needF32(RegF32 specific) {
1058         if (!isAvailable(specific.reg))
1059             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1060         allocFPU(specific.reg);
1061     }
1062 
needF64()1063     MOZ_MUST_USE RegF64 needF64() {
1064         if (!hasFPU<MIRType::Double>())
1065             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1066         return RegF64(allocFPU<MIRType::Double>());
1067     }
1068 
needF64(RegF64 specific)1069     void needF64(RegF64 specific) {
1070         if (!isAvailable(specific.reg))
1071             sync();            // TODO / OPTIMIZE: improve this (Bug 1316802)
1072         allocFPU(specific.reg);
1073     }
1074 
moveI32(RegI32 src,RegI32 dest)1075     void moveI32(RegI32 src, RegI32 dest) {
1076         if (src != dest)
1077             masm.move32(src.reg, dest.reg);
1078     }
1079 
moveI64(RegI64 src,RegI64 dest)1080     void moveI64(RegI64 src, RegI64 dest) {
1081         if (src != dest)
1082             masm.move64(src.reg, dest.reg);
1083     }
1084 
moveF64(RegF64 src,RegF64 dest)1085     void moveF64(RegF64 src, RegF64 dest) {
1086         if (src != dest)
1087             masm.moveDouble(src.reg, dest.reg);
1088     }
1089 
moveF32(RegF32 src,RegF32 dest)1090     void moveF32(RegF32 src, RegF32 dest) {
1091         if (src != dest)
1092             masm.moveFloat32(src.reg, dest.reg);
1093     }
1094 
setI64(int64_t v,RegI64 r)1095     void setI64(int64_t v, RegI64 r) {
1096         masm.move64(Imm64(v), r.reg);
1097     }
1098 
loadConstI32(Register r,Stk & src)1099     void loadConstI32(Register r, Stk& src) {
1100         masm.mov(ImmWord((uint32_t)src.i32val() & 0xFFFFFFFFU), r);
1101     }
1102 
loadMemI32(Register r,Stk & src)1103     void loadMemI32(Register r, Stk& src) {
1104         loadFromFrameI32(r, src.offs());
1105     }
1106 
loadLocalI32(Register r,Stk & src)1107     void loadLocalI32(Register r, Stk& src) {
1108         loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int32));
1109     }
1110 
loadRegisterI32(Register r,Stk & src)1111     void loadRegisterI32(Register r, Stk& src) {
1112         if (src.i32reg().reg != r)
1113             masm.move32(src.i32reg().reg, r);
1114     }
1115 
loadI32(Register r,Stk & src)1116     void loadI32(Register r, Stk& src) {
1117         switch (src.kind()) {
1118           case Stk::ConstI32:
1119             loadConstI32(r, src);
1120             break;
1121           case Stk::MemI32:
1122             loadMemI32(r, src);
1123             break;
1124           case Stk::LocalI32:
1125             loadLocalI32(r, src);
1126             break;
1127           case Stk::RegisterI32:
1128             loadRegisterI32(r, src);
1129             break;
1130           case Stk::None:
1131             break;
1132           default:
1133             MOZ_CRASH("Compiler bug: Expected int on stack");
1134         }
1135     }
1136 
1137     // TODO / OPTIMIZE: Refactor loadI64, loadF64, and loadF32 in the
1138     // same way as loadI32 to avoid redundant dispatch in callers of
1139     // these load() functions.  (Bug 1316816, also see annotations on
1140     // popI64 et al below.)
1141 
loadI64(Register64 r,Stk & src)1142     void loadI64(Register64 r, Stk& src) {
1143         switch (src.kind()) {
1144           case Stk::ConstI64:
1145             masm.move64(Imm64(src.i64val()), r);
1146             break;
1147           case Stk::MemI64:
1148             loadFromFrameI64(r, src.offs());
1149             break;
1150           case Stk::LocalI64:
1151             loadFromFrameI64(r, frameOffsetFromSlot(src.slot(), MIRType::Int64));
1152             break;
1153           case Stk::RegisterI64:
1154             if (src.i64reg().reg != r)
1155                 masm.move64(src.i64reg().reg, r);
1156             break;
1157           case Stk::None:
1158             break;
1159           default:
1160             MOZ_CRASH("Compiler bug: Expected int on stack");
1161         }
1162     }
1163 
1164 #ifdef JS_NUNBOX32
loadI64Low(Register r,Stk & src)1165     void loadI64Low(Register r, Stk& src) {
1166         switch (src.kind()) {
1167           case Stk::ConstI64:
1168             masm.move32(Imm64(src.i64val()).low(), r);
1169             break;
1170           case Stk::MemI64:
1171             loadFromFrameI32(r, src.offs() - INT64LOW_OFFSET);
1172             break;
1173           case Stk::LocalI64:
1174             loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64LOW_OFFSET);
1175             break;
1176           case Stk::RegisterI64:
1177             if (src.i64reg().reg.low != r)
1178                 masm.move32(src.i64reg().reg.low, r);
1179             break;
1180           case Stk::None:
1181             break;
1182           default:
1183             MOZ_CRASH("Compiler bug: Expected int on stack");
1184         }
1185     }
1186 
loadI64High(Register r,Stk & src)1187     void loadI64High(Register r, Stk& src) {
1188         switch (src.kind()) {
1189           case Stk::ConstI64:
1190             masm.move32(Imm64(src.i64val()).hi(), r);
1191             break;
1192           case Stk::MemI64:
1193             loadFromFrameI32(r, src.offs() - INT64HIGH_OFFSET);
1194             break;
1195           case Stk::LocalI64:
1196             loadFromFrameI32(r, frameOffsetFromSlot(src.slot(), MIRType::Int64) - INT64HIGH_OFFSET);
1197             break;
1198           case Stk::RegisterI64:
1199             if (src.i64reg().reg.high != r)
1200                 masm.move32(src.i64reg().reg.high, r);
1201             break;
1202           case Stk::None:
1203             break;
1204           default:
1205             MOZ_CRASH("Compiler bug: Expected int on stack");
1206         }
1207     }
1208 #endif
1209 
loadF64(FloatRegister r,Stk & src)1210     void loadF64(FloatRegister r, Stk& src) {
1211         switch (src.kind()) {
1212           case Stk::ConstF64:
1213             masm.loadConstantDouble(src.f64val(), r);
1214             break;
1215           case Stk::MemF64:
1216             loadFromFrameF64(r, src.offs());
1217             break;
1218           case Stk::LocalF64:
1219             loadFromFrameF64(r, frameOffsetFromSlot(src.slot(), MIRType::Double));
1220             break;
1221           case Stk::RegisterF64:
1222             if (src.f64reg().reg != r)
1223                 masm.moveDouble(src.f64reg().reg, r);
1224             break;
1225           case Stk::None:
1226             break;
1227           default:
1228             MOZ_CRASH("Compiler bug: expected double on stack");
1229         }
1230     }
1231 
loadF32(FloatRegister r,Stk & src)1232     void loadF32(FloatRegister r, Stk& src) {
1233         switch (src.kind()) {
1234           case Stk::ConstF32:
1235             masm.loadConstantFloat32(src.f32val(), r);
1236             break;
1237           case Stk::MemF32:
1238             loadFromFrameF32(r, src.offs());
1239             break;
1240           case Stk::LocalF32:
1241             loadFromFrameF32(r, frameOffsetFromSlot(src.slot(), MIRType::Float32));
1242             break;
1243           case Stk::RegisterF32:
1244             if (src.f32reg().reg != r)
1245                 masm.moveFloat32(src.f32reg().reg, r);
1246             break;
1247           case Stk::None:
1248             break;
1249           default:
1250             MOZ_CRASH("Compiler bug: expected float on stack");
1251         }
1252     }
1253 
1254     // Flush all local and register value stack elements to memory.
1255     //
1256     // TODO / OPTIMIZE: As this is fairly expensive and causes worse
1257     // code to be emitted subsequently, it is useful to avoid calling
1258     // it.  (Bug 1316802)
1259     //
1260     // Some optimization has been done already.  Remaining
1261     // opportunities:
1262     //
1263     //  - It would be interesting to see if we can specialize it
1264     //    before calls with particularly simple signatures, or where
1265     //    we can do parallel assignment of register arguments, or
1266     //    similar.  See notes in emitCall().
1267     //
1268     //  - Operations that need specific registers: multiply, quotient,
1269     //    remainder, will tend to sync because the registers we need
1270     //    will tend to be allocated.  We may be able to avoid that by
1271     //    prioritizing registers differently (takeLast instead of
1272     //    takeFirst) but we may also be able to allocate an unused
1273     //    register on demand to free up one we need, thus avoiding the
1274     //    sync.  That type of fix would go into needI32().
1275 
sync()1276     void sync() {
1277         size_t start = 0;
1278         size_t lim = stk_.length();
1279 
1280         for (size_t i = lim; i > 0; i--) {
1281             // Memory opcodes are first in the enum, single check against MemLast is fine.
1282             if (stk_[i - 1].kind() <= Stk::MemLast) {
1283                 start = i;
1284                 break;
1285             }
1286         }
1287 
1288         for (size_t i = start; i < lim; i++) {
1289             Stk& v = stk_[i];
1290             switch (v.kind()) {
1291               case Stk::LocalI32: {
1292                 ScratchI32 scratch(*this);
1293                 loadLocalI32(scratch, v);
1294                 masm.Push(scratch);
1295                 v.setOffs(Stk::MemI32, masm.framePushed());
1296                 break;
1297               }
1298               case Stk::RegisterI32: {
1299                 masm.Push(v.i32reg().reg);
1300                 freeI32(v.i32reg());
1301                 v.setOffs(Stk::MemI32, masm.framePushed());
1302                 break;
1303               }
1304               case Stk::LocalI64: {
1305                 ScratchI32 scratch(*this);
1306 #ifdef JS_PUNBOX64
1307                 loadI64(Register64(scratch), v);
1308                 masm.Push(scratch);
1309 #else
1310                 int32_t offset = frameOffsetFromSlot(v.slot(), MIRType::Int64);
1311                 loadFromFrameI32(scratch, offset - INT64HIGH_OFFSET);
1312                 masm.Push(scratch);
1313                 loadFromFrameI32(scratch, offset - INT64LOW_OFFSET);
1314                 masm.Push(scratch);
1315 #endif
1316                 v.setOffs(Stk::MemI64, masm.framePushed());
1317                 break;
1318               }
1319               case Stk::RegisterI64: {
1320 #ifdef JS_PUNBOX64
1321                 masm.Push(v.i64reg().reg.reg);
1322                 freeI64(v.i64reg());
1323 #else
1324                 masm.Push(v.i64reg().reg.high);
1325                 masm.Push(v.i64reg().reg.low);
1326                 freeI64(v.i64reg());
1327 #endif
1328                 v.setOffs(Stk::MemI64, masm.framePushed());
1329                 break;
1330               }
1331               case Stk::LocalF64: {
1332                 ScratchF64 scratch(*this);
1333                 loadF64(scratch, v);
1334                 masm.Push(scratch);
1335                 v.setOffs(Stk::MemF64, masm.framePushed());
1336                 break;
1337               }
1338               case Stk::RegisterF64: {
1339                 masm.Push(v.f64reg().reg);
1340                 freeF64(v.f64reg());
1341                 v.setOffs(Stk::MemF64, masm.framePushed());
1342                 break;
1343               }
1344               case Stk::LocalF32: {
1345                 ScratchF32 scratch(*this);
1346                 loadF32(scratch, v);
1347                 masm.Push(scratch);
1348                 v.setOffs(Stk::MemF32, masm.framePushed());
1349                 break;
1350               }
1351               case Stk::RegisterF32: {
1352                 masm.Push(v.f32reg().reg);
1353                 freeF32(v.f32reg());
1354                 v.setOffs(Stk::MemF32, masm.framePushed());
1355                 break;
1356               }
1357               default: {
1358                 break;
1359               }
1360             }
1361         }
1362 
1363         maxFramePushed_ = Max(maxFramePushed_, int32_t(masm.framePushed()));
1364     }
1365 
1366     // This is an optimization used to avoid calling sync() for
1367     // setLocal(): if the local does not exist unresolved on the stack
1368     // then we can skip the sync.
1369 
hasLocal(uint32_t slot)1370     bool hasLocal(uint32_t slot) {
1371         for (size_t i = stk_.length(); i > 0; i--) {
1372             // Memory opcodes are first in the enum, single check against MemLast is fine.
1373             Stk::Kind kind = stk_[i-1].kind();
1374             if (kind <= Stk::MemLast)
1375                 return false;
1376 
1377             // Local opcodes follow memory opcodes in the enum, single check against
1378             // LocalLast is sufficient.
1379             if (kind <= Stk::LocalLast && stk_[i-1].slot() == slot)
1380                 return true;
1381         }
1382         return false;
1383     }
1384 
syncLocal(uint32_t slot)1385     void syncLocal(uint32_t slot) {
1386         if (hasLocal(slot))
1387             sync();            // TODO / OPTIMIZE: Improve this?  (Bug 1316817)
1388     }
1389 
1390     // Push the register r onto the stack.
1391 
pushI32(RegI32 r)1392     void pushI32(RegI32 r) {
1393         MOZ_ASSERT(!isAvailable(r.reg));
1394         Stk& x = push();
1395         x.setI32Reg(r);
1396     }
1397 
pushI64(RegI64 r)1398     void pushI64(RegI64 r) {
1399         MOZ_ASSERT(!isAvailable(r.reg));
1400         Stk& x = push();
1401         x.setI64Reg(r);
1402     }
1403 
pushF64(RegF64 r)1404     void pushF64(RegF64 r) {
1405         MOZ_ASSERT(!isAvailable(r.reg));
1406         Stk& x = push();
1407         x.setF64Reg(r);
1408     }
1409 
pushF32(RegF32 r)1410     void pushF32(RegF32 r) {
1411         MOZ_ASSERT(!isAvailable(r.reg));
1412         Stk& x = push();
1413         x.setF32Reg(r);
1414     }
1415 
1416     // Push the value onto the stack.
1417 
pushI32(int32_t v)1418     void pushI32(int32_t v) {
1419         Stk& x = push();
1420         x.setI32Val(v);
1421     }
1422 
pushI64(int64_t v)1423     void pushI64(int64_t v) {
1424         Stk& x = push();
1425         x.setI64Val(v);
1426     }
1427 
pushF64(RawF64 v)1428     void pushF64(RawF64 v) {
1429         Stk& x = push();
1430         x.setF64Val(v);
1431     }
1432 
pushF32(RawF32 v)1433     void pushF32(RawF32 v) {
1434         Stk& x = push();
1435         x.setF32Val(v);
1436     }
1437 
1438     // Push the local slot onto the stack.  The slot will not be read
1439     // here; it will be read when it is consumed, or when a side
1440     // effect to the slot forces its value to be saved.
1441 
pushLocalI32(uint32_t slot)1442     void pushLocalI32(uint32_t slot) {
1443         Stk& x = push();
1444         x.setSlot(Stk::LocalI32, slot);
1445     }
1446 
pushLocalI64(uint32_t slot)1447     void pushLocalI64(uint32_t slot) {
1448         Stk& x = push();
1449         x.setSlot(Stk::LocalI64, slot);
1450     }
1451 
pushLocalF64(uint32_t slot)1452     void pushLocalF64(uint32_t slot) {
1453         Stk& x = push();
1454         x.setSlot(Stk::LocalF64, slot);
1455     }
1456 
pushLocalF32(uint32_t slot)1457     void pushLocalF32(uint32_t slot) {
1458         Stk& x = push();
1459         x.setSlot(Stk::LocalF32, slot);
1460     }
1461 
1462     // PRIVATE.  Call only from other popI32() variants.
1463     // v must be the stack top.
1464 
popI32(Stk & v,RegI32 r)1465     void popI32(Stk& v, RegI32 r) {
1466         switch (v.kind()) {
1467           case Stk::ConstI32:
1468             loadConstI32(r.reg, v);
1469             break;
1470           case Stk::LocalI32:
1471             loadLocalI32(r.reg, v);
1472             break;
1473           case Stk::MemI32:
1474             masm.Pop(r.reg);
1475             break;
1476           case Stk::RegisterI32:
1477             moveI32(v.i32reg(), r);
1478             break;
1479           case Stk::None:
1480             // This case crops up in situations where there's unreachable code that
1481             // the type system interprets as "generating" a value of the correct type:
1482             //
1483             //   (if (return) E1 E2)                    type is type(E1) meet type(E2)
1484             //   (if E (unreachable) (i32.const 1))     type is int
1485             //   (if E (i32.const 1) (unreachable))     type is int
1486             //
1487             // It becomes silly to handle this throughout the code, so just handle it
1488             // here even if that means weaker run-time checking.
1489             break;
1490           default:
1491             MOZ_CRASH("Compiler bug: expected int on stack");
1492         }
1493     }
1494 
popI32()1495     MOZ_MUST_USE RegI32 popI32() {
1496         Stk& v = stk_.back();
1497         RegI32 r;
1498         if (v.kind() == Stk::RegisterI32)
1499             r = v.i32reg();
1500         else
1501             popI32(v, (r = needI32()));
1502         stk_.popBack();
1503         return r;
1504     }
1505 
popI32(RegI32 specific)1506     RegI32 popI32(RegI32 specific) {
1507         Stk& v = stk_.back();
1508 
1509         if (!(v.kind() == Stk::RegisterI32 && v.i32reg() == specific)) {
1510             needI32(specific);
1511             popI32(v, specific);
1512             if (v.kind() == Stk::RegisterI32)
1513                 freeI32(v.i32reg());
1514         }
1515 
1516         stk_.popBack();
1517         return specific;
1518     }
1519 
1520     // PRIVATE.  Call only from other popI64() variants.
1521     // v must be the stack top.
1522 
popI64(Stk & v,RegI64 r)1523     void popI64(Stk& v, RegI64 r) {
1524         // TODO / OPTIMIZE: avoid loadI64() here.  (Bug 1316816)
1525         switch (v.kind()) {
1526           case Stk::ConstI64:
1527           case Stk::LocalI64:
1528             loadI64(r.reg, v);
1529             break;
1530           case Stk::MemI64:
1531 #ifdef JS_PUNBOX64
1532             masm.Pop(r.reg.reg);
1533 #else
1534             masm.Pop(r.reg.low);
1535             masm.Pop(r.reg.high);
1536 #endif
1537             break;
1538           case Stk::RegisterI64:
1539             moveI64(v.i64reg(), r);
1540             break;
1541           case Stk::None:
1542             // See popI32()
1543             break;
1544           default:
1545             MOZ_CRASH("Compiler bug: expected long on stack");
1546         }
1547     }
1548 
popI64()1549     MOZ_MUST_USE RegI64 popI64() {
1550         Stk& v = stk_.back();
1551         RegI64 r;
1552         if (v.kind() == Stk::RegisterI64)
1553             r = v.i64reg();
1554         else
1555             popI64(v, (r = needI64()));
1556         stk_.popBack();
1557         return r;
1558     }
1559 
1560     // Note, the stack top can be in one half of "specific" on 32-bit
1561     // systems.  We can optimize, but for simplicity, if the register
1562     // does not match exactly, then just force the stack top to memory
1563     // and then read it back in.
1564 
popI64(RegI64 specific)1565     RegI64 popI64(RegI64 specific) {
1566         Stk& v = stk_.back();
1567 
1568         if (!(v.kind() == Stk::RegisterI64 && v.i64reg() == specific)) {
1569             needI64(specific);
1570             popI64(v, specific);
1571             if (v.kind() == Stk::RegisterI64)
1572                 freeI64(v.i64reg());
1573         }
1574 
1575         stk_.popBack();
1576         return specific;
1577     }
1578 
1579     // PRIVATE.  Call only from other popF64() variants.
1580     // v must be the stack top.
1581 
popF64(Stk & v,RegF64 r)1582     void popF64(Stk& v, RegF64 r) {
1583         // TODO / OPTIMIZE: avoid loadF64 here.  (Bug 1316816)
1584         switch (v.kind()) {
1585           case Stk::ConstF64:
1586           case Stk::LocalF64:
1587             loadF64(r.reg, v);
1588             break;
1589           case Stk::MemF64:
1590             masm.Pop(r.reg);
1591             break;
1592           case Stk::RegisterF64:
1593             moveF64(v.f64reg(), r);
1594             break;
1595           case Stk::None:
1596             // See popI32()
1597             break;
1598           default:
1599             MOZ_CRASH("Compiler bug: expected double on stack");
1600         }
1601     }
1602 
popF64()1603     MOZ_MUST_USE RegF64 popF64() {
1604         Stk& v = stk_.back();
1605         RegF64 r;
1606         if (v.kind() == Stk::RegisterF64)
1607             r = v.f64reg();
1608         else
1609             popF64(v, (r = needF64()));
1610         stk_.popBack();
1611         return r;
1612     }
1613 
popF64(RegF64 specific)1614     RegF64 popF64(RegF64 specific) {
1615         Stk& v = stk_.back();
1616 
1617         if (!(v.kind() == Stk::RegisterF64 && v.f64reg() == specific)) {
1618             needF64(specific);
1619             popF64(v, specific);
1620             if (v.kind() == Stk::RegisterF64)
1621                 freeF64(v.f64reg());
1622         }
1623 
1624         stk_.popBack();
1625         return specific;
1626     }
1627 
1628     // PRIVATE.  Call only from other popF32() variants.
1629     // v must be the stack top.
1630 
popF32(Stk & v,RegF32 r)1631     void popF32(Stk& v, RegF32 r) {
1632         // TODO / OPTIMIZE: avoid loadF32 here.  (Bug 1316816)
1633         switch (v.kind()) {
1634           case Stk::ConstF32:
1635           case Stk::LocalF32:
1636             loadF32(r.reg, v);
1637             break;
1638           case Stk::MemF32:
1639             masm.Pop(r.reg);
1640             break;
1641           case Stk::RegisterF32:
1642             moveF32(v.f32reg(), r);
1643             break;
1644           case Stk::None:
1645             // See popI32()
1646             break;
1647           default:
1648             MOZ_CRASH("Compiler bug: expected float on stack");
1649         }
1650     }
1651 
popF32()1652     MOZ_MUST_USE RegF32 popF32() {
1653         Stk& v = stk_.back();
1654         RegF32 r;
1655         if (v.kind() == Stk::RegisterF32)
1656             r = v.f32reg();
1657         else
1658             popF32(v, (r = needF32()));
1659         stk_.popBack();
1660         return r;
1661     }
1662 
popF32(RegF32 specific)1663     RegF32 popF32(RegF32 specific) {
1664         Stk& v = stk_.back();
1665 
1666         if (!(v.kind() == Stk::RegisterF32 && v.f32reg() == specific)) {
1667             needF32(specific);
1668             popF32(v, specific);
1669             if (v.kind() == Stk::RegisterF32)
1670                 freeF32(v.f32reg());
1671         }
1672 
1673         stk_.popBack();
1674         return specific;
1675     }
1676 
popConstI32(int32_t & c)1677     MOZ_MUST_USE bool popConstI32(int32_t& c) {
1678         Stk& v = stk_.back();
1679         if (v.kind() != Stk::ConstI32)
1680             return false;
1681         c = v.i32val();
1682         stk_.popBack();
1683         return true;
1684     }
1685 
1686     // TODO / OPTIMIZE (Bug 1316818): At the moment we use ReturnReg
1687     // for JoinReg.  It is possible other choices would lead to better
1688     // register allocation, as ReturnReg is often first in the
1689     // register set and will be heavily wanted by the register
1690     // allocator that uses takeFirst().
1691     //
1692     // Obvious options:
1693     //  - pick a register at the back of the register set
1694     //  - pick a random register per block (different blocks have
1695     //    different join regs)
1696     //
1697     // On the other hand, we sync() before every block and only the
1698     // JoinReg is live out of the block.  But on the way out, we
1699     // currently pop the JoinReg before freeing regs to be discarded,
1700     // so there is a real risk of some pointless shuffling there.  If
1701     // we instead integrate the popping of the join reg into the
1702     // popping of the stack we can just use the JoinReg as it will
1703     // become available in that process.
1704 
popJoinReg()1705     MOZ_MUST_USE AnyReg popJoinReg() {
1706         switch (stk_.back().kind()) {
1707           case Stk::RegisterI32:
1708           case Stk::ConstI32:
1709           case Stk::MemI32:
1710           case Stk::LocalI32:
1711             return AnyReg(popI32(joinRegI32));
1712           case Stk::RegisterI64:
1713           case Stk::ConstI64:
1714           case Stk::MemI64:
1715           case Stk::LocalI64:
1716             return AnyReg(popI64(joinRegI64));
1717           case Stk::RegisterF64:
1718           case Stk::ConstF64:
1719           case Stk::MemF64:
1720           case Stk::LocalF64:
1721             return AnyReg(popF64(joinRegF64));
1722           case Stk::RegisterF32:
1723           case Stk::ConstF32:
1724           case Stk::MemF32:
1725           case Stk::LocalF32:
1726             return AnyReg(popF32(joinRegF32));
1727           case Stk::None:
1728             stk_.popBack();
1729             return AnyReg();
1730           default:
1731             MOZ_CRASH("Compiler bug: unexpected value on stack");
1732         }
1733     }
1734 
allocJoinReg(ExprType type)1735     MOZ_MUST_USE AnyReg allocJoinReg(ExprType type) {
1736         switch (type) {
1737           case ExprType::I32:
1738             allocGPR(joinRegI32.reg);
1739             return AnyReg(joinRegI32);
1740           case ExprType::I64:
1741             allocInt64(joinRegI64.reg);
1742             return AnyReg(joinRegI64);
1743           case ExprType::F32:
1744             allocFPU(joinRegF32.reg);
1745             return AnyReg(joinRegF32);
1746           case ExprType::F64:
1747             allocFPU(joinRegF64.reg);
1748             return AnyReg(joinRegF64);
1749           case ExprType::Void:
1750             MOZ_CRASH("Compiler bug: allocating void join reg");
1751           default:
1752             MOZ_CRASH("Compiler bug: unexpected type");
1753         }
1754     }
1755 
pushJoinReg(AnyReg r)1756     void pushJoinReg(AnyReg r) {
1757         switch (r.tag) {
1758           case AnyReg::NONE:
1759             MOZ_CRASH("Compile bug: attempting to push void");
1760             break;
1761           case AnyReg::I32:
1762             pushI32(r.i32());
1763             break;
1764           case AnyReg::I64:
1765             pushI64(r.i64());
1766             break;
1767           case AnyReg::F64:
1768             pushF64(r.f64());
1769             break;
1770           case AnyReg::F32:
1771             pushF32(r.f32());
1772             break;
1773         }
1774     }
1775 
freeJoinReg(AnyReg r)1776     void freeJoinReg(AnyReg r) {
1777         switch (r.tag) {
1778           case AnyReg::NONE:
1779             MOZ_CRASH("Compile bug: attempting to free void reg");
1780             break;
1781           case AnyReg::I32:
1782             freeI32(r.i32());
1783             break;
1784           case AnyReg::I64:
1785             freeI64(r.i64());
1786             break;
1787           case AnyReg::F64:
1788             freeF64(r.f64());
1789             break;
1790           case AnyReg::F32:
1791             freeF32(r.f32());
1792             break;
1793         }
1794     }
1795 
maybeReserveJoinRegI(ExprType type)1796     void maybeReserveJoinRegI(ExprType type) {
1797         if (type == ExprType::I32)
1798             needI32(joinRegI32);
1799         else if (type == ExprType::I64)
1800             needI64(joinRegI64);
1801     }
1802 
maybeUnreserveJoinRegI(ExprType type)1803     void maybeUnreserveJoinRegI(ExprType type) {
1804         if (type == ExprType::I32)
1805             freeI32(joinRegI32);
1806         else if (type == ExprType::I64)
1807             freeI64(joinRegI64);
1808     }
1809 
1810     // Return the amount of execution stack consumed by the top numval
1811     // values on the value stack.
1812 
stackConsumed(size_t numval)1813     size_t stackConsumed(size_t numval) {
1814         size_t size = 0;
1815         MOZ_ASSERT(numval <= stk_.length());
1816         for (uint32_t i = stk_.length() - 1; numval > 0; numval--, i--) {
1817             // The size computations come from the implementation of Push() in
1818             // MacroAssembler-x86-shared.cpp and MacroAssembler-arm-shared.cpp,
1819             // and from VFPRegister::size() in Architecture-arm.h.
1820             //
1821             // On ARM unlike on x86 we push a single for float.
1822 
1823             Stk& v = stk_[i];
1824             switch (v.kind()) {
1825               case Stk::MemI32:
1826 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
1827                 size += sizeof(intptr_t);
1828 #else
1829                 MOZ_CRASH("BaseCompiler platform hook: stackConsumed I32");
1830 #endif
1831                 break;
1832               case Stk::MemI64:
1833 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
1834                 size += sizeof(int64_t);
1835 #else
1836                 MOZ_CRASH("BaseCompiler platform hook: stackConsumed I64");
1837 #endif
1838                 break;
1839               case Stk::MemF64:
1840 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
1841                 size += sizeof(double);
1842 #else
1843                 MOZ_CRASH("BaseCompiler platform hook: stackConsumed F64");
1844 #endif
1845                 break;
1846               case Stk::MemF32:
1847 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
1848                 size += sizeof(double);
1849 #elif defined(JS_CODEGEN_ARM)
1850                 size += sizeof(float);
1851 #else
1852                 MOZ_CRASH("BaseCompiler platform hook: stackConsumed F32");
1853 #endif
1854                 break;
1855               default:
1856                 break;
1857             }
1858         }
1859         return size;
1860     }
1861 
popValueStackTo(uint32_t stackSize)1862     void popValueStackTo(uint32_t stackSize) {
1863         for (uint32_t i = stk_.length(); i > stackSize; i--) {
1864             Stk& v = stk_[i-1];
1865             switch (v.kind()) {
1866               case Stk::RegisterI32:
1867                 freeI32(v.i32reg());
1868                 break;
1869               case Stk::RegisterI64:
1870                 freeI64(v.i64reg());
1871                 break;
1872               case Stk::RegisterF64:
1873                 freeF64(v.f64reg());
1874                 break;
1875               case Stk::RegisterF32:
1876                 freeF32(v.f32reg());
1877                 break;
1878               default:
1879                 break;
1880             }
1881         }
1882         stk_.shrinkTo(stackSize);
1883     }
1884 
popValueStackBy(uint32_t items)1885     void popValueStackBy(uint32_t items) {
1886         popValueStackTo(stk_.length() - items);
1887     }
1888 
1889     // Before branching to an outer control label, pop the execution
1890     // stack to the level expected by that region, but do not free the
1891     // stack as that will happen as compilation leaves the block.
1892 
popStackBeforeBranch(uint32_t framePushed)1893     void popStackBeforeBranch(uint32_t framePushed) {
1894         uint32_t frameHere = masm.framePushed();
1895         if (frameHere > framePushed)
1896             masm.addPtr(ImmWord(frameHere - framePushed), StackPointer);
1897     }
1898 
1899     // Before exiting a nested control region, pop the execution stack
1900     // to the level expected by the nesting region, and free the
1901     // stack.
1902 
popStackOnBlockExit(uint32_t framePushed)1903     void popStackOnBlockExit(uint32_t framePushed) {
1904         uint32_t frameHere = masm.framePushed();
1905         if (frameHere > framePushed) {
1906             if (deadCode_)
1907                 masm.adjustStack(frameHere - framePushed);
1908             else
1909                 masm.freeStack(frameHere - framePushed);
1910         }
1911     }
1912 
popStackIfMemory()1913     void popStackIfMemory() {
1914         if (peek(0).isMem())
1915             masm.freeStack(stackConsumed(1));
1916     }
1917 
1918     // Peek at the stack, for calls.
1919 
peek(uint32_t relativeDepth)1920     Stk& peek(uint32_t relativeDepth) {
1921         return stk_[stk_.length()-1-relativeDepth];
1922     }
1923 
1924     ////////////////////////////////////////////////////////////
1925     //
1926     // Control stack
1927 
1928     Vector<Control, 8, SystemAllocPolicy> ctl_;
1929 
pushControl(UniquePooledLabel * label,UniquePooledLabel * otherLabel=nullptr)1930     MOZ_MUST_USE bool pushControl(UniquePooledLabel* label, UniquePooledLabel* otherLabel = nullptr)
1931     {
1932         uint32_t framePushed = masm.framePushed();
1933         uint32_t stackSize = stk_.length();
1934 
1935         if (!ctl_.emplaceBack(Control(framePushed, stackSize)))
1936             return false;
1937         if (label)
1938             ctl_.back().label = label->release();
1939         if (otherLabel)
1940             ctl_.back().otherLabel = otherLabel->release();
1941         ctl_.back().deadOnArrival = deadCode_;
1942         return true;
1943     }
1944 
popControl()1945     void popControl() {
1946         Control last = ctl_.popCopy();
1947         if (last.label)
1948             freeLabel(last.label);
1949         if (last.otherLabel)
1950             freeLabel(last.otherLabel);
1951 
1952         if (deadCode_ && !ctl_.empty())
1953             popValueStackTo(ctl_.back().stackSize);
1954     }
1955 
controlItem(uint32_t relativeDepth)1956     Control& controlItem(uint32_t relativeDepth) {
1957         return ctl_[ctl_.length() - 1 - relativeDepth];
1958     }
1959 
newLabel()1960     MOZ_MUST_USE PooledLabel* newLabel() {
1961         // TODO / INVESTIGATE (Bug 1316819): allocate() is fallible, but we can
1962         // probably rely on an infallible allocator here.  That would simplify
1963         // code later.
1964         PooledLabel* candidate = labelPool_.allocate();
1965         if (!candidate)
1966             return nullptr;
1967         return new (candidate) PooledLabel(this);
1968     }
1969 
freeLabel(PooledLabel * label)1970     void freeLabel(PooledLabel* label) {
1971         label->~PooledLabel();
1972         labelPool_.free(label);
1973     }
1974 
1975     //////////////////////////////////////////////////////////////////////
1976     //
1977     // Function prologue and epilogue.
1978 
beginFunction()1979     void beginFunction() {
1980         JitSpew(JitSpew_Codegen, "# Emitting wasm baseline code");
1981 
1982         SigIdDesc sigId = mg_.funcSigs[func_.index()]->id;
1983         GenerateFunctionPrologue(masm, localSize_, sigId, &compileResults_.offsets());
1984 
1985         MOZ_ASSERT(masm.framePushed() == uint32_t(localSize_));
1986 
1987         maxFramePushed_ = localSize_;
1988 
1989         // We won't know until after we've generated code how big the
1990         // frame will be (we may need arbitrary spill slots and
1991         // outgoing param slots) so branch to code emitted after the
1992         // function body that will perform the check.
1993         //
1994         // Code there will also assume that the fixed-size stack frame
1995         // has been allocated.
1996 
1997         masm.jump(&outOfLinePrologue_);
1998         masm.bind(&bodyLabel_);
1999 
2000         // Copy arguments from registers to stack.
2001 
2002         const ValTypeVector& args = func_.sig().args();
2003 
2004         for (ABIArgIter<const ValTypeVector> i(args); !i.done(); i++) {
2005             Local& l = localInfo_[i.index()];
2006             switch (i.mirType()) {
2007               case MIRType::Int32:
2008                 if (i->argInRegister())
2009                     storeToFrameI32(i->gpr(), l.offs());
2010                 break;
2011               case MIRType::Int64:
2012                 if (i->argInRegister())
2013                     storeToFrameI64(i->gpr64(), l.offs());
2014                 break;
2015               case MIRType::Double:
2016                 if (i->argInRegister())
2017                     storeToFrameF64(i->fpu(), l.offs());
2018                 break;
2019               case MIRType::Float32:
2020                 if (i->argInRegister())
2021                     storeToFrameF32(i->fpu(), l.offs());
2022                 break;
2023               default:
2024                 MOZ_CRASH("Function argument type");
2025             }
2026         }
2027 
2028         // The TLS pointer is always passed as a hidden argument in WasmTlsReg.
2029         // Save it into its assigned local slot.
2030         storeToFramePtr(WasmTlsReg, localInfo_[tlsSlot_].offs());
2031 
2032         // Initialize the stack locals to zero.
2033         //
2034         // The following are all Bug 1316820:
2035         //
2036         // TODO / OPTIMIZE: on x64, at least, scratch will be a 64-bit
2037         // register and we can move 64 bits at a time.
2038         //
2039         // TODO / OPTIMIZE: On SSE2 or better SIMD systems we may be
2040         // able to store 128 bits at a time.  (I suppose on some
2041         // systems we have 512-bit SIMD for that matter.)
2042         //
2043         // TODO / OPTIMIZE: if we have only one initializing store
2044         // then it's better to store a zero literal, probably.
2045 
2046         if (varLow_ < varHigh_) {
2047             ScratchI32 scratch(*this);
2048             masm.mov(ImmWord(0), scratch);
2049             for (int32_t i = varLow_ ; i < varHigh_ ; i += 4)
2050                 storeToFrameI32(scratch, i + 4);
2051         }
2052     }
2053 
endFunction()2054     bool endFunction() {
2055         // Out-of-line prologue.  Assumes that the in-line prologue has
2056         // been executed and that a frame of size = localSize_ + sizeof(Frame)
2057         // has been allocated.
2058 
2059         masm.bind(&outOfLinePrologue_);
2060 
2061         MOZ_ASSERT(maxFramePushed_ >= localSize_);
2062 
2063         // ABINonArgReg0 != ScratchReg, which can be used by branchPtr().
2064 
2065         masm.movePtr(masm.getStackPointer(), ABINonArgReg0);
2066         if (maxFramePushed_ - localSize_)
2067             masm.subPtr(Imm32(maxFramePushed_ - localSize_), ABINonArgReg0);
2068         masm.branchPtr(Assembler::Below,
2069                        Address(WasmTlsReg, offsetof(TlsData, stackLimit)),
2070                        ABINonArgReg0,
2071                        &bodyLabel_);
2072 
2073         // Since we just overflowed the stack, to be on the safe side, pop the
2074         // stack so that, when the trap exit stub executes, it is a safe
2075         // distance away from the end of the native stack.
2076         if (localSize_)
2077             masm.addToStackPtr(Imm32(localSize_));
2078         masm.jump(TrapDesc(prologueTrapOffset_, Trap::StackOverflow, /* framePushed = */ 0));
2079 
2080         masm.bind(&returnLabel_);
2081 
2082         // Restore the TLS register in case it was overwritten by the function.
2083         loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
2084 
2085         GenerateFunctionEpilogue(masm, localSize_, &compileResults_.offsets());
2086 
2087 #if defined(JS_ION_PERF)
2088         // FIXME - profiling code missing.  Bug 1286948.
2089 
2090         // Note the end of the inline code and start of the OOL code.
2091         //gen->perfSpewer().noteEndInlineCode(masm);
2092 #endif
2093 
2094         if (!generateOutOfLineCode())
2095             return false;
2096 
2097         masm.wasmEmitTrapOutOfLineCode();
2098 
2099         compileResults_.offsets().end = masm.currentOffset();
2100 
2101         // A frame greater than 256KB is implausible, probably an attack,
2102         // so fail the compilation.
2103 
2104         if (maxFramePushed_ > 256 * 1024)
2105             return false;
2106 
2107         return true;
2108     }
2109 
2110     //////////////////////////////////////////////////////////////////////
2111     //
2112     // Calls.
2113 
2114     struct FunctionCall
2115     {
FunctionCalljs::wasm::BaseCompiler::FunctionCall2116         explicit FunctionCall(uint32_t lineOrBytecode)
2117           : lineOrBytecode(lineOrBytecode),
2118             reloadMachineStateAfter(false),
2119             usesSystemAbi(false),
2120             loadTlsBefore(false),
2121 #ifdef JS_CODEGEN_ARM
2122             hardFP(true),
2123 #endif
2124             frameAlignAdjustment(0),
2125             stackArgAreaSize(0)
2126         {}
2127 
2128         uint32_t lineOrBytecode;
2129         ABIArgGenerator abi;
2130         bool reloadMachineStateAfter;
2131         bool usesSystemAbi;
2132         bool loadTlsBefore;
2133 #ifdef JS_CODEGEN_ARM
2134         bool hardFP;
2135 #endif
2136         size_t frameAlignAdjustment;
2137         size_t stackArgAreaSize;
2138     };
2139 
beginCall(FunctionCall & call,UseABI useABI,InterModule interModule)2140     void beginCall(FunctionCall& call, UseABI useABI, InterModule interModule)
2141     {
2142         call.reloadMachineStateAfter = interModule == InterModule::True || useABI == UseABI::System;
2143         call.usesSystemAbi = useABI == UseABI::System;
2144         call.loadTlsBefore = useABI == UseABI::Wasm;
2145 
2146         if (call.usesSystemAbi) {
2147             // Call-outs need to use the appropriate system ABI.
2148 #if defined(JS_CODEGEN_ARM)
2149 # if defined(JS_SIMULATOR_ARM)
2150             call.hardFP = UseHardFpABI();
2151 # elif defined(JS_CODEGEN_ARM_HARDFP)
2152             call.hardFP = true;
2153 # else
2154             call.hardFP = false;
2155 # endif
2156             call.abi.setUseHardFp(call.hardFP);
2157 #endif
2158         }
2159 
2160         call.frameAlignAdjustment = ComputeByteAlignment(masm.framePushed() + sizeof(Frame),
2161                                                          JitStackAlignment);
2162     }
2163 
endCall(FunctionCall & call)2164     void endCall(FunctionCall& call)
2165     {
2166         size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
2167         if (adjustment)
2168             masm.freeStack(adjustment);
2169 
2170         if (call.reloadMachineStateAfter) {
2171             loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
2172             masm.loadWasmPinnedRegsFromTls();
2173         }
2174     }
2175 
2176     // TODO / OPTIMIZE (Bug 1316820): This is expensive; let's roll the iterator
2177     // walking into the walking done for passArg.  See comments in passArg.
2178 
stackArgAreaSize(const ValTypeVector & args)2179     size_t stackArgAreaSize(const ValTypeVector& args) {
2180         ABIArgIter<const ValTypeVector> i(args);
2181         while (!i.done())
2182             i++;
2183         return AlignBytes(i.stackBytesConsumedSoFar(), 16u);
2184     }
2185 
startCallArgs(FunctionCall & call,size_t stackArgAreaSize)2186     void startCallArgs(FunctionCall& call, size_t stackArgAreaSize)
2187     {
2188         call.stackArgAreaSize = stackArgAreaSize;
2189 
2190         size_t adjustment = call.stackArgAreaSize + call.frameAlignAdjustment;
2191         if (adjustment)
2192             masm.reserveStack(adjustment);
2193     }
2194 
reservePointerArgument(FunctionCall & call)2195     const ABIArg reservePointerArgument(FunctionCall& call) {
2196         return call.abi.next(MIRType::Pointer);
2197     }
2198 
2199     // TODO / OPTIMIZE (Bug 1316820): Note passArg is used only in one place.
2200     // (Or it was, until Luke wandered through, but that can be fixed again.)
2201     // I'm not saying we should manually inline it, but we could hoist the
2202     // dispatch into the caller and have type-specific implementations of
2203     // passArg: passArgI32(), etc.  Then those might be inlined, at least in PGO
2204     // builds.
2205     //
2206     // The bulk of the work here (60%) is in the next() call, though.
2207     //
2208     // Notably, since next() is so expensive, stackArgAreaSize() becomes
2209     // expensive too.
2210     //
2211     // Somehow there could be a trick here where the sequence of
2212     // argument types (read from the input stream) leads to a cached
2213     // entry for stackArgAreaSize() and for how to pass arguments...
2214     //
2215     // But at least we could reduce the cost of stackArgAreaSize() by
2216     // first reading the argument types into a (reusable) vector, then
2217     // we have the outgoing size at low cost, and then we can pass
2218     // args based on the info we read.
2219 
passArg(FunctionCall & call,ValType type,Stk & arg)2220     void passArg(FunctionCall& call, ValType type, Stk& arg) {
2221         switch (type) {
2222           case ValType::I32: {
2223             ABIArg argLoc = call.abi.next(MIRType::Int32);
2224             if (argLoc.kind() == ABIArg::Stack) {
2225                 ScratchI32 scratch(*this);
2226                 loadI32(scratch, arg);
2227                 masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
2228             } else {
2229                 loadI32(argLoc.gpr(), arg);
2230             }
2231             break;
2232           }
2233           case ValType::I64: {
2234             ABIArg argLoc = call.abi.next(MIRType::Int64);
2235             if (argLoc.kind() == ABIArg::Stack) {
2236                 ScratchI32 scratch(*this);
2237 #if defined(JS_CODEGEN_X64)
2238                 loadI64(Register64(scratch), arg);
2239                 masm.movq(scratch, Operand(StackPointer, argLoc.offsetFromArgBase()));
2240 #elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
2241                 loadI64Low(scratch, arg);
2242                 masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase() + INT64LOW_OFFSET));
2243                 loadI64High(scratch, arg);
2244                 masm.store32(scratch, Address(StackPointer, argLoc.offsetFromArgBase() + INT64HIGH_OFFSET));
2245 #else
2246                 MOZ_CRASH("BaseCompiler platform hook: passArg I64");
2247 #endif
2248             } else {
2249                 loadI64(argLoc.gpr64(), arg);
2250             }
2251             break;
2252           }
2253           case ValType::F64: {
2254             ABIArg argLoc = call.abi.next(MIRType::Double);
2255             switch (argLoc.kind()) {
2256               case ABIArg::Stack: {
2257                 ScratchF64 scratch(*this);
2258                 loadF64(scratch, arg);
2259                 masm.storeDouble(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
2260                 break;
2261               }
2262 #if defined(JS_CODEGEN_REGISTER_PAIR)
2263               case ABIArg::GPR_PAIR: {
2264 # ifdef JS_CODEGEN_ARM
2265                 ScratchF64 scratch(*this);
2266                 loadF64(scratch, arg);
2267                 masm.ma_vxfer(scratch, argLoc.evenGpr(), argLoc.oddGpr());
2268                 break;
2269 # else
2270                 MOZ_CRASH("BaseCompiler platform hook: passArg F64 pair");
2271 # endif
2272               }
2273 #endif
2274               case ABIArg::FPU: {
2275                 loadF64(argLoc.fpu(), arg);
2276                 break;
2277               }
2278               case ABIArg::GPR: {
2279                 MOZ_CRASH("Unexpected parameter passing discipline");
2280               }
2281             }
2282             break;
2283           }
2284           case ValType::F32: {
2285             ABIArg argLoc = call.abi.next(MIRType::Float32);
2286             switch (argLoc.kind()) {
2287               case ABIArg::Stack: {
2288                 ScratchF32 scratch(*this);
2289                 loadF32(scratch, arg);
2290                 masm.storeFloat32(scratch, Address(StackPointer, argLoc.offsetFromArgBase()));
2291                 break;
2292               }
2293               case ABIArg::GPR: {
2294                 ScratchF32 scratch(*this);
2295                 loadF32(scratch, arg);
2296                 masm.moveFloat32ToGPR(scratch, argLoc.gpr());
2297                 break;
2298               }
2299               case ABIArg::FPU: {
2300                 loadF32(argLoc.fpu(), arg);
2301                 break;
2302               }
2303 #if defined(JS_CODEGEN_REGISTER_PAIR)
2304               case ABIArg::GPR_PAIR: {
2305                 MOZ_CRASH("Unexpected parameter passing discipline");
2306               }
2307 #endif
2308             }
2309             break;
2310           }
2311           default:
2312             MOZ_CRASH("Function argument type");
2313         }
2314     }
2315 
callDefinition(uint32_t funcIndex,const FunctionCall & call)2316     void callDefinition(uint32_t funcIndex, const FunctionCall& call)
2317     {
2318         CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Func);
2319         masm.call(desc, funcIndex);
2320     }
2321 
callSymbolic(SymbolicAddress callee,const FunctionCall & call)2322     void callSymbolic(SymbolicAddress callee, const FunctionCall& call) {
2323         CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Symbolic);
2324         masm.call(callee);
2325     }
2326 
2327     // Precondition: sync()
2328 
callIndirect(uint32_t sigIndex,Stk & indexVal,const FunctionCall & call)2329     void callIndirect(uint32_t sigIndex, Stk& indexVal, const FunctionCall& call)
2330     {
2331         loadI32(WasmTableCallIndexReg, indexVal);
2332 
2333         const SigWithId& sig = mg_.sigs[sigIndex];
2334 
2335         CalleeDesc callee;
2336         if (isCompilingAsmJS()) {
2337             MOZ_ASSERT(sig.id.kind() == SigIdDesc::Kind::None);
2338             const TableDesc& table = mg_.tables[mg_.asmJSSigToTableIndex[sigIndex]];
2339 
2340             MOZ_ASSERT(IsPowerOfTwo(table.limits.initial));
2341             masm.andPtr(Imm32((table.limits.initial - 1)), WasmTableCallIndexReg);
2342 
2343             callee = CalleeDesc::asmJSTable(table);
2344         } else {
2345             MOZ_ASSERT(sig.id.kind() != SigIdDesc::Kind::None);
2346             MOZ_ASSERT(mg_.tables.length() == 1);
2347             const TableDesc& table = mg_.tables[0];
2348 
2349             callee = CalleeDesc::wasmTable(table, sig.id);
2350         }
2351 
2352         CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
2353         masm.wasmCallIndirect(desc, callee);
2354     }
2355 
2356     // Precondition: sync()
2357 
callImport(unsigned globalDataOffset,const FunctionCall & call)2358     void callImport(unsigned globalDataOffset, const FunctionCall& call)
2359     {
2360         CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Dynamic);
2361         CalleeDesc callee = CalleeDesc::import(globalDataOffset);
2362         masm.wasmCallImport(desc, callee);
2363     }
2364 
builtinCall(SymbolicAddress builtin,const FunctionCall & call)2365     void builtinCall(SymbolicAddress builtin, const FunctionCall& call)
2366     {
2367         callSymbolic(builtin, call);
2368     }
2369 
builtinInstanceMethodCall(SymbolicAddress builtin,const ABIArg & instanceArg,const FunctionCall & call)2370     void builtinInstanceMethodCall(SymbolicAddress builtin, const ABIArg& instanceArg,
2371                                    const FunctionCall& call)
2372     {
2373         // Builtin method calls assume the TLS register has been set.
2374         loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
2375 
2376         CallSiteDesc desc(call.lineOrBytecode, CallSiteDesc::Symbolic);
2377         masm.wasmCallBuiltinInstanceMethod(instanceArg, builtin);
2378     }
2379 
2380     //////////////////////////////////////////////////////////////////////
2381     //
2382     // Sundry low-level code generators.
2383 
addInterruptCheck()2384     void addInterruptCheck()
2385     {
2386         // Always use signals for interrupts with Asm.JS/Wasm
2387         MOZ_RELEASE_ASSERT(HaveSignalHandlers());
2388     }
2389 
jumpTable(LabelVector & labels)2390     void jumpTable(LabelVector& labels) {
2391 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
2392         for (uint32_t i = 0; i < labels.length(); i++) {
2393             CodeLabel cl;
2394             masm.writeCodePointer(cl.patchAt());
2395             cl.target()->bind(labels[i]->offset());
2396             masm.addCodeLabel(cl);
2397         }
2398 #else
2399         MOZ_CRASH("BaseCompiler platform hook: jumpTable");
2400 #endif
2401     }
2402 
tableSwitch(Label * theTable,RegI32 switchValue)2403     void tableSwitch(Label* theTable, RegI32 switchValue) {
2404 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
2405         ScratchI32 scratch(*this);
2406         CodeLabel tableCl;
2407 
2408         masm.mov(tableCl.patchAt(), scratch);
2409 
2410         tableCl.target()->bind(theTable->offset());
2411         masm.addCodeLabel(tableCl);
2412 
2413         masm.jmp(Operand(scratch, switchValue.reg, ScalePointer));
2414 #elif defined(JS_CODEGEN_ARM)
2415         ScratchI32 scratch(*this);
2416 
2417         // Compute the offset from the next instruction to the jump table
2418         Label here;
2419         masm.bind(&here);
2420         uint32_t offset = here.offset() - theTable->offset();
2421 
2422         // Read PC+8
2423         masm.ma_mov(pc, scratch);
2424 
2425         // Required by ma_sub.
2426         ScratchRegisterScope arm_scratch(*this);
2427 
2428         // Compute the table base pointer
2429         masm.ma_sub(Imm32(offset + 8), scratch, arm_scratch);
2430 
2431         // Jump indirect via table element
2432         masm.ma_ldr(DTRAddr(scratch, DtrRegImmShift(switchValue.reg, LSL, 2)), pc, Offset,
2433                     Assembler::Always);
2434 #else
2435         MOZ_CRASH("BaseCompiler platform hook: tableSwitch");
2436 #endif
2437     }
2438 
captureReturnedI32()2439     RegI32 captureReturnedI32() {
2440         RegI32 rv = RegI32(ReturnReg);
2441         MOZ_ASSERT(isAvailable(rv.reg));
2442         needI32(rv);
2443         return rv;
2444     }
2445 
captureReturnedI64()2446     RegI64 captureReturnedI64() {
2447         RegI64 rv = RegI64(ReturnReg64);
2448         MOZ_ASSERT(isAvailable(rv.reg));
2449         needI64(rv);
2450         return rv;
2451     }
2452 
captureReturnedF32(const FunctionCall & call)2453     RegF32 captureReturnedF32(const FunctionCall& call) {
2454         RegF32 rv = RegF32(ReturnFloat32Reg);
2455         MOZ_ASSERT(isAvailable(rv.reg));
2456         needF32(rv);
2457 #if defined(JS_CODEGEN_X86)
2458         if (call.usesSystemAbi) {
2459             masm.reserveStack(sizeof(float));
2460             Operand op(esp, 0);
2461             masm.fstp32(op);
2462             masm.loadFloat32(op, rv.reg);
2463             masm.freeStack(sizeof(float));
2464         }
2465 #elif defined(JS_CODEGEN_ARM)
2466         if (call.usesSystemAbi && !call.hardFP)
2467             masm.ma_vxfer(r0, rv.reg);
2468 #endif
2469         return rv;
2470     }
2471 
captureReturnedF64(const FunctionCall & call)2472     RegF64 captureReturnedF64(const FunctionCall& call) {
2473         RegF64 rv = RegF64(ReturnDoubleReg);
2474         MOZ_ASSERT(isAvailable(rv.reg));
2475         needF64(rv);
2476 #if defined(JS_CODEGEN_X86)
2477         if (call.usesSystemAbi) {
2478             masm.reserveStack(sizeof(double));
2479             Operand op(esp, 0);
2480             masm.fstp(op);
2481             masm.loadDouble(op, rv.reg);
2482             masm.freeStack(sizeof(double));
2483         }
2484 #elif defined(JS_CODEGEN_ARM)
2485         if (call.usesSystemAbi && !call.hardFP)
2486             masm.ma_vxfer(r0, r1, rv.reg);
2487 #endif
2488         return rv;
2489     }
2490 
returnCleanup()2491     void returnCleanup() {
2492         popStackBeforeBranch(ctl_[0].framePushed);
2493         masm.jump(&returnLabel_);
2494     }
2495 
pop2xI32ForIntMulDiv(RegI32 * r0,RegI32 * r1)2496     void pop2xI32ForIntMulDiv(RegI32* r0, RegI32* r1) {
2497 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2498         // srcDest must be eax, and edx will be clobbered.
2499         need2xI32(specific_eax, specific_edx);
2500         *r1 = popI32();
2501         *r0 = popI32ToSpecific(specific_eax);
2502         freeI32(specific_edx);
2503 #else
2504         pop2xI32(r0, r1);
2505 #endif
2506     }
2507 
pop2xI64ForIntDiv(RegI64 * r0,RegI64 * r1)2508     void pop2xI64ForIntDiv(RegI64* r0, RegI64* r1) {
2509 #ifdef JS_CODEGEN_X64
2510         // srcDest must be rax, and rdx will be clobbered.
2511         need2xI64(specific_rax, specific_rdx);
2512         *r1 = popI64();
2513         *r0 = popI64ToSpecific(specific_rax);
2514         freeI64(specific_rdx);
2515 #else
2516         pop2xI64(r0, r1);
2517 #endif
2518     }
2519 
checkDivideByZeroI32(RegI32 rhs,RegI32 srcDest,Label * done)2520     void checkDivideByZeroI32(RegI32 rhs, RegI32 srcDest, Label* done) {
2521         if (isCompilingAsmJS()) {
2522             // Truncated division by zero is zero (Infinity|0 == 0)
2523             Label notDivByZero;
2524             masm.branchTest32(Assembler::NonZero, rhs.reg, rhs.reg, &notDivByZero);
2525             masm.move32(Imm32(0), srcDest.reg);
2526             masm.jump(done);
2527             masm.bind(&notDivByZero);
2528         } else {
2529             masm.branchTest32(Assembler::Zero, rhs.reg, rhs.reg, trap(Trap::IntegerDivideByZero));
2530         }
2531     }
2532 
checkDivideByZeroI64(RegI64 r)2533     void checkDivideByZeroI64(RegI64 r) {
2534         MOZ_ASSERT(!isCompilingAsmJS());
2535         ScratchI32 scratch(*this);
2536         masm.branchTest64(Assembler::Zero, r.reg, r.reg, scratch, trap(Trap::IntegerDivideByZero));
2537     }
2538 
checkDivideSignedOverflowI32(RegI32 rhs,RegI32 srcDest,Label * done,bool zeroOnOverflow)2539     void checkDivideSignedOverflowI32(RegI32 rhs, RegI32 srcDest, Label* done, bool zeroOnOverflow) {
2540         Label notMin;
2541         masm.branch32(Assembler::NotEqual, srcDest.reg, Imm32(INT32_MIN), &notMin);
2542         if (zeroOnOverflow) {
2543             masm.branch32(Assembler::NotEqual, rhs.reg, Imm32(-1), &notMin);
2544             masm.move32(Imm32(0), srcDest.reg);
2545             masm.jump(done);
2546         } else if (isCompilingAsmJS()) {
2547             // (-INT32_MIN)|0 == INT32_MIN and INT32_MIN is already in srcDest.
2548             masm.branch32(Assembler::Equal, rhs.reg, Imm32(-1), done);
2549         } else {
2550             masm.branch32(Assembler::Equal, rhs.reg, Imm32(-1), trap(Trap::IntegerOverflow));
2551         }
2552         masm.bind(&notMin);
2553     }
2554 
checkDivideSignedOverflowI64(RegI64 rhs,RegI64 srcDest,Label * done,bool zeroOnOverflow)2555     void checkDivideSignedOverflowI64(RegI64 rhs, RegI64 srcDest, Label* done, bool zeroOnOverflow) {
2556         MOZ_ASSERT(!isCompilingAsmJS());
2557         Label notmin;
2558         masm.branch64(Assembler::NotEqual, srcDest.reg, Imm64(INT64_MIN), &notmin);
2559         masm.branch64(Assembler::NotEqual, rhs.reg, Imm64(-1), &notmin);
2560         if (zeroOnOverflow) {
2561             masm.xor64(srcDest.reg, srcDest.reg);
2562             masm.jump(done);
2563         } else {
2564             masm.jump(trap(Trap::IntegerOverflow));
2565         }
2566         masm.bind(&notmin);
2567     }
2568 
2569 #ifndef INT_DIV_I64_CALLOUT
quotientI64(RegI64 rhs,RegI64 srcDest,IsUnsigned isUnsigned)2570     void quotientI64(RegI64 rhs, RegI64 srcDest, IsUnsigned isUnsigned) {
2571         Label done;
2572 
2573         checkDivideByZeroI64(rhs);
2574 
2575         if (!isUnsigned)
2576             checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(false));
2577 
2578 # if defined(JS_CODEGEN_X64)
2579         // The caller must set up the following situation.
2580         MOZ_ASSERT(srcDest.reg.reg == rax);
2581         MOZ_ASSERT(isAvailable(rdx));
2582         if (isUnsigned) {
2583             masm.xorq(rdx, rdx);
2584             masm.udivq(rhs.reg.reg);
2585         } else {
2586             masm.cqo();
2587             masm.idivq(rhs.reg.reg);
2588         }
2589 # else
2590         MOZ_CRASH("BaseCompiler platform hook: quotientI64");
2591 # endif
2592         masm.bind(&done);
2593     }
2594 
remainderI64(RegI64 rhs,RegI64 srcDest,IsUnsigned isUnsigned)2595     void remainderI64(RegI64 rhs, RegI64 srcDest, IsUnsigned isUnsigned) {
2596         Label done;
2597 
2598         checkDivideByZeroI64(rhs);
2599 
2600         if (!isUnsigned)
2601             checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(true));
2602 
2603 # if defined(JS_CODEGEN_X64)
2604         // The caller must set up the following situation.
2605         MOZ_ASSERT(srcDest.reg.reg == rax);
2606         MOZ_ASSERT(isAvailable(rdx));
2607 
2608         if (isUnsigned) {
2609             masm.xorq(rdx, rdx);
2610             masm.udivq(rhs.reg.reg);
2611         } else {
2612             masm.cqo();
2613             masm.idivq(rhs.reg.reg);
2614         }
2615         masm.movq(rdx, rax);
2616 # else
2617         MOZ_CRASH("BaseCompiler platform hook: remainderI64");
2618 # endif
2619         masm.bind(&done);
2620     }
2621 #endif // INT_DIV_I64_CALLOUT
2622 
pop2xI32ForShiftOrRotate(RegI32 * r0,RegI32 * r1)2623     void pop2xI32ForShiftOrRotate(RegI32* r0, RegI32* r1) {
2624 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2625         *r1 = popI32(specific_ecx);
2626         *r0 = popI32();
2627 #else
2628         pop2xI32(r0, r1);
2629 #endif
2630     }
2631 
pop2xI64ForShiftOrRotate(RegI64 * r0,RegI64 * r1)2632     void pop2xI64ForShiftOrRotate(RegI64* r0, RegI64* r1) {
2633 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2634         needI32(specific_ecx);
2635         *r1 = widenI32(specific_ecx);
2636         *r1 = popI64ToSpecific(*r1);
2637         *r0 = popI64();
2638 #else
2639         pop2xI64(r0, r1);
2640 #endif
2641     }
2642 
maskShiftCount32(RegI32 r)2643     void maskShiftCount32(RegI32 r) {
2644 #if defined(JS_CODEGEN_ARM)
2645         masm.and32(Imm32(31), r.reg);
2646 #endif
2647     }
2648 
popcnt32NeedsTemp() const2649     bool popcnt32NeedsTemp() const {
2650 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2651         return !AssemblerX86Shared::HasPOPCNT();
2652 #elif defined(JS_CODEGEN_ARM)
2653         return true;
2654 #else
2655         MOZ_CRASH("BaseCompiler platform hook: popcnt32NeedsTemp");
2656 #endif
2657     }
2658 
popcnt64NeedsTemp() const2659     bool popcnt64NeedsTemp() const {
2660 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2661         return !AssemblerX86Shared::HasPOPCNT();
2662 #elif defined(JS_CODEGEN_ARM)
2663         return true;
2664 #else
2665         MOZ_CRASH("BaseCompiler platform hook: popcnt64NeedsTemp");
2666 #endif
2667     }
2668 
reinterpretI64AsF64(RegI64 src,RegF64 dest)2669     void reinterpretI64AsF64(RegI64 src, RegF64 dest) {
2670 #if defined(JS_CODEGEN_X64)
2671         masm.vmovq(src.reg.reg, dest.reg);
2672 #elif defined(JS_CODEGEN_X86)
2673         masm.Push(src.reg.high);
2674         masm.Push(src.reg.low);
2675         masm.vmovq(Operand(esp, 0), dest.reg);
2676         masm.freeStack(sizeof(uint64_t));
2677 #elif defined(JS_CODEGEN_ARM)
2678         masm.ma_vxfer(src.reg.low, src.reg.high, dest.reg);
2679 #else
2680         MOZ_CRASH("BaseCompiler platform hook: reinterpretI64AsF64");
2681 #endif
2682     }
2683 
reinterpretF64AsI64(RegF64 src,RegI64 dest)2684     void reinterpretF64AsI64(RegF64 src, RegI64 dest) {
2685 #if defined(JS_CODEGEN_X64)
2686         masm.vmovq(src.reg, dest.reg.reg);
2687 #elif defined(JS_CODEGEN_X86)
2688         masm.reserveStack(sizeof(uint64_t));
2689         masm.vmovq(src.reg, Operand(esp, 0));
2690         masm.Pop(dest.reg.low);
2691         masm.Pop(dest.reg.high);
2692 #elif defined(JS_CODEGEN_ARM)
2693         masm.ma_vxfer(src.reg, dest.reg.low, dest.reg.high);
2694 #else
2695         MOZ_CRASH("BaseCompiler platform hook: reinterpretF64AsI64");
2696 #endif
2697     }
2698 
wrapI64ToI32(RegI64 src,RegI32 dest)2699     void wrapI64ToI32(RegI64 src, RegI32 dest) {
2700 #if defined(JS_CODEGEN_X64)
2701         // movl clears the high bits if the two registers are the same.
2702         masm.movl(src.reg.reg, dest.reg);
2703 #elif defined(JS_NUNBOX32)
2704         masm.move32(src.reg.low, dest.reg);
2705 #else
2706         MOZ_CRASH("BaseCompiler platform hook: wrapI64ToI32");
2707 #endif
2708     }
2709 
popI32ForSignExtendI64()2710     RegI64 popI32ForSignExtendI64() {
2711 #if defined(JS_CODEGEN_X86)
2712         need2xI32(specific_edx, specific_eax);
2713         RegI32 r0 = popI32ToSpecific(specific_eax);
2714         RegI64 x0 = RegI64(Register64(specific_edx.reg, specific_eax.reg));
2715         (void)r0;               // x0 is the widening of r0
2716 #else
2717         RegI32 r0 = popI32();
2718         RegI64 x0 = widenI32(r0);
2719 #endif
2720         return x0;
2721     }
2722 
signExtendI32ToI64(RegI32 src,RegI64 dest)2723     void signExtendI32ToI64(RegI32 src, RegI64 dest) {
2724 #if defined(JS_CODEGEN_X64)
2725         masm.movslq(src.reg, dest.reg.reg);
2726 #elif defined(JS_CODEGEN_X86)
2727         MOZ_ASSERT(dest.reg.low == src.reg);
2728         MOZ_ASSERT(dest.reg.low == eax);
2729         MOZ_ASSERT(dest.reg.high == edx);
2730         masm.cdq();
2731 #elif defined(JS_CODEGEN_ARM)
2732         masm.ma_mov(src.reg, dest.reg.low);
2733         masm.ma_asr(Imm32(31), src.reg, dest.reg.high);
2734 #else
2735         MOZ_CRASH("BaseCompiler platform hook: signExtendI32ToI64");
2736 #endif
2737     }
2738 
extendU32ToI64(RegI32 src,RegI64 dest)2739     void extendU32ToI64(RegI32 src, RegI64 dest) {
2740 #if defined(JS_CODEGEN_X64)
2741         masm.movl(src.reg, dest.reg.reg);
2742 #elif defined(JS_NUNBOX32)
2743         masm.move32(src.reg, dest.reg.low);
2744         masm.move32(Imm32(0), dest.reg.high);
2745 #else
2746         MOZ_CRASH("BaseCompiler platform hook: extendU32ToI64");
2747 #endif
2748     }
2749 
2750     class OutOfLineTruncateF32OrF64ToI32 : public OutOfLineCode
2751     {
2752         AnyReg src;
2753         RegI32 dest;
2754         bool isAsmJS;
2755         bool isUnsigned;
2756         TrapOffset off;
2757 
2758       public:
OutOfLineTruncateF32OrF64ToI32(AnyReg src,RegI32 dest,bool isAsmJS,bool isUnsigned,TrapOffset off)2759         OutOfLineTruncateF32OrF64ToI32(AnyReg src, RegI32 dest, bool isAsmJS, bool isUnsigned,
2760                                        TrapOffset off)
2761           : src(src),
2762             dest(dest),
2763             isAsmJS(isAsmJS),
2764             isUnsigned(isUnsigned),
2765             off(off)
2766         {
2767             MOZ_ASSERT_IF(isAsmJS, !isUnsigned);
2768         }
2769 
generate(MacroAssembler & masm)2770         virtual void generate(MacroAssembler& masm) {
2771             bool isFloat = src.tag == AnyReg::F32;
2772             FloatRegister fsrc = isFloat ? src.f32().reg : src.f64().reg;
2773             if (isAsmJS) {
2774                 saveVolatileReturnGPR(masm);
2775                 masm.outOfLineTruncateSlow(fsrc, dest.reg, isFloat, /* isAsmJS */ true);
2776                 restoreVolatileReturnGPR(masm);
2777                 masm.jump(rejoin());
2778             } else {
2779 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2780                 if (isFloat)
2781                     masm.outOfLineWasmTruncateFloat32ToInt32(fsrc, isUnsigned, off, rejoin());
2782                 else
2783                     masm.outOfLineWasmTruncateDoubleToInt32(fsrc, isUnsigned, off, rejoin());
2784 #elif defined(JS_CODEGEN_ARM)
2785                 masm.outOfLineWasmTruncateToIntCheck(fsrc,
2786                                                      isFloat ? MIRType::Float32 : MIRType::Double,
2787                                                      MIRType::Int32, isUnsigned, rejoin(), off);
2788 #else
2789                 (void)isUnsigned; // Suppress warnings
2790                 (void)off;        //  for unused private
2791                 MOZ_CRASH("BaseCompiler platform hook: OutOfLineTruncateF32OrF64ToI32 wasm");
2792 #endif
2793             }
2794         }
2795     };
2796 
truncateF32ToI32(RegF32 src,RegI32 dest,bool isUnsigned)2797     MOZ_MUST_USE bool truncateF32ToI32(RegF32 src, RegI32 dest, bool isUnsigned) {
2798         TrapOffset off = trapOffset();
2799         OutOfLineCode* ool;
2800         if (isCompilingAsmJS()) {
2801             ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, true, false, off);
2802             ool = addOutOfLineCode(ool);
2803             if (!ool)
2804                 return false;
2805             masm.branchTruncateFloat32ToInt32(src.reg, dest.reg, ool->entry());
2806         } else {
2807 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM)
2808             ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, false, isUnsigned,
2809                                                              off);
2810             ool = addOutOfLineCode(ool);
2811             if (!ool)
2812                 return false;
2813             if (isUnsigned)
2814                 masm.wasmTruncateFloat32ToUInt32(src.reg, dest.reg, ool->entry());
2815             else
2816                 masm.wasmTruncateFloat32ToInt32(src.reg, dest.reg, ool->entry());
2817 #else
2818             MOZ_CRASH("BaseCompiler platform hook: truncateF32ToI32 wasm");
2819 #endif
2820         }
2821         masm.bind(ool->rejoin());
2822         return true;
2823     }
2824 
truncateF64ToI32(RegF64 src,RegI32 dest,bool isUnsigned)2825     MOZ_MUST_USE bool truncateF64ToI32(RegF64 src, RegI32 dest, bool isUnsigned) {
2826         TrapOffset off = trapOffset();
2827         OutOfLineCode* ool;
2828         if (isCompilingAsmJS()) {
2829             ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, true, false, off);
2830             ool = addOutOfLineCode(ool);
2831             if (!ool)
2832                 return false;
2833             masm.branchTruncateDoubleToInt32(src.reg, dest.reg, ool->entry());
2834         } else {
2835 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM)
2836             ool = new(alloc_) OutOfLineTruncateF32OrF64ToI32(AnyReg(src), dest, false, isUnsigned,
2837                                                              off);
2838             ool = addOutOfLineCode(ool);
2839             if (!ool)
2840                 return false;
2841             if (isUnsigned)
2842                 masm.wasmTruncateDoubleToUInt32(src.reg, dest.reg, ool->entry());
2843             else
2844                 masm.wasmTruncateDoubleToInt32(src.reg, dest.reg, ool->entry());
2845 #else
2846             MOZ_CRASH("BaseCompiler platform hook: truncateF64ToI32 wasm");
2847 #endif
2848         }
2849         masm.bind(ool->rejoin());
2850         return true;
2851     }
2852 
2853     // This does not generate a value; if the truncation failed then it traps.
2854 
2855     class OutOfLineTruncateCheckF32OrF64ToI64 : public OutOfLineCode
2856     {
2857         AnyReg src;
2858         bool isUnsigned;
2859         TrapOffset off;
2860 
2861       public:
OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src,bool isUnsigned,TrapOffset off)2862         OutOfLineTruncateCheckF32OrF64ToI64(AnyReg src, bool isUnsigned, TrapOffset off)
2863           : src(src),
2864             isUnsigned(isUnsigned),
2865             off(off)
2866         {}
2867 
generate(MacroAssembler & masm)2868         virtual void generate(MacroAssembler& masm) {
2869 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
2870             if (src.tag == AnyReg::F32)
2871                 masm.outOfLineWasmTruncateFloat32ToInt64(src.f32().reg, isUnsigned, off, rejoin());
2872             else if (src.tag == AnyReg::F64)
2873                 masm.outOfLineWasmTruncateDoubleToInt64(src.f64().reg, isUnsigned, off, rejoin());
2874             else
2875                 MOZ_CRASH("unexpected type");
2876 #elif defined(JS_CODEGEN_ARM)
2877             if (src.tag == AnyReg::F32)
2878                 masm.outOfLineWasmTruncateToIntCheck(src.f32().reg, MIRType::Float32,
2879                                                      MIRType::Int64, isUnsigned, rejoin(), off);
2880             else if (src.tag == AnyReg::F64)
2881                 masm.outOfLineWasmTruncateToIntCheck(src.f64().reg, MIRType::Double, MIRType::Int64,
2882                                                      isUnsigned, rejoin(), off);
2883             else
2884                 MOZ_CRASH("unexpected type");
2885 #else
2886             (void)src;
2887             (void)isUnsigned;
2888             (void)off;
2889             MOZ_CRASH("BaseCompiler platform hook: OutOfLineTruncateCheckF32OrF64ToI64");
2890 #endif
2891         }
2892     };
2893 
2894 #ifndef FLOAT_TO_I64_CALLOUT
truncateF32ToI64(RegF32 src,RegI64 dest,bool isUnsigned,RegF64 temp)2895     MOZ_MUST_USE bool truncateF32ToI64(RegF32 src, RegI64 dest, bool isUnsigned, RegF64 temp) {
2896 # if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
2897         OutOfLineCode* ool =
2898             addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(src),
2899                                                                               isUnsigned,
2900                                                                               trapOffset()));
2901         if (!ool)
2902             return false;
2903         if (isUnsigned)
2904             masm.wasmTruncateFloat32ToUInt64(src.reg, dest.reg, ool->entry(),
2905                                              ool->rejoin(), temp.reg);
2906         else
2907             masm.wasmTruncateFloat32ToInt64(src.reg, dest.reg, ool->entry(),
2908                                             ool->rejoin(), temp.reg);
2909 # else
2910         MOZ_CRASH("BaseCompiler platform hook: truncateF32ToI64");
2911 # endif
2912         return true;
2913     }
2914 
truncateF64ToI64(RegF64 src,RegI64 dest,bool isUnsigned,RegF64 temp)2915     MOZ_MUST_USE bool truncateF64ToI64(RegF64 src, RegI64 dest, bool isUnsigned, RegF64 temp) {
2916 # if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
2917         OutOfLineCode* ool =
2918             addOutOfLineCode(new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(src),
2919                                                                               isUnsigned,
2920                                                                               trapOffset()));
2921         if (!ool)
2922             return false;
2923         if (isUnsigned)
2924             masm.wasmTruncateDoubleToUInt64(src.reg, dest.reg, ool->entry(),
2925                                             ool->rejoin(), temp.reg);
2926         else
2927             masm.wasmTruncateDoubleToInt64(src.reg, dest.reg, ool->entry(),
2928                                            ool->rejoin(), temp.reg);
2929 # else
2930         MOZ_CRASH("BaseCompiler platform hook: truncateF64ToI64");
2931 # endif
2932         return true;
2933     }
2934 #endif // FLOAT_TO_I64_CALLOUT
2935 
2936 #ifndef I64_TO_FLOAT_CALLOUT
convertI64ToFloatNeedsTemp(bool isUnsigned) const2937     bool convertI64ToFloatNeedsTemp(bool isUnsigned) const {
2938 # if defined(JS_CODEGEN_X86)
2939         return isUnsigned && AssemblerX86Shared::HasSSE3();
2940 # else
2941         return false;
2942 # endif
2943     }
2944 
convertI64ToF32(RegI64 src,bool isUnsigned,RegF32 dest,RegI32 temp)2945     void convertI64ToF32(RegI64 src, bool isUnsigned, RegF32 dest, RegI32 temp) {
2946 # if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
2947         if (isUnsigned)
2948             masm.convertUInt64ToFloat32(src.reg, dest.reg, temp.reg);
2949         else
2950             masm.convertInt64ToFloat32(src.reg, dest.reg);
2951 # else
2952         MOZ_CRASH("BaseCompiler platform hook: convertI64ToF32");
2953 # endif
2954     }
2955 
convertI64ToF64(RegI64 src,bool isUnsigned,RegF64 dest,RegI32 temp)2956     void convertI64ToF64(RegI64 src, bool isUnsigned, RegF64 dest, RegI32 temp) {
2957 # if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
2958         if (isUnsigned)
2959             masm.convertUInt64ToDouble(src.reg, dest.reg, temp.reg);
2960         else
2961             masm.convertInt64ToDouble(src.reg, dest.reg);
2962 # else
2963         MOZ_CRASH("BaseCompiler platform hook: convertI64ToF64");
2964 # endif
2965     }
2966 #endif // I64_TO_FLOAT_CALLOUT
2967 
cmp64Set(Assembler::Condition cond,RegI64 lhs,RegI64 rhs,RegI32 dest)2968     void cmp64Set(Assembler::Condition cond, RegI64 lhs, RegI64 rhs, RegI32 dest) {
2969 #if defined(JS_CODEGEN_X64)
2970         masm.cmpq(rhs.reg.reg, lhs.reg.reg);
2971         masm.emitSet(cond, dest.reg);
2972 #elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
2973         // TODO / OPTIMIZE (Bug 1316822): This is pretty branchy, we should be
2974         // able to do better.
2975         Label done, condTrue;
2976         masm.branch64(cond, lhs.reg, rhs.reg, &condTrue);
2977         masm.move32(Imm32(0), dest.reg);
2978         masm.jump(&done);
2979         masm.bind(&condTrue);
2980         masm.move32(Imm32(1), dest.reg);
2981         masm.bind(&done);
2982 #else
2983         MOZ_CRASH("BaseCompiler platform hook: cmp64Set");
2984 #endif
2985     }
2986 
unreachableTrap()2987     void unreachableTrap()
2988     {
2989         masm.jump(trap(Trap::Unreachable));
2990 #ifdef DEBUG
2991         masm.breakpoint();
2992 #endif
2993     }
2994 
2995     //////////////////////////////////////////////////////////////////////
2996     //
2997     // Global variable access.
2998 
2999     // CodeGenerator{X86,X64}::visitWasmLoadGlobal()
3000 
loadGlobalVarI32(unsigned globalDataOffset,RegI32 r)3001     void loadGlobalVarI32(unsigned globalDataOffset, RegI32 r)
3002     {
3003 #if defined(JS_CODEGEN_X64)
3004         CodeOffset label = masm.loadRipRelativeInt32(r.reg);
3005         masm.append(GlobalAccess(label, globalDataOffset));
3006 #elif defined(JS_CODEGEN_X86)
3007         CodeOffset label = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg);
3008         masm.append(GlobalAccess(label, globalDataOffset));
3009 #elif defined(JS_CODEGEN_ARM)
3010         ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
3011         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3012         masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr), r.reg, scratch);
3013 #else
3014         MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI32");
3015 #endif
3016     }
3017 
loadGlobalVarI64(unsigned globalDataOffset,RegI64 r)3018     void loadGlobalVarI64(unsigned globalDataOffset, RegI64 r)
3019     {
3020 #if defined(JS_CODEGEN_X64)
3021         CodeOffset label = masm.loadRipRelativeInt64(r.reg.reg);
3022         masm.append(GlobalAccess(label, globalDataOffset));
3023 #elif defined(JS_CODEGEN_X86)
3024         CodeOffset labelLow = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.low);
3025         masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET));
3026         CodeOffset labelHigh = masm.movlWithPatch(PatchedAbsoluteAddress(), r.reg.high);
3027         masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET));
3028 #elif defined(JS_CODEGEN_ARM)
3029         ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
3030         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3031         masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch);
3032         masm.ma_dtr(js::jit::IsLoad, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high,
3033                     scratch);
3034 #else
3035         MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarI64");
3036 #endif
3037     }
3038 
loadGlobalVarF32(unsigned globalDataOffset,RegF32 r)3039     void loadGlobalVarF32(unsigned globalDataOffset, RegF32 r)
3040     {
3041 #if defined(JS_CODEGEN_X64)
3042         CodeOffset label = masm.loadRipRelativeFloat32(r.reg);
3043         masm.append(GlobalAccess(label, globalDataOffset));
3044 #elif defined(JS_CODEGEN_X86)
3045         CodeOffset label = masm.vmovssWithPatch(PatchedAbsoluteAddress(), r.reg);
3046         masm.append(GlobalAccess(label, globalDataOffset));
3047 #elif defined(JS_CODEGEN_ARM)
3048         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3049         VFPRegister vd(r.reg);
3050         masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), vd.singleOverlay());
3051 #else
3052         MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF32");
3053 #endif
3054     }
3055 
loadGlobalVarF64(unsigned globalDataOffset,RegF64 r)3056     void loadGlobalVarF64(unsigned globalDataOffset, RegF64 r)
3057     {
3058 #if defined(JS_CODEGEN_X64)
3059         CodeOffset label = masm.loadRipRelativeDouble(r.reg);
3060         masm.append(GlobalAccess(label, globalDataOffset));
3061 #elif defined(JS_CODEGEN_X86)
3062         CodeOffset label = masm.vmovsdWithPatch(PatchedAbsoluteAddress(), r.reg);
3063         masm.append(GlobalAccess(label, globalDataOffset));
3064 #elif defined(JS_CODEGEN_ARM)
3065         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3066         masm.ma_vldr(VFPAddr(GlobalReg, VFPOffImm(addr)), r.reg);
3067 #else
3068         MOZ_CRASH("BaseCompiler platform hook: loadGlobalVarF64");
3069 #endif
3070     }
3071 
3072     // CodeGeneratorX64::visitWasmStoreGlobal()
3073 
storeGlobalVarI32(unsigned globalDataOffset,RegI32 r)3074     void storeGlobalVarI32(unsigned globalDataOffset, RegI32 r)
3075     {
3076 #if defined(JS_CODEGEN_X64)
3077         CodeOffset label = masm.storeRipRelativeInt32(r.reg);
3078         masm.append(GlobalAccess(label, globalDataOffset));
3079 #elif defined(JS_CODEGEN_X86)
3080         CodeOffset label = masm.movlWithPatch(r.reg, PatchedAbsoluteAddress());
3081         masm.append(GlobalAccess(label, globalDataOffset));
3082 #elif defined(JS_CODEGEN_ARM)
3083         ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
3084         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3085         masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr), r.reg, scratch);
3086 #else
3087         MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI32");
3088 #endif
3089     }
3090 
storeGlobalVarI64(unsigned globalDataOffset,RegI64 r)3091     void storeGlobalVarI64(unsigned globalDataOffset, RegI64 r)
3092     {
3093 #if defined(JS_CODEGEN_X64)
3094         CodeOffset label = masm.storeRipRelativeInt64(r.reg.reg);
3095         masm.append(GlobalAccess(label, globalDataOffset));
3096 #elif defined(JS_CODEGEN_X86)
3097         CodeOffset labelLow = masm.movlWithPatch(r.reg.low, PatchedAbsoluteAddress());
3098         masm.append(GlobalAccess(labelLow, globalDataOffset + INT64LOW_OFFSET));
3099         CodeOffset labelHigh = masm.movlWithPatch(r.reg.high, PatchedAbsoluteAddress());
3100         masm.append(GlobalAccess(labelHigh, globalDataOffset + INT64HIGH_OFFSET));
3101 #elif defined(JS_CODEGEN_ARM)
3102         ScratchRegisterScope scratch(*this); // Really must be the ARM scratchreg
3103         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3104         masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64LOW_OFFSET), r.reg.low, scratch);
3105         masm.ma_dtr(js::jit::IsStore, GlobalReg, Imm32(addr + INT64HIGH_OFFSET), r.reg.high,
3106                     scratch);
3107 #else
3108         MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarI64");
3109 #endif
3110     }
3111 
storeGlobalVarF32(unsigned globalDataOffset,RegF32 r)3112     void storeGlobalVarF32(unsigned globalDataOffset, RegF32 r)
3113     {
3114 #if defined(JS_CODEGEN_X64)
3115         CodeOffset label = masm.storeRipRelativeFloat32(r.reg);
3116         masm.append(GlobalAccess(label, globalDataOffset));
3117 #elif defined(JS_CODEGEN_X86)
3118         CodeOffset label = masm.vmovssWithPatch(r.reg, PatchedAbsoluteAddress());
3119         masm.append(GlobalAccess(label, globalDataOffset));
3120 #elif defined(JS_CODEGEN_ARM)
3121         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3122         VFPRegister vd(r.reg);
3123         masm.ma_vstr(vd.singleOverlay(), VFPAddr(GlobalReg, VFPOffImm(addr)));
3124 #else
3125         MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF32");
3126 #endif
3127     }
3128 
storeGlobalVarF64(unsigned globalDataOffset,RegF64 r)3129     void storeGlobalVarF64(unsigned globalDataOffset, RegF64 r)
3130     {
3131 #if defined(JS_CODEGEN_X64)
3132         CodeOffset label = masm.storeRipRelativeDouble(r.reg);
3133         masm.append(GlobalAccess(label, globalDataOffset));
3134 #elif defined(JS_CODEGEN_X86)
3135         CodeOffset label = masm.vmovsdWithPatch(r.reg, PatchedAbsoluteAddress());
3136         masm.append(GlobalAccess(label, globalDataOffset));
3137 #elif defined(JS_CODEGEN_ARM)
3138         unsigned addr = globalDataOffset - WasmGlobalRegBias;
3139         masm.ma_vstr(r.reg, VFPAddr(GlobalReg, VFPOffImm(addr)));
3140 #else
3141         MOZ_CRASH("BaseCompiler platform hook: storeGlobalVarF64");
3142 #endif
3143     }
3144 
3145     //////////////////////////////////////////////////////////////////////
3146     //
3147     // Heap access.
3148 
3149 #ifndef WASM_HUGE_MEMORY
3150     class AsmJSLoadOOB : public OutOfLineCode
3151     {
3152         Scalar::Type viewType;
3153         AnyRegister dest;
3154 
3155       public:
AsmJSLoadOOB(Scalar::Type viewType,AnyRegister dest)3156         AsmJSLoadOOB(Scalar::Type viewType, AnyRegister dest)
3157           : viewType(viewType),
3158             dest(dest)
3159         {}
3160 
generate(MacroAssembler & masm)3161         void generate(MacroAssembler& masm) {
3162 # if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
3163             switch (viewType) {
3164               case Scalar::Float32x4:
3165               case Scalar::Int32x4:
3166               case Scalar::Int8x16:
3167               case Scalar::Int16x8:
3168               case Scalar::MaxTypedArrayViewType:
3169                 MOZ_CRASH("unexpected array type");
3170               case Scalar::Float32:
3171                 masm.loadConstantFloat32(float(GenericNaN()), dest.fpu());
3172                 break;
3173               case Scalar::Float64:
3174                 masm.loadConstantDouble(GenericNaN(), dest.fpu());
3175                 break;
3176               case Scalar::Int8:
3177               case Scalar::Uint8:
3178               case Scalar::Int16:
3179               case Scalar::Uint16:
3180               case Scalar::Int32:
3181               case Scalar::Uint32:
3182               case Scalar::Uint8Clamped:
3183                 masm.movePtr(ImmWord(0), dest.gpr());
3184                 break;
3185               case Scalar::Int64:
3186                 MOZ_CRASH("unexpected array type");
3187             }
3188             masm.jump(rejoin());
3189 # else
3190             Unused << viewType;
3191             Unused << dest;
3192             MOZ_CRASH("Compiler bug: Unexpected platform.");
3193 # endif
3194         }
3195     };
3196 #endif
3197 
checkOffset(MemoryAccessDesc * access,RegI32 ptr)3198     void checkOffset(MemoryAccessDesc* access, RegI32 ptr) {
3199         if (access->offset() >= OffsetGuardLimit) {
3200             masm.branchAdd32(Assembler::CarrySet, Imm32(access->offset()), ptr.reg,
3201                              trap(Trap::OutOfBounds));
3202             access->clearOffset();
3203         }
3204     }
3205 
3206     // This is the temp register passed as the last argument to load()
loadStoreTemps(MemoryAccessDesc & access)3207     MOZ_MUST_USE size_t loadStoreTemps(MemoryAccessDesc& access) {
3208 #if defined(JS_CODEGEN_ARM)
3209         if (access.isUnaligned()) {
3210             switch (access.type()) {
3211               case Scalar::Float32:
3212                 return 1;
3213               case Scalar::Float64:
3214                 return 2;
3215               default:
3216                 break;
3217             }
3218         }
3219         return 0;
3220 #else
3221         return 0;
3222 #endif
3223     }
3224 
3225     // ptr and dest may be the same iff dest is I32.
3226     // This may destroy ptr even if ptr and dest are not the same.
load(MemoryAccessDesc & access,RegI32 ptr,AnyReg dest,RegI32 tmp1,RegI32 tmp2)3227     MOZ_MUST_USE bool load(MemoryAccessDesc& access, RegI32 ptr, AnyReg dest, RegI32 tmp1,
3228                            RegI32 tmp2)
3229     {
3230         checkOffset(&access, ptr);
3231 
3232         OutOfLineCode* ool = nullptr;
3233 #ifndef WASM_HUGE_MEMORY
3234         if (access.isPlainAsmJS()) {
3235             ool = new (alloc_) AsmJSLoadOOB(access.type(), dest.any());
3236             if (!addOutOfLineCode(ool))
3237                 return false;
3238 
3239             masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, ool->entry());
3240         } else {
3241             masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, trap(Trap::OutOfBounds));
3242         }
3243 #endif
3244 
3245 #if defined(JS_CODEGEN_X64)
3246         Operand srcAddr(HeapReg, ptr.reg, TimesOne, access.offset());
3247 
3248         if (dest.tag == AnyReg::I64)
3249             masm.wasmLoadI64(access, srcAddr, dest.i64().reg);
3250         else
3251             masm.wasmLoad(access, srcAddr, dest.any());
3252 #elif defined(JS_CODEGEN_X86)
3253         Operand srcAddr(ptr.reg, access.offset());
3254 
3255         if (dest.tag == AnyReg::I64) {
3256             masm.wasmLoadI64(access, srcAddr, dest.i64().reg);
3257         } else {
3258             bool byteRegConflict = access.byteSize() == 1 && !singleByteRegs_.has(dest.i32().reg);
3259             AnyRegister out = byteRegConflict ? AnyRegister(ScratchRegX86) : dest.any();
3260 
3261             masm.wasmLoad(access, srcAddr, out);
3262 
3263             if (byteRegConflict)
3264                 masm.mov(ScratchRegX86, dest.i32().reg);
3265         }
3266 #elif defined(JS_CODEGEN_ARM)
3267         if (access.offset() != 0)
3268             masm.add32(Imm32(access.offset()), ptr.reg);
3269 
3270         bool isSigned = true;
3271         switch (access.type()) {
3272           case Scalar::Uint8:
3273           case Scalar::Uint16:
3274           case Scalar::Uint32: {
3275             isSigned = false;
3276             MOZ_FALLTHROUGH;
3277           case Scalar::Int8:
3278           case Scalar::Int16:
3279           case Scalar::Int32:
3280             Register rt = dest.tag == AnyReg::I64 ? dest.i64().reg.low : dest.i32().reg;
3281             loadI32(access, isSigned, ptr, rt);
3282             if (dest.tag == AnyReg::I64) {
3283                 if (isSigned)
3284                     masm.ma_asr(Imm32(31), rt, dest.i64().reg.high);
3285                 else
3286                     masm.move32(Imm32(0), dest.i64().reg.high);
3287             }
3288             break;
3289           }
3290           case Scalar::Int64:
3291             loadI64(access, ptr, dest.i64());
3292             break;
3293           case Scalar::Float32:
3294             loadF32(access, ptr, dest.f32(), tmp1);
3295             break;
3296           case Scalar::Float64:
3297             loadF64(access, ptr, dest.f64(), tmp1, tmp2);
3298             break;
3299           default:
3300             MOZ_CRASH("Compiler bug: unexpected array type");
3301         }
3302 #else
3303         MOZ_CRASH("BaseCompiler platform hook: load");
3304 #endif
3305 
3306         if (ool)
3307             masm.bind(ool->rejoin());
3308         return true;
3309     }
3310 
3311     // ptr and src must not be the same register.
3312     // This may destroy ptr.
store(MemoryAccessDesc access,RegI32 ptr,AnyReg src,RegI32 tmp1,RegI32 tmp2)3313     MOZ_MUST_USE bool store(MemoryAccessDesc access, RegI32 ptr, AnyReg src, RegI32 tmp1,
3314                             RegI32 tmp2)
3315     {
3316         checkOffset(&access, ptr);
3317 
3318         Label rejoin;
3319 #ifndef WASM_HUGE_MEMORY
3320         if (access.isPlainAsmJS())
3321             masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, &rejoin);
3322         else
3323             masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr.reg, trap(Trap::OutOfBounds));
3324 #endif
3325 
3326         // Emit the store
3327 #if defined(JS_CODEGEN_X64)
3328         Operand dstAddr(HeapReg, ptr.reg, TimesOne, access.offset());
3329 
3330         masm.wasmStore(access, src.any(), dstAddr);
3331 #elif defined(JS_CODEGEN_X86)
3332         Operand dstAddr(ptr.reg, access.offset());
3333 
3334         if (access.type() == Scalar::Int64) {
3335             masm.wasmStoreI64(access, src.i64().reg, dstAddr);
3336         } else {
3337             AnyRegister value;
3338             if (src.tag == AnyReg::I64) {
3339                 value = AnyRegister(src.i64().reg.low);
3340             } else if (access.byteSize() == 1 && !singleByteRegs_.has(src.i32().reg)) {
3341                 masm.mov(src.i32().reg, ScratchRegX86);
3342                 value = AnyRegister(ScratchRegX86);
3343             } else {
3344                 value = src.any();
3345             }
3346 
3347             masm.wasmStore(access, value, dstAddr);
3348         }
3349 #elif defined(JS_CODEGEN_ARM)
3350         if (access.offset() != 0)
3351             masm.add32(Imm32(access.offset()), ptr.reg);
3352 
3353         switch (access.type()) {
3354           case Scalar::Uint8:
3355             MOZ_FALLTHROUGH;
3356           case Scalar::Uint16:
3357             MOZ_FALLTHROUGH;
3358           case Scalar::Int8:
3359             MOZ_FALLTHROUGH;
3360           case Scalar::Int16:
3361             MOZ_FALLTHROUGH;
3362           case Scalar::Int32:
3363             MOZ_FALLTHROUGH;
3364           case Scalar::Uint32: {
3365             Register rt = src.tag == AnyReg::I64 ? src.i64().reg.low : src.i32().reg;
3366             storeI32(access, ptr, rt);
3367             break;
3368           }
3369           case Scalar::Int64:
3370             storeI64(access, ptr, src.i64());
3371             break;
3372           case Scalar::Float32:
3373             storeF32(access, ptr, src.f32(), tmp1);
3374             break;
3375           case Scalar::Float64:
3376             storeF64(access, ptr, src.f64(), tmp1, tmp2);
3377             break;
3378           default:
3379             MOZ_CRASH("Compiler bug: unexpected array type");
3380         }
3381 #else
3382         MOZ_CRASH("BaseCompiler platform hook: store");
3383 #endif
3384 
3385         if (rejoin.used())
3386             masm.bind(&rejoin);
3387 
3388         return true;
3389     }
3390 
3391 #ifdef JS_CODEGEN_ARM
3392     void
loadI32(MemoryAccessDesc access,bool isSigned,RegI32 ptr,Register rt)3393     loadI32(MemoryAccessDesc access, bool isSigned, RegI32 ptr, Register rt) {
3394         if (access.byteSize() > 1 && access.isUnaligned()) {
3395             masm.add32(HeapReg, ptr.reg);
3396             SecondScratchRegisterScope scratch(*this);
3397             masm.emitUnalignedLoad(isSigned, access.byteSize(), ptr.reg, scratch, rt, 0);
3398         } else {
3399             BufferOffset ld =
3400                 masm.ma_dataTransferN(js::jit::IsLoad, BitSize(access.byteSize()*8),
3401                                       isSigned, HeapReg, ptr.reg, rt, Offset, Assembler::Always);
3402             masm.append(access, ld.getOffset(), masm.framePushed());
3403         }
3404     }
3405 
3406     void
storeI32(MemoryAccessDesc access,RegI32 ptr,Register rt)3407     storeI32(MemoryAccessDesc access, RegI32 ptr, Register rt) {
3408         if (access.byteSize() > 1 && access.isUnaligned()) {
3409             masm.add32(HeapReg, ptr.reg);
3410             masm.emitUnalignedStore(access.byteSize(), ptr.reg, rt, 0);
3411         } else {
3412             BufferOffset st =
3413                 masm.ma_dataTransferN(js::jit::IsStore, BitSize(access.byteSize()*8),
3414                                       IsSigned(false), ptr.reg, HeapReg, rt, Offset,
3415                                       Assembler::Always);
3416             masm.append(access, st.getOffset(), masm.framePushed());
3417         }
3418     }
3419 
3420     void
loadI64(MemoryAccessDesc access,RegI32 ptr,RegI64 dest)3421     loadI64(MemoryAccessDesc access, RegI32 ptr, RegI64 dest) {
3422         if (access.isUnaligned()) {
3423             masm.add32(HeapReg, ptr.reg);
3424             SecondScratchRegisterScope scratch(*this);
3425             masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.low,
3426                                    0);
3427             masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, dest.reg.high,
3428                                    4);
3429         } else {
3430             BufferOffset ld;
3431             ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
3432                                        ptr.reg, dest.reg.low, Offset, Assembler::Always);
3433             masm.append(access, ld.getOffset(), masm.framePushed());
3434             masm.add32(Imm32(4), ptr.reg);
3435             ld = masm.ma_dataTransferN(js::jit::IsLoad, BitSize(32), IsSigned(false), HeapReg,
3436                                        ptr.reg, dest.reg.high, Offset, Assembler::Always);
3437             masm.append(access, ld.getOffset(), masm.framePushed());
3438         }
3439     }
3440 
3441     void
storeI64(MemoryAccessDesc access,RegI32 ptr,RegI64 src)3442     storeI64(MemoryAccessDesc access, RegI32 ptr, RegI64 src) {
3443         if (access.isUnaligned()) {
3444             masm.add32(HeapReg, ptr.reg);
3445             masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.low, 0);
3446             masm.emitUnalignedStore(ByteSize(4), ptr.reg, src.reg.high, 4);
3447         } else {
3448             BufferOffset st;
3449             st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
3450                                        ptr.reg, src.reg.low, Offset, Assembler::Always);
3451             masm.append(access, st.getOffset(), masm.framePushed());
3452             masm.add32(Imm32(4), ptr.reg);
3453             st = masm.ma_dataTransferN(js::jit::IsStore, BitSize(32), IsSigned(false), HeapReg,
3454                                        ptr.reg, src.reg.high, Offset, Assembler::Always);
3455             masm.append(access, st.getOffset(), masm.framePushed());
3456         }
3457     }
3458 
3459     void
loadF32(MemoryAccessDesc access,RegI32 ptr,RegF32 dest,RegI32 tmp1)3460     loadF32(MemoryAccessDesc access, RegI32 ptr, RegF32 dest, RegI32 tmp1) {
3461         masm.add32(HeapReg, ptr.reg);
3462         if (access.isUnaligned()) {
3463             SecondScratchRegisterScope scratch(*this);
3464             masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
3465             masm.ma_vxfer(tmp1.reg, dest.reg);
3466         } else {
3467             BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
3468                                            Assembler::Always);
3469             masm.append(access, ld.getOffset(), masm.framePushed());
3470         }
3471     }
3472 
3473     void
storeF32(MemoryAccessDesc access,RegI32 ptr,RegF32 src,RegI32 tmp1)3474     storeF32(MemoryAccessDesc access, RegI32 ptr, RegF32 src, RegI32 tmp1) {
3475         masm.add32(HeapReg, ptr.reg);
3476         if (access.isUnaligned()) {
3477             masm.ma_vxfer(src.reg, tmp1.reg);
3478             masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
3479         } else {
3480             BufferOffset st =
3481                 masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
3482             masm.append(access, st.getOffset(), masm.framePushed());
3483         }
3484     }
3485 
3486     void
loadF64(MemoryAccessDesc access,RegI32 ptr,RegF64 dest,RegI32 tmp1,RegI32 tmp2)3487     loadF64(MemoryAccessDesc access, RegI32 ptr, RegF64 dest, RegI32 tmp1, RegI32 tmp2) {
3488         masm.add32(HeapReg, ptr.reg);
3489         if (access.isUnaligned()) {
3490             SecondScratchRegisterScope scratch(*this);
3491             masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp1.reg, 0);
3492             masm.emitUnalignedLoad(IsSigned(false), ByteSize(4), ptr.reg, scratch, tmp2.reg, 4);
3493             masm.ma_vxfer(tmp1.reg, tmp2.reg, dest.reg);
3494         } else {
3495             BufferOffset ld = masm.ma_vldr(VFPAddr(ptr.reg, VFPOffImm(0)), dest.reg,
3496                                            Assembler::Always);
3497             masm.append(access, ld.getOffset(), masm.framePushed());
3498         }
3499     }
3500 
3501     void
storeF64(MemoryAccessDesc access,RegI32 ptr,RegF64 src,RegI32 tmp1,RegI32 tmp2)3502     storeF64(MemoryAccessDesc access, RegI32 ptr, RegF64 src, RegI32 tmp1, RegI32 tmp2) {
3503         masm.add32(HeapReg, ptr.reg);
3504         if (access.isUnaligned()) {
3505             masm.ma_vxfer(src.reg, tmp1.reg, tmp2.reg);
3506             masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp1.reg, 0);
3507             masm.emitUnalignedStore(ByteSize(4), ptr.reg, tmp2.reg, 4);
3508         } else {
3509             BufferOffset st =
3510                 masm.ma_vstr(src.reg, VFPAddr(ptr.reg, VFPOffImm(0)), Assembler::Always);
3511             masm.append(access, st.getOffset(), masm.framePushed());
3512         }
3513     }
3514 #endif // JS_CODEGEN_ARM
3515 
3516     ////////////////////////////////////////////////////////////
3517 
3518     // Generally speaking, ABOVE this point there should be no value
3519     // stack manipulation (calls to popI32 etc).
3520 
3521     // Generally speaking, BELOW this point there should be no
3522     // platform dependencies.  We make an exception for x86 register
3523     // targeting, which is not too hard to keep clean.
3524 
3525     ////////////////////////////////////////////////////////////
3526     //
3527     // Sundry wrappers.
3528 
pop2xI32(RegI32 * r0,RegI32 * r1)3529     void pop2xI32(RegI32* r0, RegI32* r1) {
3530         *r1 = popI32();
3531         *r0 = popI32();
3532     }
3533 
popI32ToSpecific(RegI32 specific)3534     RegI32 popI32ToSpecific(RegI32 specific) {
3535         freeI32(specific);
3536         return popI32(specific);
3537     }
3538 
pop2xI64(RegI64 * r0,RegI64 * r1)3539     void pop2xI64(RegI64* r0, RegI64* r1) {
3540         *r1 = popI64();
3541         *r0 = popI64();
3542     }
3543 
popI64ToSpecific(RegI64 specific)3544     RegI64 popI64ToSpecific(RegI64 specific) {
3545         freeI64(specific);
3546         return popI64(specific);
3547     }
3548 
pop2xF32(RegF32 * r0,RegF32 * r1)3549     void pop2xF32(RegF32* r0, RegF32* r1) {
3550         *r1 = popF32();
3551         *r0 = popF32();
3552     }
3553 
pop2xF64(RegF64 * r0,RegF64 * r1)3554     void pop2xF64(RegF64* r0, RegF64* r1) {
3555         *r1 = popF64();
3556         *r0 = popF64();
3557     }
3558 
3559     ////////////////////////////////////////////////////////////
3560     //
3561     // Sundry helpers.
3562 
readCallSiteLineOrBytecode()3563     uint32_t readCallSiteLineOrBytecode() {
3564         if (!func_.callSiteLineNums().empty())
3565             return func_.callSiteLineNums()[lastReadCallSite_++];
3566         return trapOffset().bytecodeOffset;
3567     }
3568 
done() const3569     bool done() const {
3570         return iter_.done();
3571     }
3572 
isCompilingAsmJS() const3573     bool isCompilingAsmJS() const {
3574         return mg_.kind == ModuleKind::AsmJS;
3575     }
3576 
trapOffset() const3577     TrapOffset trapOffset() const {
3578         return iter_.trapOffset();
3579     }
trapIfNotAsmJS() const3580     Maybe<TrapOffset> trapIfNotAsmJS() const {
3581         return isCompilingAsmJS() ? Nothing() : Some(trapOffset());
3582     }
trap(Trap t) const3583     TrapDesc trap(Trap t) const {
3584         return TrapDesc(trapOffset(), t, masm.framePushed());
3585     }
3586 
3587     //////////////////////////////////////////////////////////////////////
3588 
3589     MOZ_MUST_USE bool emitBody();
3590     MOZ_MUST_USE bool emitBlock();
3591     MOZ_MUST_USE bool emitLoop();
3592     MOZ_MUST_USE bool emitIf();
3593     MOZ_MUST_USE bool emitElse();
3594     MOZ_MUST_USE bool emitEnd();
3595     MOZ_MUST_USE bool emitBr();
3596     MOZ_MUST_USE bool emitBrIf();
3597     MOZ_MUST_USE bool emitBrTable();
3598     MOZ_MUST_USE bool emitDrop();
3599     MOZ_MUST_USE bool emitReturn();
3600     MOZ_MUST_USE bool emitCallArgs(const ValTypeVector& args, FunctionCall& baselineCall);
3601     MOZ_MUST_USE bool emitCall();
3602     MOZ_MUST_USE bool emitCallIndirect(bool oldStyle);
3603     MOZ_MUST_USE bool emitCommonMathCall(uint32_t lineOrBytecode, SymbolicAddress callee,
3604                                          ValTypeVector& signature, ExprType retType);
3605     MOZ_MUST_USE bool emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType);
3606     MOZ_MUST_USE bool emitBinaryMathBuiltinCall(SymbolicAddress callee, ValType operandType);
3607 #ifdef INT_DIV_I64_CALLOUT
3608     MOZ_MUST_USE bool emitDivOrModI64BuiltinCall(SymbolicAddress callee, ValType operandType);
3609 #endif
3610     MOZ_MUST_USE bool emitGetLocal();
3611     MOZ_MUST_USE bool emitSetLocal();
3612     MOZ_MUST_USE bool emitTeeLocal();
3613     MOZ_MUST_USE bool emitGetGlobal();
3614     MOZ_MUST_USE bool emitSetGlobal();
3615     MOZ_MUST_USE bool emitTeeGlobal();
3616     MOZ_MUST_USE bool emitLoad(ValType type, Scalar::Type viewType);
3617     MOZ_MUST_USE bool emitStore(ValType resultType, Scalar::Type viewType);
3618     MOZ_MUST_USE bool emitTeeStore(ValType resultType, Scalar::Type viewType);
3619     MOZ_MUST_USE bool emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType);
3620     MOZ_MUST_USE bool emitSelect();
3621 
3622     void endBlock(ExprType type, bool isFunctionBody);
3623     void endLoop(ExprType type);
3624     void endIfThen();
3625     void endIfThenElse(ExprType type);
3626 
3627     void doReturn(ExprType returnType);
3628     void pushReturned(const FunctionCall& call, ExprType type);
3629 
3630     void emitCompareI32(JSOp compareOp, MCompare::CompareType compareType);
3631     void emitCompareI64(JSOp compareOp, MCompare::CompareType compareType);
3632     void emitCompareF32(JSOp compareOp, MCompare::CompareType compareType);
3633     void emitCompareF64(JSOp compareOp, MCompare::CompareType compareType);
3634 
3635     void emitAddI32();
3636     void emitAddI64();
3637     void emitAddF64();
3638     void emitAddF32();
3639     void emitSubtractI32();
3640     void emitSubtractI64();
3641     void emitSubtractF32();
3642     void emitSubtractF64();
3643     void emitMultiplyI32();
3644     void emitMultiplyI64();
3645     void emitMultiplyF32();
3646     void emitMultiplyF64();
3647     void emitQuotientI32();
3648     void emitQuotientU32();
3649     void emitRemainderI32();
3650     void emitRemainderU32();
3651 #ifndef INT_DIV_I64_CALLOUT
3652     void emitQuotientI64();
3653     void emitQuotientU64();
3654     void emitRemainderI64();
3655     void emitRemainderU64();
3656 #endif
3657     void emitDivideF32();
3658     void emitDivideF64();
3659     void emitMinI32();
3660     void emitMaxI32();
3661     void emitMinMaxI32(Assembler::Condition cond);
3662     void emitMinF32();
3663     void emitMaxF32();
3664     void emitMinF64();
3665     void emitMaxF64();
3666     void emitCopysignF32();
3667     void emitCopysignF64();
3668     void emitOrI32();
3669     void emitOrI64();
3670     void emitAndI32();
3671     void emitAndI64();
3672     void emitXorI32();
3673     void emitXorI64();
3674     void emitShlI32();
3675     void emitShlI64();
3676     void emitShrI32();
3677     void emitShrI64();
3678     void emitShrU32();
3679     void emitShrU64();
3680     void emitRotrI32();
3681     void emitRotrI64();
3682     void emitRotlI32();
3683     void emitRotlI64();
3684     void emitEqzI32();
3685     void emitEqzI64();
3686     void emitClzI32();
3687     void emitClzI64();
3688     void emitCtzI32();
3689     void emitCtzI64();
3690     void emitPopcntI32();
3691     void emitPopcntI64();
3692     void emitBitNotI32();
3693     void emitAbsI32();
3694     void emitAbsF32();
3695     void emitAbsF64();
3696     void emitNegateI32();
3697     void emitNegateF32();
3698     void emitNegateF64();
3699     void emitSqrtF32();
3700     void emitSqrtF64();
3701     template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF32ToI32();
3702     template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF64ToI32();
3703 #ifdef FLOAT_TO_I64_CALLOUT
3704     MOZ_MUST_USE bool emitConvertFloatingToInt64Callout(SymbolicAddress callee, ValType operandType,
3705                                                         ValType resultType);
3706 #else
3707     template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF32ToI64();
3708     template<bool isUnsigned> MOZ_MUST_USE bool emitTruncateF64ToI64();
3709 #endif
3710     void emitWrapI64ToI32();
3711     void emitExtendI32ToI64();
3712     void emitExtendU32ToI64();
3713     void emitReinterpretF32AsI32();
3714     void emitReinterpretF64AsI64();
3715     void emitConvertF64ToF32();
3716     void emitConvertI32ToF32();
3717     void emitConvertU32ToF32();
3718     void emitConvertF32ToF64();
3719     void emitConvertI32ToF64();
3720     void emitConvertU32ToF64();
3721 #ifdef I64_TO_FLOAT_CALLOUT
3722     MOZ_MUST_USE bool emitConvertInt64ToFloatingCallout(SymbolicAddress callee, ValType operandType,
3723                                                         ValType resultType);
3724 #else
3725     void emitConvertI64ToF32();
3726     void emitConvertU64ToF32();
3727     void emitConvertI64ToF64();
3728     void emitConvertU64ToF64();
3729 #endif
3730     void emitReinterpretI32AsF32();
3731     void emitReinterpretI64AsF64();
3732     MOZ_MUST_USE bool emitGrowMemory();
3733     MOZ_MUST_USE bool emitCurrentMemory();
3734 };
3735 
3736 void
emitAddI32()3737 BaseCompiler::emitAddI32()
3738 {
3739     int32_t c;
3740     if (popConstI32(c)) {
3741         RegI32 r = popI32();
3742         masm.add32(Imm32(c), r.reg);
3743         pushI32(r);
3744     } else {
3745         RegI32 r0, r1;
3746         pop2xI32(&r0, &r1);
3747         masm.add32(r1.reg, r0.reg);
3748         freeI32(r1);
3749         pushI32(r0);
3750     }
3751 }
3752 
3753 void
emitAddI64()3754 BaseCompiler::emitAddI64()
3755 {
3756     // TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
3757     RegI64 r0, r1;
3758     pop2xI64(&r0, &r1);
3759     masm.add64(r1.reg, r0.reg);
3760     freeI64(r1);
3761     pushI64(r0);
3762 }
3763 
3764 void
emitAddF64()3765 BaseCompiler::emitAddF64()
3766 {
3767     // TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
3768     RegF64 r0, r1;
3769     pop2xF64(&r0, &r1);
3770     masm.addDouble(r1.reg, r0.reg);
3771     freeF64(r1);
3772     pushF64(r0);
3773 }
3774 
3775 void
emitAddF32()3776 BaseCompiler::emitAddF32()
3777 {
3778     // TODO / OPTIMIZE: Ditto check for constant here (Bug 1316803)
3779     RegF32 r0, r1;
3780     pop2xF32(&r0, &r1);
3781     masm.addFloat32(r1.reg, r0.reg);
3782     freeF32(r1);
3783     pushF32(r0);
3784 }
3785 
3786 void
emitSubtractI32()3787 BaseCompiler::emitSubtractI32()
3788 {
3789     RegI32 r0, r1;
3790     pop2xI32(&r0, &r1);
3791     masm.sub32(r1.reg, r0.reg);
3792     freeI32(r1);
3793     pushI32(r0);
3794 }
3795 
3796 void
emitSubtractI64()3797 BaseCompiler::emitSubtractI64()
3798 {
3799     RegI64 r0, r1;
3800     pop2xI64(&r0, &r1);
3801     masm.sub64(r1.reg, r0.reg);
3802     freeI64(r1);
3803     pushI64(r0);
3804 }
3805 
3806 void
emitSubtractF32()3807 BaseCompiler::emitSubtractF32()
3808 {
3809     RegF32 r0, r1;
3810     pop2xF32(&r0, &r1);
3811     masm.subFloat32(r1.reg, r0.reg);
3812     freeF32(r1);
3813     pushF32(r0);
3814 }
3815 
3816 void
emitSubtractF64()3817 BaseCompiler::emitSubtractF64()
3818 {
3819     RegF64 r0, r1;
3820     pop2xF64(&r0, &r1);
3821     masm.subDouble(r1.reg, r0.reg);
3822     freeF64(r1);
3823     pushF64(r0);
3824 }
3825 
3826 void
emitMultiplyI32()3827 BaseCompiler::emitMultiplyI32()
3828 {
3829     // TODO / OPTIMIZE: Multiplication by constant is common (Bug 1275442, 1316803)
3830     RegI32 r0, r1;
3831     pop2xI32ForIntMulDiv(&r0, &r1);
3832     masm.mul32(r1.reg, r0.reg);
3833     freeI32(r1);
3834     pushI32(r0);
3835 }
3836 
3837 void
emitMultiplyI64()3838 BaseCompiler::emitMultiplyI64()
3839 {
3840     // TODO / OPTIMIZE: Multiplication by constant is common (Bug 1275442, 1316803)
3841     RegI64 r0, r1;
3842     RegI32 temp;
3843 #if defined(JS_CODEGEN_X64)
3844     // srcDest must be rax, and rdx will be clobbered.
3845     need2xI64(specific_rax, specific_rdx);
3846     r1 = popI64();
3847     r0 = popI64ToSpecific(specific_rax);
3848     freeI64(specific_rdx);
3849 #elif defined(JS_CODEGEN_X86)
3850     need2xI32(specific_eax, specific_edx);
3851     r1 = popI64();
3852     r0 = popI64ToSpecific(RegI64(Register64(specific_edx.reg, specific_eax.reg)));
3853     temp = needI32();
3854 #else
3855     pop2xI64(&r0, &r1);
3856     temp = needI32();
3857 #endif
3858     masm.mul64(r1.reg, r0.reg, temp.reg);
3859     if (temp.reg != Register::Invalid())
3860         freeI32(temp);
3861     freeI64(r1);
3862     pushI64(r0);
3863 }
3864 
3865 void
emitMultiplyF32()3866 BaseCompiler::emitMultiplyF32()
3867 {
3868     RegF32 r0, r1;
3869     pop2xF32(&r0, &r1);
3870     masm.mulFloat32(r1.reg, r0.reg);
3871     freeF32(r1);
3872     pushF32(r0);
3873 }
3874 
3875 void
emitMultiplyF64()3876 BaseCompiler::emitMultiplyF64()
3877 {
3878     RegF64 r0, r1;
3879     pop2xF64(&r0, &r1);
3880     masm.mulDouble(r1.reg, r0.reg);
3881     freeF64(r1);
3882     pushF64(r0);
3883 }
3884 
3885 void
emitQuotientI32()3886 BaseCompiler::emitQuotientI32()
3887 {
3888     // TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
3889     RegI32 r0, r1;
3890     pop2xI32ForIntMulDiv(&r0, &r1);
3891 
3892     Label done;
3893     checkDivideByZeroI32(r1, r0, &done);
3894     checkDivideSignedOverflowI32(r1, r0, &done, ZeroOnOverflow(false));
3895     masm.quotient32(r1.reg, r0.reg, IsUnsigned(false));
3896     masm.bind(&done);
3897 
3898     freeI32(r1);
3899     pushI32(r0);
3900 }
3901 
3902 void
emitQuotientU32()3903 BaseCompiler::emitQuotientU32()
3904 {
3905     // TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
3906     RegI32 r0, r1;
3907     pop2xI32ForIntMulDiv(&r0, &r1);
3908 
3909     Label done;
3910     checkDivideByZeroI32(r1, r0, &done);
3911     masm.quotient32(r1.reg, r0.reg, IsUnsigned(true));
3912     masm.bind(&done);
3913 
3914     freeI32(r1);
3915     pushI32(r0);
3916 }
3917 
3918 void
emitRemainderI32()3919 BaseCompiler::emitRemainderI32()
3920 {
3921     // TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
3922     RegI32 r0, r1;
3923     pop2xI32ForIntMulDiv(&r0, &r1);
3924 
3925     Label done;
3926     checkDivideByZeroI32(r1, r0, &done);
3927     checkDivideSignedOverflowI32(r1, r0, &done, ZeroOnOverflow(true));
3928     masm.remainder32(r1.reg, r0.reg, IsUnsigned(false));
3929     masm.bind(&done);
3930 
3931     freeI32(r1);
3932     pushI32(r0);
3933 }
3934 
3935 void
emitRemainderU32()3936 BaseCompiler::emitRemainderU32()
3937 {
3938     // TODO / OPTIMIZE: Fast case if lhs >= 0 and rhs is power of two (Bug 1316803)
3939     RegI32 r0, r1;
3940     pop2xI32ForIntMulDiv(&r0, &r1);
3941 
3942     Label done;
3943     checkDivideByZeroI32(r1, r0, &done);
3944     masm.remainder32(r1.reg, r0.reg, IsUnsigned(true));
3945     masm.bind(&done);
3946 
3947     freeI32(r1);
3948     pushI32(r0);
3949 }
3950 
3951 #ifndef INT_DIV_I64_CALLOUT
3952 void
emitQuotientI64()3953 BaseCompiler::emitQuotientI64()
3954 {
3955 # ifdef JS_PUNBOX64
3956     RegI64 r0, r1;
3957     pop2xI64ForIntDiv(&r0, &r1);
3958     quotientI64(r1, r0, IsUnsigned(false));
3959     freeI64(r1);
3960     pushI64(r0);
3961 # else
3962     MOZ_CRASH("BaseCompiler platform hook: emitQuotientI64");
3963 # endif
3964 }
3965 
3966 void
emitQuotientU64()3967 BaseCompiler::emitQuotientU64()
3968 {
3969 # ifdef JS_PUNBOX64
3970     RegI64 r0, r1;
3971     pop2xI64ForIntDiv(&r0, &r1);
3972     quotientI64(r1, r0, IsUnsigned(true));
3973     freeI64(r1);
3974     pushI64(r0);
3975 # else
3976     MOZ_CRASH("BaseCompiler platform hook: emitQuotientU64");
3977 # endif
3978 }
3979 
3980 void
emitRemainderI64()3981 BaseCompiler::emitRemainderI64()
3982 {
3983 # ifdef JS_PUNBOX64
3984     RegI64 r0, r1;
3985     pop2xI64ForIntDiv(&r0, &r1);
3986     remainderI64(r1, r0, IsUnsigned(false));
3987     freeI64(r1);
3988     pushI64(r0);
3989 # else
3990     MOZ_CRASH("BaseCompiler platform hook: emitRemainderI64");
3991 # endif
3992 }
3993 
3994 void
emitRemainderU64()3995 BaseCompiler::emitRemainderU64()
3996 {
3997 # ifdef JS_PUNBOX64
3998     RegI64 r0, r1;
3999     pop2xI64ForIntDiv(&r0, &r1);
4000     remainderI64(r1, r0, IsUnsigned(true));
4001     freeI64(r1);
4002     pushI64(r0);
4003 # else
4004     MOZ_CRASH("BaseCompiler platform hook: emitRemainderU64");
4005 # endif
4006 }
4007 #endif // INT_DIV_I64_CALLOUT
4008 
4009 void
emitDivideF32()4010 BaseCompiler::emitDivideF32()
4011 {
4012     RegF32 r0, r1;
4013     pop2xF32(&r0, &r1);
4014     masm.divFloat32(r1.reg, r0.reg);
4015     freeF32(r1);
4016     pushF32(r0);
4017 }
4018 
4019 void
emitDivideF64()4020 BaseCompiler::emitDivideF64()
4021 {
4022     RegF64 r0, r1;
4023     pop2xF64(&r0, &r1);
4024     masm.divDouble(r1.reg, r0.reg);
4025     freeF64(r1);
4026     pushF64(r0);
4027 }
4028 
4029 void
emitMinI32()4030 BaseCompiler::emitMinI32()
4031 {
4032     emitMinMaxI32(Assembler::LessThan);
4033 }
4034 
4035 void
emitMaxI32()4036 BaseCompiler::emitMaxI32()
4037 {
4038     emitMinMaxI32(Assembler::GreaterThan);
4039 }
4040 
4041 void
emitMinMaxI32(Assembler::Condition cond)4042 BaseCompiler::emitMinMaxI32(Assembler::Condition cond)
4043 {
4044     Label done;
4045     RegI32 r0, r1;
4046     pop2xI32(&r0, &r1);
4047     // TODO / OPTIMIZE (bug 1316823): Use conditional move on some platforms?
4048     masm.branch32(cond, r0.reg, r1.reg, &done);
4049     moveI32(r1, r0);
4050     masm.bind(&done);
4051     freeI32(r1);
4052     pushI32(r0);
4053 }
4054 
4055 void
emitMinF32()4056 BaseCompiler::emitMinF32()
4057 {
4058     RegF32 r0, r1;
4059     pop2xF32(&r0, &r1);
4060     if (!isCompilingAsmJS()) {
4061         // Convert signaling NaN to quiet NaNs.
4062         //
4063         // TODO / OPTIMIZE (bug 1316824): Don't do this if one of the operands
4064         // is known to be a constant.
4065         ScratchF32 zero(*this);
4066         masm.loadConstantFloat32(0.f, zero);
4067         masm.subFloat32(zero, r0.reg);
4068         masm.subFloat32(zero, r1.reg);
4069     }
4070     masm.minFloat32(r1.reg, r0.reg, HandleNaNSpecially(true));
4071     freeF32(r1);
4072     pushF32(r0);
4073 }
4074 
4075 void
emitMaxF32()4076 BaseCompiler::emitMaxF32()
4077 {
4078     RegF32 r0, r1;
4079     pop2xF32(&r0, &r1);
4080     if (!isCompilingAsmJS()) {
4081         // Convert signaling NaN to quiet NaNs.
4082         //
4083         // TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
4084         ScratchF32 zero(*this);
4085         masm.loadConstantFloat32(0.f, zero);
4086         masm.subFloat32(zero, r0.reg);
4087         masm.subFloat32(zero, r1.reg);
4088     }
4089     masm.maxFloat32(r1.reg, r0.reg, HandleNaNSpecially(true));
4090     freeF32(r1);
4091     pushF32(r0);
4092 }
4093 
4094 void
emitMinF64()4095 BaseCompiler::emitMinF64()
4096 {
4097     RegF64 r0, r1;
4098     pop2xF64(&r0, &r1);
4099     if (!isCompilingAsmJS()) {
4100         // Convert signaling NaN to quiet NaNs.
4101         //
4102         // TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
4103         ScratchF64 zero(*this);
4104         masm.loadConstantDouble(0, zero);
4105         masm.subDouble(zero, r0.reg);
4106         masm.subDouble(zero, r1.reg);
4107     }
4108     masm.minDouble(r1.reg, r0.reg, HandleNaNSpecially(true));
4109     freeF64(r1);
4110     pushF64(r0);
4111 }
4112 
4113 void
emitMaxF64()4114 BaseCompiler::emitMaxF64()
4115 {
4116     RegF64 r0, r1;
4117     pop2xF64(&r0, &r1);
4118     if (!isCompilingAsmJS()) {
4119         // Convert signaling NaN to quiet NaNs.
4120         //
4121         // TODO / OPTIMIZE (bug 1316824): see comment in emitMinF32.
4122         ScratchF64 zero(*this);
4123         masm.loadConstantDouble(0, zero);
4124         masm.subDouble(zero, r0.reg);
4125         masm.subDouble(zero, r1.reg);
4126     }
4127     masm.maxDouble(r1.reg, r0.reg, HandleNaNSpecially(true));
4128     freeF64(r1);
4129     pushF64(r0);
4130 }
4131 
4132 void
emitCopysignF32()4133 BaseCompiler::emitCopysignF32()
4134 {
4135     RegF32 r0, r1;
4136     pop2xF32(&r0, &r1);
4137     RegI32 i0 = needI32();
4138     RegI32 i1 = needI32();
4139     masm.moveFloat32ToGPR(r0.reg, i0.reg);
4140     masm.moveFloat32ToGPR(r1.reg, i1.reg);
4141     masm.and32(Imm32(INT32_MAX), i0.reg);
4142     masm.and32(Imm32(INT32_MIN), i1.reg);
4143     masm.or32(i1.reg, i0.reg);
4144     masm.moveGPRToFloat32(i0.reg, r0.reg);
4145     freeI32(i0);
4146     freeI32(i1);
4147     freeF32(r1);
4148     pushF32(r0);
4149 }
4150 
4151 void
emitCopysignF64()4152 BaseCompiler::emitCopysignF64()
4153 {
4154     RegF64 r0, r1;
4155     pop2xF64(&r0, &r1);
4156     RegI64 x0 = needI64();
4157     RegI64 x1 = needI64();
4158     reinterpretF64AsI64(r0, x0);
4159     reinterpretF64AsI64(r1, x1);
4160     masm.and64(Imm64(INT64_MAX), x0.reg);
4161     masm.and64(Imm64(INT64_MIN), x1.reg);
4162     masm.or64(x1.reg, x0.reg);
4163     reinterpretI64AsF64(x0, r0);
4164     freeI64(x0);
4165     freeI64(x1);
4166     freeF64(r1);
4167     pushF64(r0);
4168 }
4169 
4170 void
emitOrI32()4171 BaseCompiler::emitOrI32()
4172 {
4173     RegI32 r0, r1;
4174     pop2xI32(&r0, &r1);
4175     masm.or32(r1.reg, r0.reg);
4176     freeI32(r1);
4177     pushI32(r0);
4178 }
4179 
4180 void
emitOrI64()4181 BaseCompiler::emitOrI64()
4182 {
4183     RegI64 r0, r1;
4184     pop2xI64(&r0, &r1);
4185     masm.or64(r1.reg, r0.reg);
4186     freeI64(r1);
4187     pushI64(r0);
4188 }
4189 
4190 void
emitAndI32()4191 BaseCompiler::emitAndI32()
4192 {
4193     RegI32 r0, r1;
4194     pop2xI32(&r0, &r1);
4195     masm.and32(r1.reg, r0.reg);
4196     freeI32(r1);
4197     pushI32(r0);
4198 }
4199 
4200 void
emitAndI64()4201 BaseCompiler::emitAndI64()
4202 {
4203     RegI64 r0, r1;
4204     pop2xI64(&r0, &r1);
4205     masm.and64(r1.reg, r0.reg);
4206     freeI64(r1);
4207     pushI64(r0);
4208 }
4209 
4210 void
emitXorI32()4211 BaseCompiler::emitXorI32()
4212 {
4213     RegI32 r0, r1;
4214     pop2xI32(&r0, &r1);
4215     masm.xor32(r1.reg, r0.reg);
4216     freeI32(r1);
4217     pushI32(r0);
4218 }
4219 
4220 void
emitXorI64()4221 BaseCompiler::emitXorI64()
4222 {
4223     RegI64 r0, r1;
4224     pop2xI64(&r0, &r1);
4225     masm.xor64(r1.reg, r0.reg);
4226     freeI64(r1);
4227     pushI64(r0);
4228 }
4229 
4230 void
emitShlI32()4231 BaseCompiler::emitShlI32()
4232 {
4233     int32_t c;
4234     if (popConstI32(c)) {
4235         RegI32 r = popI32();
4236         masm.lshift32(Imm32(c & 31), r.reg);
4237         pushI32(r);
4238     } else {
4239         RegI32 r0, r1;
4240         pop2xI32ForShiftOrRotate(&r0, &r1);
4241         maskShiftCount32(r1);
4242         masm.lshift32(r1.reg, r0.reg);
4243         freeI32(r1);
4244         pushI32(r0);
4245     }
4246 }
4247 
4248 void
emitShlI64()4249 BaseCompiler::emitShlI64()
4250 {
4251     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4252     RegI64 r0, r1;
4253     pop2xI64ForShiftOrRotate(&r0, &r1);
4254     masm.lshift64(lowPart(r1), r0.reg);
4255     freeI64(r1);
4256     pushI64(r0);
4257 }
4258 
4259 void
emitShrI32()4260 BaseCompiler::emitShrI32()
4261 {
4262     int32_t c;
4263     if (popConstI32(c)) {
4264         RegI32 r = popI32();
4265         masm.rshift32Arithmetic(Imm32(c & 31), r.reg);
4266         pushI32(r);
4267     } else {
4268         RegI32 r0, r1;
4269         pop2xI32ForShiftOrRotate(&r0, &r1);
4270         maskShiftCount32(r1);
4271         masm.rshift32Arithmetic(r1.reg, r0.reg);
4272         freeI32(r1);
4273         pushI32(r0);
4274     }
4275 }
4276 
4277 void
emitShrI64()4278 BaseCompiler::emitShrI64()
4279 {
4280     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4281     RegI64 r0, r1;
4282     pop2xI64ForShiftOrRotate(&r0, &r1);
4283     masm.rshift64Arithmetic(lowPart(r1), r0.reg);
4284     freeI64(r1);
4285     pushI64(r0);
4286 }
4287 
4288 void
emitShrU32()4289 BaseCompiler::emitShrU32()
4290 {
4291     int32_t c;
4292     if (popConstI32(c)) {
4293         RegI32 r = popI32();
4294         masm.rshift32(Imm32(c & 31), r.reg);
4295         pushI32(r);
4296     } else {
4297         RegI32 r0, r1;
4298         pop2xI32ForShiftOrRotate(&r0, &r1);
4299         maskShiftCount32(r1);
4300         masm.rshift32(r1.reg, r0.reg);
4301         freeI32(r1);
4302         pushI32(r0);
4303     }
4304 }
4305 
4306 void
emitShrU64()4307 BaseCompiler::emitShrU64()
4308 {
4309     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4310     RegI64 r0, r1;
4311     pop2xI64ForShiftOrRotate(&r0, &r1);
4312     masm.rshift64(lowPart(r1), r0.reg);
4313     freeI64(r1);
4314     pushI64(r0);
4315 }
4316 
4317 void
emitRotrI32()4318 BaseCompiler::emitRotrI32()
4319 {
4320     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4321     RegI32 r0, r1;
4322     pop2xI32ForShiftOrRotate(&r0, &r1);
4323     masm.rotateRight(r1.reg, r0.reg, r0.reg);
4324     freeI32(r1);
4325     pushI32(r0);
4326 }
4327 
4328 void
emitRotrI64()4329 BaseCompiler::emitRotrI64()
4330 {
4331     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4332     RegI64 r0, r1;
4333     pop2xI64ForShiftOrRotate(&r0, &r1);
4334     masm.rotateRight64(lowPart(r1), r0.reg, r0.reg, maybeHighPart(r1));
4335     freeI64(r1);
4336     pushI64(r0);
4337 }
4338 
4339 void
emitRotlI32()4340 BaseCompiler::emitRotlI32()
4341 {
4342     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4343     RegI32 r0, r1;
4344     pop2xI32ForShiftOrRotate(&r0, &r1);
4345     masm.rotateLeft(r1.reg, r0.reg, r0.reg);
4346     freeI32(r1);
4347     pushI32(r0);
4348 }
4349 
4350 void
emitRotlI64()4351 BaseCompiler::emitRotlI64()
4352 {
4353     // TODO / OPTIMIZE: Constant rhs (Bug 1316803)
4354     RegI64 r0, r1;
4355     pop2xI64ForShiftOrRotate(&r0, &r1);
4356     masm.rotateLeft64(lowPart(r1), r0.reg, r0.reg, maybeHighPart(r1));
4357     freeI64(r1);
4358     pushI64(r0);
4359 }
4360 
4361 void
emitEqzI32()4362 BaseCompiler::emitEqzI32()
4363 {
4364     // TODO / OPTIMIZE: Boolean evaluation for control (Bug 1286816)
4365     RegI32 r0 = popI32();
4366     masm.cmp32Set(Assembler::Equal, r0.reg, Imm32(0), r0.reg);
4367     pushI32(r0);
4368 }
4369 
4370 void
emitEqzI64()4371 BaseCompiler::emitEqzI64()
4372 {
4373     // TODO / OPTIMIZE: Boolean evaluation for control (Bug 1286816)
4374     // TODO / OPTIMIZE: Avoid the temp register (Bug 1316848)
4375     RegI64 r0 = popI64();
4376     RegI64 r1 = needI64();
4377     setI64(0, r1);
4378     RegI32 i0 = fromI64(r0);
4379     cmp64Set(Assembler::Equal, r0, r1, i0);
4380     freeI64(r1);
4381     freeI64Except(r0, i0);
4382     pushI32(i0);
4383 }
4384 
4385 void
emitClzI32()4386 BaseCompiler::emitClzI32()
4387 {
4388     RegI32 r0 = popI32();
4389     masm.clz32(r0.reg, r0.reg, IsKnownNotZero(false));
4390     pushI32(r0);
4391 }
4392 
4393 void
emitClzI64()4394 BaseCompiler::emitClzI64()
4395 {
4396     RegI64 r0 = popI64();
4397     masm.clz64(r0.reg, lowPart(r0));
4398     maybeClearHighPart(r0);
4399     pushI64(r0);
4400 }
4401 
4402 void
emitCtzI32()4403 BaseCompiler::emitCtzI32()
4404 {
4405     RegI32 r0 = popI32();
4406     masm.ctz32(r0.reg, r0.reg, IsKnownNotZero(false));
4407     pushI32(r0);
4408 }
4409 
4410 void
emitCtzI64()4411 BaseCompiler::emitCtzI64()
4412 {
4413     RegI64 r0 = popI64();
4414     masm.ctz64(r0.reg, lowPart(r0));
4415     maybeClearHighPart(r0);
4416     pushI64(r0);
4417 }
4418 
4419 void
emitPopcntI32()4420 BaseCompiler::emitPopcntI32()
4421 {
4422     RegI32 r0 = popI32();
4423     if (popcnt32NeedsTemp()) {
4424         RegI32 tmp = needI32();
4425         masm.popcnt32(r0.reg, r0.reg, tmp.reg);
4426         freeI32(tmp);
4427     } else {
4428         masm.popcnt32(r0.reg, r0.reg, invalidI32().reg);
4429     }
4430     pushI32(r0);
4431 }
4432 
4433 void
emitPopcntI64()4434 BaseCompiler::emitPopcntI64()
4435 {
4436     RegI64 r0 = popI64();
4437     if (popcnt64NeedsTemp()) {
4438         RegI32 tmp = needI32();
4439         masm.popcnt64(r0.reg, r0.reg, tmp.reg);
4440         freeI32(tmp);
4441     } else {
4442         masm.popcnt64(r0.reg, r0.reg, invalidI32().reg);
4443     }
4444     pushI64(r0);
4445 }
4446 
4447 void
emitBitNotI32()4448 BaseCompiler::emitBitNotI32()
4449 {
4450     RegI32 r0 = popI32();
4451     masm.not32(r0.reg);
4452     pushI32(r0);
4453 }
4454 
4455 void
emitAbsI32()4456 BaseCompiler::emitAbsI32()
4457 {
4458     // TODO / OPTIMIZE (bug 1316823): Use conditional move on some platforms?
4459     Label nonnegative;
4460     RegI32 r0 = popI32();
4461     masm.branch32(Assembler::GreaterThanOrEqual, r0.reg, Imm32(0), &nonnegative);
4462     masm.neg32(r0.reg);
4463     masm.bind(&nonnegative);
4464     pushI32(r0);
4465 }
4466 
4467 void
emitAbsF32()4468 BaseCompiler::emitAbsF32()
4469 {
4470     RegF32 r0 = popF32();
4471     masm.absFloat32(r0.reg, r0.reg);
4472     pushF32(r0);
4473 }
4474 
4475 void
emitAbsF64()4476 BaseCompiler::emitAbsF64()
4477 {
4478     RegF64 r0 = popF64();
4479     masm.absDouble(r0.reg, r0.reg);
4480     pushF64(r0);
4481 }
4482 
4483 void
emitNegateI32()4484 BaseCompiler::emitNegateI32()
4485 {
4486     RegI32 r0 = popI32();
4487     masm.neg32(r0.reg);
4488     pushI32(r0);
4489 }
4490 
4491 void
emitNegateF32()4492 BaseCompiler::emitNegateF32()
4493 {
4494     RegF32 r0 = popF32();
4495     masm.negateFloat(r0.reg);
4496     pushF32(r0);
4497 }
4498 
4499 void
emitNegateF64()4500 BaseCompiler::emitNegateF64()
4501 {
4502     RegF64 r0 = popF64();
4503     masm.negateDouble(r0.reg);
4504     pushF64(r0);
4505 }
4506 
4507 void
emitSqrtF32()4508 BaseCompiler::emitSqrtF32()
4509 {
4510     RegF32 r0 = popF32();
4511     masm.sqrtFloat32(r0.reg, r0.reg);
4512     pushF32(r0);
4513 }
4514 
4515 void
emitSqrtF64()4516 BaseCompiler::emitSqrtF64()
4517 {
4518     RegF64 r0 = popF64();
4519     masm.sqrtDouble(r0.reg, r0.reg);
4520     pushF64(r0);
4521 }
4522 
4523 template<bool isUnsigned>
4524 bool
emitTruncateF32ToI32()4525 BaseCompiler::emitTruncateF32ToI32()
4526 {
4527     RegF32 r0 = popF32();
4528     RegI32 i0 = needI32();
4529     if (!truncateF32ToI32(r0, i0, isUnsigned))
4530         return false;
4531     freeF32(r0);
4532     pushI32(i0);
4533     return true;
4534 }
4535 
4536 template<bool isUnsigned>
4537 bool
emitTruncateF64ToI32()4538 BaseCompiler::emitTruncateF64ToI32()
4539 {
4540     RegF64 r0 = popF64();
4541     RegI32 i0 = needI32();
4542     if (!truncateF64ToI32(r0, i0, isUnsigned))
4543         return false;
4544     freeF64(r0);
4545     pushI32(i0);
4546     return true;
4547 }
4548 
4549 #ifndef FLOAT_TO_I64_CALLOUT
4550 template<bool isUnsigned>
4551 bool
emitTruncateF32ToI64()4552 BaseCompiler::emitTruncateF32ToI64()
4553 {
4554     RegF32 r0 = popF32();
4555     RegI64 x0 = needI64();
4556     if (isUnsigned) {
4557         RegF64 tmp = needF64();
4558         if (!truncateF32ToI64(r0, x0, isUnsigned, tmp))
4559             return false;
4560         freeF64(tmp);
4561     } else {
4562         if (!truncateF32ToI64(r0, x0, isUnsigned, invalidF64()))
4563             return false;
4564     }
4565     freeF32(r0);
4566     pushI64(x0);
4567     return true;
4568 }
4569 
4570 template<bool isUnsigned>
4571 bool
emitTruncateF64ToI64()4572 BaseCompiler::emitTruncateF64ToI64()
4573 {
4574     RegF64 r0 = popF64();
4575     RegI64 x0 = needI64();
4576     if (isUnsigned) {
4577         RegF64 tmp = needF64();
4578         if (!truncateF64ToI64(r0, x0, isUnsigned, tmp))
4579             return false;
4580         freeF64(tmp);
4581     } else {
4582         if (!truncateF64ToI64(r0, x0, isUnsigned, invalidF64()))
4583             return false;
4584     }
4585     freeF64(r0);
4586     pushI64(x0);
4587     return true;
4588 }
4589 #endif // FLOAT_TO_I64_CALLOUT
4590 
4591 void
emitWrapI64ToI32()4592 BaseCompiler::emitWrapI64ToI32()
4593 {
4594     RegI64 r0 = popI64();
4595     RegI32 i0 = fromI64(r0);
4596     wrapI64ToI32(r0, i0);
4597     freeI64Except(r0, i0);
4598     pushI32(i0);
4599 }
4600 
4601 void
emitExtendI32ToI64()4602 BaseCompiler::emitExtendI32ToI64()
4603 {
4604     RegI64 x0 = popI32ForSignExtendI64();
4605     RegI32 r0 = RegI32(lowPart(x0));
4606     signExtendI32ToI64(r0, x0);
4607     pushI64(x0);
4608     // Note: no need to free r0, since it is part of x0
4609 }
4610 
4611 void
emitExtendU32ToI64()4612 BaseCompiler::emitExtendU32ToI64()
4613 {
4614     RegI32 r0 = popI32();
4615     RegI64 x0 = widenI32(r0);
4616     extendU32ToI64(r0, x0);
4617     pushI64(x0);
4618     // Note: no need to free r0, since it is part of x0
4619 }
4620 
4621 void
emitReinterpretF32AsI32()4622 BaseCompiler::emitReinterpretF32AsI32()
4623 {
4624     RegF32 r0 = popF32();
4625     RegI32 i0 = needI32();
4626     masm.moveFloat32ToGPR(r0.reg, i0.reg);
4627     freeF32(r0);
4628     pushI32(i0);
4629 }
4630 
4631 void
emitReinterpretF64AsI64()4632 BaseCompiler::emitReinterpretF64AsI64()
4633 {
4634     RegF64 r0 = popF64();
4635     RegI64 x0 = needI64();
4636     reinterpretF64AsI64(r0, x0);
4637     freeF64(r0);
4638     pushI64(x0);
4639 }
4640 
4641 void
emitConvertF64ToF32()4642 BaseCompiler::emitConvertF64ToF32()
4643 {
4644     RegF64 r0 = popF64();
4645     RegF32 f0 = needF32();
4646     masm.convertDoubleToFloat32(r0.reg, f0.reg);
4647     freeF64(r0);
4648     pushF32(f0);
4649 }
4650 
4651 void
emitConvertI32ToF32()4652 BaseCompiler::emitConvertI32ToF32()
4653 {
4654     RegI32 r0 = popI32();
4655     RegF32 f0 = needF32();
4656     masm.convertInt32ToFloat32(r0.reg, f0.reg);
4657     freeI32(r0);
4658     pushF32(f0);
4659 }
4660 
4661 void
emitConvertU32ToF32()4662 BaseCompiler::emitConvertU32ToF32()
4663 {
4664     RegI32 r0 = popI32();
4665     RegF32 f0 = needF32();
4666     masm.convertUInt32ToFloat32(r0.reg, f0.reg);
4667     freeI32(r0);
4668     pushF32(f0);
4669 }
4670 
4671 #ifndef I64_TO_FLOAT_CALLOUT
4672 void
emitConvertI64ToF32()4673 BaseCompiler::emitConvertI64ToF32()
4674 {
4675     RegI64 r0 = popI64();
4676     RegF32 f0 = needF32();
4677     convertI64ToF32(r0, IsUnsigned(false), f0, RegI32());
4678     freeI64(r0);
4679     pushF32(f0);
4680 }
4681 
4682 void
emitConvertU64ToF32()4683 BaseCompiler::emitConvertU64ToF32()
4684 {
4685     RegI64 r0 = popI64();
4686     RegF32 f0 = needF32();
4687     RegI32 temp;
4688     if (convertI64ToFloatNeedsTemp(IsUnsigned(true)))
4689         temp = needI32();
4690     convertI64ToF32(r0, IsUnsigned(true), f0, temp);
4691     if (temp.reg != Register::Invalid())
4692         freeI32(temp);
4693     freeI64(r0);
4694     pushF32(f0);
4695 }
4696 #endif
4697 
4698 void
emitConvertF32ToF64()4699 BaseCompiler::emitConvertF32ToF64()
4700 {
4701     RegF32 r0 = popF32();
4702     RegF64 d0 = needF64();
4703     masm.convertFloat32ToDouble(r0.reg, d0.reg);
4704     freeF32(r0);
4705     pushF64(d0);
4706 }
4707 
4708 void
emitConvertI32ToF64()4709 BaseCompiler::emitConvertI32ToF64()
4710 {
4711     RegI32 r0 = popI32();
4712     RegF64 d0 = needF64();
4713     masm.convertInt32ToDouble(r0.reg, d0.reg);
4714     freeI32(r0);
4715     pushF64(d0);
4716 }
4717 
4718 void
emitConvertU32ToF64()4719 BaseCompiler::emitConvertU32ToF64()
4720 {
4721     RegI32 r0 = popI32();
4722     RegF64 d0 = needF64();
4723     masm.convertUInt32ToDouble(r0.reg, d0.reg);
4724     freeI32(r0);
4725     pushF64(d0);
4726 }
4727 
4728 #ifndef I64_TO_FLOAT_CALLOUT
4729 void
emitConvertI64ToF64()4730 BaseCompiler::emitConvertI64ToF64()
4731 {
4732     RegI64 r0 = popI64();
4733     RegF64 d0 = needF64();
4734     convertI64ToF64(r0, IsUnsigned(false), d0, RegI32());
4735     freeI64(r0);
4736     pushF64(d0);
4737 }
4738 
4739 void
emitConvertU64ToF64()4740 BaseCompiler::emitConvertU64ToF64()
4741 {
4742     RegI64 r0 = popI64();
4743     RegF64 d0 = needF64();
4744     RegI32 temp;
4745     if (convertI64ToFloatNeedsTemp(IsUnsigned(true)))
4746         temp = needI32();
4747     convertI64ToF64(r0, IsUnsigned(true), d0, temp);
4748     if (temp.reg != Register::Invalid())
4749         freeI32(temp);
4750     freeI64(r0);
4751     pushF64(d0);
4752 }
4753 #endif // I64_TO_FLOAT_CALLOUT
4754 
4755 void
emitReinterpretI32AsF32()4756 BaseCompiler::emitReinterpretI32AsF32()
4757 {
4758     RegI32 r0 = popI32();
4759     RegF32 f0 = needF32();
4760     masm.moveGPRToFloat32(r0.reg, f0.reg);
4761     freeI32(r0);
4762     pushF32(f0);
4763 }
4764 
4765 void
emitReinterpretI64AsF64()4766 BaseCompiler::emitReinterpretI64AsF64()
4767 {
4768     RegI64 r0 = popI64();
4769     RegF64 d0 = needF64();
4770     reinterpretI64AsF64(r0, d0);
4771     freeI64(r0);
4772     pushF64(d0);
4773 }
4774 
4775 // For blocks and loops and ifs:
4776 //
4777 //  - Sync the value stack before going into the block in order to simplify exit
4778 //    from the block: all exits from the block can assume that there are no
4779 //    live registers except the one carrying the exit value.
4780 //  - The block can accumulate a number of dead values on the stacks, so when
4781 //    branching out of the block or falling out at the end be sure to
4782 //    pop the appropriate stacks back to where they were on entry, while
4783 //    preserving the exit value.
4784 //  - A continue branch in a loop is much like an exit branch, but the branch
4785 //    value must not be preserved.
4786 //  - The exit value is always in a designated join register (type dependent).
4787 
4788 bool
emitBlock()4789 BaseCompiler::emitBlock()
4790 {
4791     if (!iter_.readBlock())
4792         return false;
4793 
4794     UniquePooledLabel blockEnd(newLabel());
4795     if (!blockEnd)
4796         return false;
4797 
4798     if (!deadCode_)
4799         sync();                    // Simplifies branching out from block
4800 
4801     return pushControl(&blockEnd);
4802 }
4803 
4804 void
endBlock(ExprType type,bool isFunctionBody)4805 BaseCompiler::endBlock(ExprType type, bool isFunctionBody)
4806 {
4807     Control& block = controlItem(0);
4808 
4809     // Save the value.
4810     AnyReg r;
4811     if (!deadCode_ && !IsVoid(type))
4812         r = popJoinReg();
4813 
4814     // Leave the block.
4815     popStackOnBlockExit(block.framePushed);
4816 
4817     // Bind after cleanup: branches out will have popped the stack.
4818     if (block.label->used()) {
4819         masm.bind(block.label);
4820         if (deadCode_ && !IsVoid(type))
4821             r = allocJoinReg(type);
4822         deadCode_ = false;
4823     }
4824 
4825     MOZ_ASSERT(stk_.length() == block.stackSize);
4826 
4827     // Retain the value stored in joinReg by all paths.
4828     if (!deadCode_) {
4829         if (!IsVoid(type))
4830             pushJoinReg(r);
4831 
4832         if (isFunctionBody)
4833             doReturn(func_.sig().ret());
4834     }
4835 
4836     popControl();
4837 }
4838 
4839 bool
emitLoop()4840 BaseCompiler::emitLoop()
4841 {
4842     if (!iter_.readLoop())
4843         return false;
4844 
4845     UniquePooledLabel blockCont(newLabel());
4846     if (!blockCont)
4847         return false;
4848 
4849     if (!deadCode_)
4850         sync();                    // Simplifies branching out from block
4851 
4852     if (!pushControl(&blockCont))
4853         return false;
4854 
4855     if (!deadCode_) {
4856         masm.bind(controlItem(0).label);
4857         addInterruptCheck();
4858     }
4859 
4860     return true;
4861 }
4862 
4863 void
endLoop(ExprType type)4864 BaseCompiler::endLoop(ExprType type)
4865 {
4866     Control& block = controlItem(0);
4867 
4868     AnyReg r;
4869     if (!deadCode_ && !IsVoid(type))
4870         r = popJoinReg();
4871 
4872     popStackOnBlockExit(block.framePushed);
4873 
4874     MOZ_ASSERT(stk_.length() == block.stackSize);
4875 
4876     popControl();
4877 
4878     // Retain the value stored in joinReg by all paths.
4879     if (!deadCode_ && !IsVoid(type))
4880         pushJoinReg(r);
4881 }
4882 
4883 // The bodies of the "then" and "else" arms can be arbitrary sequences
4884 // of expressions, they push control and increment the nesting and can
4885 // even be targeted by jumps.  A branch to the "if" block branches to
4886 // the exit of the if, ie, it's like "break".  Consider:
4887 //
4888 //      (func (result i32)
4889 //       (if (i32.const 1)
4890 //           (begin (br 1) (unreachable))
4891 //           (begin (unreachable)))
4892 //       (i32.const 1))
4893 //
4894 // The branch causes neither of the unreachable expressions to be
4895 // evaluated.
4896 
4897 bool
emitIf()4898 BaseCompiler::emitIf()
4899 {
4900     Nothing unused_cond;
4901     if (!iter_.readIf(&unused_cond))
4902         return false;
4903 
4904     UniquePooledLabel endLabel(newLabel());
4905     if (!endLabel)
4906         return false;
4907 
4908     UniquePooledLabel elseLabel(newLabel());
4909     if (!elseLabel)
4910         return false;
4911 
4912     RegI32 rc;
4913     if (!deadCode_) {
4914         rc = popI32();
4915         sync();                    // Simplifies branching out from the arms
4916     }
4917 
4918     if (!pushControl(&endLabel, &elseLabel))
4919         return false;
4920 
4921     if (!deadCode_) {
4922         masm.branch32(Assembler::Equal, rc.reg, Imm32(0), controlItem(0).otherLabel);
4923         freeI32(rc);
4924     }
4925 
4926     return true;
4927 }
4928 
4929 void
endIfThen()4930 BaseCompiler::endIfThen()
4931 {
4932     Control& ifThen = controlItem(0);
4933 
4934     popStackOnBlockExit(ifThen.framePushed);
4935 
4936     if (ifThen.otherLabel->used())
4937         masm.bind(ifThen.otherLabel);
4938 
4939     if (ifThen.label->used())
4940         masm.bind(ifThen.label);
4941 
4942     deadCode_ = ifThen.deadOnArrival;
4943 
4944     MOZ_ASSERT(stk_.length() == ifThen.stackSize);
4945 
4946     popControl();
4947 }
4948 
4949 bool
emitElse()4950 BaseCompiler::emitElse()
4951 {
4952     ExprType thenType;
4953     Nothing unused_thenValue;
4954     if (!iter_.readElse(&thenType, &unused_thenValue))
4955         return false;
4956 
4957     Control& ifThenElse = controlItem(0);
4958 
4959     // See comment in endIfThenElse, below.
4960 
4961     // Exit the "then" branch.
4962 
4963     ifThenElse.deadThenBranch = deadCode_;
4964 
4965     AnyReg r;
4966     if (!deadCode_ && !IsVoid(thenType))
4967         r = popJoinReg();
4968 
4969     popStackOnBlockExit(ifThenElse.framePushed);
4970 
4971     if (!deadCode_)
4972         masm.jump(ifThenElse.label);
4973 
4974     if (ifThenElse.otherLabel->used())
4975         masm.bind(ifThenElse.otherLabel);
4976 
4977     // Reset to the "else" branch.
4978 
4979     MOZ_ASSERT(stk_.length() == ifThenElse.stackSize);
4980 
4981     if (!deadCode_ && !IsVoid(thenType))
4982         freeJoinReg(r);
4983 
4984     deadCode_ = ifThenElse.deadOnArrival;
4985 
4986     return true;
4987 }
4988 
4989 void
endIfThenElse(ExprType type)4990 BaseCompiler::endIfThenElse(ExprType type)
4991 {
4992     Control& ifThenElse = controlItem(0);
4993 
4994     // The expression type is not a reliable guide to what we'll find
4995     // on the stack, we could have (if E (i32.const 1) (unreachable))
4996     // in which case the "else" arm is AnyType but the type of the
4997     // full expression is I32.  So restore whatever's there, not what
4998     // we want to find there.  The "then" arm has the same constraint.
4999 
5000     AnyReg r;
5001     if (!deadCode_ && !IsVoid(type))
5002         r = popJoinReg();
5003 
5004     popStackOnBlockExit(ifThenElse.framePushed);
5005 
5006     if (ifThenElse.label->used())
5007         masm.bind(ifThenElse.label);
5008 
5009     if (!ifThenElse.deadOnArrival &&
5010         (!ifThenElse.deadThenBranch || !deadCode_ || ifThenElse.label->bound())) {
5011         if (deadCode_ && !IsVoid(type))
5012             r = allocJoinReg(type);
5013         deadCode_ = false;
5014     }
5015 
5016     MOZ_ASSERT(stk_.length() == ifThenElse.stackSize);
5017 
5018     popControl();
5019 
5020     if (!deadCode_ && !IsVoid(type))
5021         pushJoinReg(r);
5022 }
5023 
5024 bool
emitEnd()5025 BaseCompiler::emitEnd()
5026 {
5027     LabelKind kind;
5028     ExprType type;
5029     Nothing unused_value;
5030     if (!iter_.readEnd(&kind, &type, &unused_value))
5031         return false;
5032 
5033     switch (kind) {
5034       case LabelKind::Block: endBlock(type, iter_.controlStackEmpty()); break;
5035       case LabelKind::Loop:  endLoop(type); break;
5036       case LabelKind::UnreachableThen:
5037       case LabelKind::Then:  endIfThen(); break;
5038       case LabelKind::Else:  endIfThenElse(type); break;
5039     }
5040 
5041     return true;
5042 }
5043 
5044 bool
emitBr()5045 BaseCompiler::emitBr()
5046 {
5047     uint32_t relativeDepth;
5048     ExprType type;
5049     Nothing unused_value;
5050     if (!iter_.readBr(&relativeDepth, &type, &unused_value))
5051         return false;
5052 
5053     if (deadCode_)
5054         return true;
5055 
5056     Control& target = controlItem(relativeDepth);
5057 
5058     // Save any value in the designated join register, where the
5059     // normal block exit code will also leave it.
5060 
5061     AnyReg r;
5062     if (!IsVoid(type))
5063         r = popJoinReg();
5064 
5065     popStackBeforeBranch(target.framePushed);
5066     masm.jump(target.label);
5067 
5068     // The register holding the join value is free for the remainder
5069     // of this block.
5070 
5071     if (!IsVoid(type))
5072         freeJoinReg(r);
5073 
5074     deadCode_ = true;
5075 
5076     popValueStackTo(ctl_.back().stackSize);
5077 
5078     return true;
5079 }
5080 
5081 bool
emitBrIf()5082 BaseCompiler::emitBrIf()
5083 {
5084     uint32_t relativeDepth;
5085     ExprType type;
5086     Nothing unused_value, unused_condition;
5087     if (!iter_.readBrIf(&relativeDepth, &type, &unused_value, &unused_condition))
5088         return false;
5089 
5090     if (deadCode_)
5091         return true;
5092 
5093     Control& target = controlItem(relativeDepth);
5094 
5095     // TODO / OPTIMIZE (Bug 1286816): Optimize boolean evaluation for control by
5096     // allowing a conditional expression to be left on the stack and reified
5097     // here as part of the branch instruction.
5098 
5099     // Don't use joinReg for rc
5100     maybeReserveJoinRegI(type);
5101 
5102     // Condition value is on top, always I32.
5103     RegI32 rc = popI32();
5104 
5105     maybeUnreserveJoinRegI(type);
5106 
5107     // Save any value in the designated join register, where the
5108     // normal block exit code will also leave it.
5109     AnyReg r;
5110     if (!IsVoid(type))
5111         r = popJoinReg();
5112 
5113     Label notTaken;
5114     masm.branch32(Assembler::Equal, rc.reg, Imm32(0), &notTaken);
5115     popStackBeforeBranch(target.framePushed);
5116     masm.jump(target.label);
5117     masm.bind(&notTaken);
5118 
5119     // This register is free in the remainder of the block.
5120     freeI32(rc);
5121 
5122     // br_if returns its value(s).
5123     if (!IsVoid(type))
5124         pushJoinReg(r);
5125 
5126     return true;
5127 }
5128 
5129 bool
emitBrTable()5130 BaseCompiler::emitBrTable()
5131 {
5132     uint32_t tableLength;
5133     ExprType type;
5134     Nothing unused_value, unused_index;
5135     if (!iter_.readBrTable(&tableLength, &type, &unused_value, &unused_index))
5136         return false;
5137 
5138     LabelVector stubs;
5139     if (!stubs.reserve(tableLength+1))
5140         return false;
5141 
5142     Uint32Vector depths;
5143     if (!depths.reserve(tableLength))
5144         return false;
5145 
5146     for (size_t i = 0; i < tableLength; ++i) {
5147         uint32_t depth;
5148         if (!iter_.readBrTableEntry(&type, &unused_value, &depth))
5149             return false;
5150         depths.infallibleAppend(depth);
5151     }
5152 
5153     uint32_t defaultDepth;
5154     if (!iter_.readBrTableDefault(&type, &unused_value, &defaultDepth))
5155         return false;
5156 
5157     if (deadCode_)
5158         return true;
5159 
5160     // Don't use joinReg for rc
5161     maybeReserveJoinRegI(type);
5162 
5163     // Table switch value always on top.
5164     RegI32 rc = popI32();
5165 
5166     maybeUnreserveJoinRegI(type);
5167 
5168     AnyReg r;
5169     if (!IsVoid(type))
5170         r = popJoinReg();
5171 
5172     Label dispatchCode;
5173     masm.branch32(Assembler::Below, rc.reg, Imm32(tableLength), &dispatchCode);
5174 
5175     // This is the out-of-range stub.  rc is dead here but we don't need it.
5176 
5177     popStackBeforeBranch(controlItem(defaultDepth).framePushed);
5178     masm.jump(controlItem(defaultDepth).label);
5179 
5180     // Emit stubs.  rc is dead in all of these but we don't need it.
5181     //
5182     // TODO / OPTIMIZE (Bug 1316804): Branch directly to the case code if we
5183     // can, don't emit an intermediate stub.
5184 
5185     for (uint32_t i = 0; i < tableLength; i++) {
5186         PooledLabel* stubLabel = newLabel();
5187         // The labels in the vector are in the TempAllocator and will
5188         // be freed by and by.
5189         if (!stubLabel)
5190             return false;
5191         stubs.infallibleAppend(stubLabel);
5192         masm.bind(stubLabel);
5193         uint32_t k = depths[i];
5194         popStackBeforeBranch(controlItem(k).framePushed);
5195         masm.jump(controlItem(k).label);
5196     }
5197 
5198     // Emit table.
5199 
5200     Label theTable;
5201     masm.bind(&theTable);
5202     jumpTable(stubs);
5203 
5204     // Emit indirect jump.  rc is live here.
5205 
5206     masm.bind(&dispatchCode);
5207     tableSwitch(&theTable, rc);
5208 
5209     deadCode_ = true;
5210 
5211     // Clean up.
5212 
5213     freeI32(rc);
5214     if (!IsVoid(type))
5215         freeJoinReg(r);
5216 
5217     for (uint32_t i = 0; i < tableLength; i++)
5218         freeLabel(stubs[i]);
5219 
5220     popValueStackTo(ctl_.back().stackSize);
5221 
5222     return true;
5223 }
5224 
5225 bool
emitDrop()5226 BaseCompiler::emitDrop()
5227 {
5228     if (!iter_.readDrop())
5229         return false;
5230 
5231     if (deadCode_)
5232         return true;
5233 
5234     popStackIfMemory();
5235     popValueStackBy(1);
5236     return true;
5237 }
5238 
5239 void
doReturn(ExprType type)5240 BaseCompiler::doReturn(ExprType type)
5241 {
5242     switch (type) {
5243       case ExprType::Void: {
5244         returnCleanup();
5245         break;
5246       }
5247       case ExprType::I32: {
5248         RegI32 rv = popI32(RegI32(ReturnReg));
5249         returnCleanup();
5250         freeI32(rv);
5251         break;
5252       }
5253       case ExprType::I64: {
5254         RegI64 rv = popI64(RegI64(ReturnReg64));
5255         returnCleanup();
5256         freeI64(rv);
5257         break;
5258       }
5259       case ExprType::F64: {
5260         RegF64 rv = popF64(RegF64(ReturnDoubleReg));
5261         returnCleanup();
5262         freeF64(rv);
5263         break;
5264       }
5265       case ExprType::F32: {
5266         RegF32 rv = popF32(RegF32(ReturnFloat32Reg));
5267         returnCleanup();
5268         freeF32(rv);
5269         break;
5270       }
5271       default: {
5272         MOZ_CRASH("Function return type");
5273       }
5274     }
5275 }
5276 
5277 bool
emitReturn()5278 BaseCompiler::emitReturn()
5279 {
5280     Nothing unused_value;
5281     if (!iter_.readReturn(&unused_value))
5282         return false;
5283 
5284     if (deadCode_)
5285         return true;
5286 
5287     doReturn(func_.sig().ret());
5288     deadCode_ = true;
5289 
5290     popValueStackTo(ctl_.back().stackSize);
5291 
5292     return true;
5293 }
5294 
5295 bool
emitCallArgs(const ValTypeVector & args,FunctionCall & baselineCall)5296 BaseCompiler::emitCallArgs(const ValTypeVector& args, FunctionCall& baselineCall)
5297 {
5298     MOZ_ASSERT(!deadCode_);
5299 
5300     startCallArgs(baselineCall, stackArgAreaSize(args));
5301 
5302     uint32_t numArgs = args.length();
5303     for (size_t i = 0; i < numArgs; ++i) {
5304         ValType argType = args[i];
5305         Nothing arg_;
5306         if (!iter_.readCallArg(argType, numArgs, i, &arg_))
5307             return false;
5308         Stk& arg = peek(numArgs - 1 - i);
5309         passArg(baselineCall, argType, arg);
5310     }
5311 
5312     // Pass the TLS pointer as a hidden argument in WasmTlsReg.  Load
5313     // it directly out if its stack slot so we don't interfere with
5314     // the stk_.
5315     if (baselineCall.loadTlsBefore)
5316         loadFromFramePtr(WasmTlsReg, frameOffsetFromSlot(tlsSlot_, MIRType::Pointer));
5317 
5318     if (!iter_.readCallArgsEnd(numArgs))
5319         return false;
5320 
5321     return true;
5322 }
5323 
5324 void
pushReturned(const FunctionCall & call,ExprType type)5325 BaseCompiler::pushReturned(const FunctionCall& call, ExprType type)
5326 {
5327     switch (type) {
5328       case ExprType::Void:
5329         MOZ_CRASH("Compiler bug: attempt to push void return");
5330         break;
5331       case ExprType::I32: {
5332         RegI32 rv = captureReturnedI32();
5333         pushI32(rv);
5334         break;
5335       }
5336       case ExprType::I64: {
5337         RegI64 rv = captureReturnedI64();
5338         pushI64(rv);
5339         break;
5340       }
5341       case ExprType::F32: {
5342         RegF32 rv = captureReturnedF32(call);
5343         pushF32(rv);
5344         break;
5345       }
5346       case ExprType::F64: {
5347         RegF64 rv = captureReturnedF64(call);
5348         pushF64(rv);
5349         break;
5350       }
5351       default:
5352         MOZ_CRASH("Function return type");
5353     }
5354 }
5355 
5356 // For now, always sync() at the beginning of the call to easily save live
5357 // values.
5358 //
5359 // TODO / OPTIMIZE (Bug 1316806): We may be able to avoid a full sync(), since
5360 // all we want is to save live registers that won't be saved by the callee or
5361 // that we need for outgoing args - we don't need to sync the locals.  We can
5362 // just push the necessary registers, it'll be like a lightweight sync.
5363 //
5364 // Even some of the pushing may be unnecessary if the registers will be consumed
5365 // by the call, because then what we want is parallel assignment to the argument
5366 // registers or onto the stack for outgoing arguments.  A sync() is just
5367 // simpler.
5368 
5369 bool
emitCall()5370 BaseCompiler::emitCall()
5371 {
5372     uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
5373 
5374     uint32_t funcIndex;
5375     if (!iter_.readCall(&funcIndex))
5376         return false;
5377 
5378     if (deadCode_)
5379         return true;
5380 
5381     sync();
5382 
5383     const Sig& sig = *mg_.funcSigs[funcIndex];
5384     bool import = mg_.funcIsImport(funcIndex);
5385 
5386     uint32_t numArgs = sig.args().length();
5387     size_t stackSpace = stackConsumed(numArgs);
5388 
5389     FunctionCall baselineCall(lineOrBytecode);
5390     beginCall(baselineCall, UseABI::Wasm, import ? InterModule::True : InterModule::False);
5391 
5392     if (!emitCallArgs(sig.args(), baselineCall))
5393         return false;
5394 
5395     if (!iter_.readCallReturn(sig.ret()))
5396         return false;
5397 
5398     if (import)
5399         callImport(mg_.funcImportGlobalDataOffsets[funcIndex], baselineCall);
5400     else
5401         callDefinition(funcIndex, baselineCall);
5402 
5403     endCall(baselineCall);
5404 
5405     // TODO / OPTIMIZE (bug 1316827): It would be better to merge this
5406     // freeStack() into the one in endCall, if we can.
5407 
5408     popValueStackBy(numArgs);
5409     masm.freeStack(stackSpace);
5410 
5411     if (!IsVoid(sig.ret()))
5412         pushReturned(baselineCall, sig.ret());
5413 
5414     return true;
5415 }
5416 
5417 bool
emitCallIndirect(bool oldStyle)5418 BaseCompiler::emitCallIndirect(bool oldStyle)
5419 {
5420     uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
5421 
5422     uint32_t sigIndex;
5423     Nothing callee_;
5424     if (oldStyle) {
5425         if (!iter_.readOldCallIndirect(&sigIndex))
5426             return false;
5427     } else {
5428         if (!iter_.readCallIndirect(&sigIndex, &callee_))
5429             return false;
5430     }
5431 
5432     if (deadCode_)
5433         return true;
5434 
5435     sync();
5436 
5437     const SigWithId& sig = mg_.sigs[sigIndex];
5438 
5439     // new style: Stack: ... arg1 .. argn callee
5440     // old style: Stack: ... callee arg1 .. argn
5441 
5442     uint32_t numArgs = sig.args().length();
5443     size_t stackSpace = stackConsumed(numArgs + 1);
5444 
5445     // The arguments must be at the stack top for emitCallArgs, so pop the
5446     // callee if it is on top.  Note this only pops the compiler's stack,
5447     // not the CPU stack.
5448 
5449     Stk callee = oldStyle ? peek(numArgs) : stk_.popCopy();
5450 
5451     FunctionCall baselineCall(lineOrBytecode);
5452     beginCall(baselineCall, UseABI::Wasm, InterModule::True);
5453 
5454     if (!emitCallArgs(sig.args(), baselineCall))
5455         return false;
5456 
5457     if (oldStyle) {
5458         if (!iter_.readOldCallIndirectCallee(&callee_))
5459             return false;
5460     }
5461 
5462     if (!iter_.readCallReturn(sig.ret()))
5463         return false;
5464 
5465     callIndirect(sigIndex, callee, baselineCall);
5466 
5467     endCall(baselineCall);
5468 
5469     // For new style calls, the callee was popped off the compiler's
5470     // stack above.
5471 
5472     popValueStackBy(oldStyle ? numArgs + 1 : numArgs);
5473 
5474     // TODO / OPTIMIZE (bug 1316827): It would be better to merge this
5475     // freeStack() into the one in endCall, if we can.
5476 
5477     masm.freeStack(stackSpace);
5478 
5479     if (!IsVoid(sig.ret()))
5480         pushReturned(baselineCall, sig.ret());
5481 
5482     return true;
5483 }
5484 
5485 bool
emitCommonMathCall(uint32_t lineOrBytecode,SymbolicAddress callee,ValTypeVector & signature,ExprType retType)5486 BaseCompiler::emitCommonMathCall(uint32_t lineOrBytecode, SymbolicAddress callee,
5487                                  ValTypeVector& signature, ExprType retType)
5488 {
5489     sync();
5490 
5491     uint32_t numArgs = signature.length();
5492     size_t stackSpace = stackConsumed(numArgs);
5493 
5494     FunctionCall baselineCall(lineOrBytecode);
5495     beginCall(baselineCall, UseABI::System, InterModule::False);
5496 
5497     if (!emitCallArgs(signature, baselineCall))
5498         return false;
5499 
5500     if (!iter_.readCallReturn(retType))
5501       return false;
5502 
5503     builtinCall(callee, baselineCall);
5504 
5505     endCall(baselineCall);
5506 
5507     // TODO / OPTIMIZE (bug 1316827): It would be better to merge this
5508     // freeStack() into the one in endCall, if we can.
5509 
5510     popValueStackBy(numArgs);
5511     masm.freeStack(stackSpace);
5512 
5513     pushReturned(baselineCall, retType);
5514 
5515     return true;
5516 }
5517 
5518 bool
emitUnaryMathBuiltinCall(SymbolicAddress callee,ValType operandType)5519 BaseCompiler::emitUnaryMathBuiltinCall(SymbolicAddress callee, ValType operandType)
5520 {
5521     uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
5522 
5523     if (deadCode_)
5524         return true;
5525 
5526     return emitCommonMathCall(lineOrBytecode, callee,
5527                               operandType == ValType::F32 ? SigF_ : SigD_,
5528                               operandType == ValType::F32 ? ExprType::F32 : ExprType::F64);
5529 }
5530 
5531 bool
emitBinaryMathBuiltinCall(SymbolicAddress callee,ValType operandType)5532 BaseCompiler::emitBinaryMathBuiltinCall(SymbolicAddress callee, ValType operandType)
5533 {
5534     MOZ_ASSERT(operandType == ValType::F64);
5535 
5536     uint32_t lineOrBytecode = 0;
5537     if (callee == SymbolicAddress::ModD) {
5538         // Not actually a call in the binary representation
5539     } else {
5540         lineOrBytecode = readCallSiteLineOrBytecode();
5541     }
5542 
5543     if (deadCode_)
5544         return true;
5545 
5546     return emitCommonMathCall(lineOrBytecode, callee, SigDD_, ExprType::F64);
5547 }
5548 
5549 #ifdef INT_DIV_I64_CALLOUT
5550 bool
emitDivOrModI64BuiltinCall(SymbolicAddress callee,ValType operandType)5551 BaseCompiler::emitDivOrModI64BuiltinCall(SymbolicAddress callee, ValType operandType)
5552 {
5553     MOZ_ASSERT(operandType == ValType::I64);
5554 
5555     if (deadCode_)
5556         return true;
5557 
5558     sync();
5559 
5560     needI64(abiReturnRegI64);
5561 
5562     RegI32 temp = needI32();
5563     RegI64 rhs = popI64();
5564     RegI64 srcDest = popI64ToSpecific(abiReturnRegI64);
5565 
5566     Label done;
5567 
5568     checkDivideByZeroI64(rhs);
5569 
5570     if (callee == SymbolicAddress::DivI64)
5571         checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(false));
5572     else if (callee == SymbolicAddress::ModI64)
5573         checkDivideSignedOverflowI64(rhs, srcDest, &done, ZeroOnOverflow(true));
5574 
5575     masm.setupUnalignedABICall(temp.reg);
5576     masm.passABIArg(srcDest.reg.high);
5577     masm.passABIArg(srcDest.reg.low);
5578     masm.passABIArg(rhs.reg.high);
5579     masm.passABIArg(rhs.reg.low);
5580     masm.callWithABI(callee);
5581 
5582     masm.bind(&done);
5583 
5584     freeI32(temp);
5585     freeI64(rhs);
5586     pushI64(srcDest);
5587 
5588     return true;
5589 }
5590 #endif // INT_DIV_I64_CALLOUT
5591 
5592 #ifdef I64_TO_FLOAT_CALLOUT
5593 bool
emitConvertInt64ToFloatingCallout(SymbolicAddress callee,ValType operandType,ValType resultType)5594 BaseCompiler::emitConvertInt64ToFloatingCallout(SymbolicAddress callee, ValType operandType,
5595                                                 ValType resultType)
5596 {
5597     sync();
5598 
5599     RegI32 temp = needI32();
5600     RegI64 input = popI64();
5601 
5602     FunctionCall call(0);
5603 
5604     masm.setupUnalignedABICall(temp.reg);
5605 # ifdef JS_NUNBOX32
5606     masm.passABIArg(input.reg.high);
5607     masm.passABIArg(input.reg.low);
5608 # else
5609     MOZ_CRASH("BaseCompiler platform hook: emitConvertInt64ToFloatingCallout");
5610 # endif
5611     masm.callWithABI(callee, MoveOp::DOUBLE);
5612 
5613     freeI32(temp);
5614     freeI64(input);
5615 
5616     RegF64 rv = captureReturnedF64(call);
5617 
5618     if (resultType == ValType::F32) {
5619         RegF32 rv2 = needF32();
5620         masm.convertDoubleToFloat32(rv.reg, rv2.reg);
5621         freeF64(rv);
5622         pushF32(rv2);
5623     } else {
5624         pushF64(rv);
5625     }
5626 
5627     return true;
5628 }
5629 #endif // I64_TO_FLOAT_CALLOUT
5630 
5631 #ifdef FLOAT_TO_I64_CALLOUT
5632 // `Callee` always takes a double, so a float32 input must be converted.
5633 bool
emitConvertFloatingToInt64Callout(SymbolicAddress callee,ValType operandType,ValType resultType)5634 BaseCompiler::emitConvertFloatingToInt64Callout(SymbolicAddress callee, ValType operandType,
5635                                                 ValType resultType)
5636 {
5637     RegF64 doubleInput;
5638     if (operandType == ValType::F32) {
5639         doubleInput = needF64();
5640         RegF32 input = popF32();
5641         masm.convertFloat32ToDouble(input.reg, doubleInput.reg);
5642         freeF32(input);
5643     } else {
5644         doubleInput = popF64();
5645     }
5646 
5647     // We may need the value after the call for the ool check.
5648     RegF64 otherReg = needF64();
5649     moveF64(doubleInput, otherReg);
5650     pushF64(otherReg);
5651 
5652     sync();
5653 
5654     RegI32 temp = needI32();
5655     FunctionCall call(0);
5656 
5657     masm.setupUnalignedABICall(temp.reg);
5658     masm.passABIArg(doubleInput.reg, MoveOp::DOUBLE);
5659     masm.callWithABI(callee);
5660 
5661     freeI32(temp);
5662     freeF64(doubleInput);
5663 
5664     RegI64 rv = captureReturnedI64();
5665 
5666     RegF64 inputVal = popF64();
5667 
5668     bool isUnsigned = callee == SymbolicAddress::TruncateDoubleToUint64;
5669 
5670     // The OOL check just succeeds or fails, it does not generate a value.
5671     OutOfLineCode* ool = new (alloc_) OutOfLineTruncateCheckF32OrF64ToI64(AnyReg(inputVal),
5672                                                                           isUnsigned,
5673                                                                           trapOffset());
5674     ool = addOutOfLineCode(ool);
5675     if (!ool)
5676         return false;
5677 
5678     masm.branch64(Assembler::Equal, rv.reg, Imm64(0x8000000000000000), ool->entry());
5679     masm.bind(ool->rejoin());
5680 
5681     pushI64(rv);
5682     freeF64(inputVal);
5683 
5684     return true;
5685 }
5686 #endif // FLOAT_TO_I64_CALLOUT
5687 
5688 bool
emitGetLocal()5689 BaseCompiler::emitGetLocal()
5690 {
5691     uint32_t slot;
5692     if (!iter_.readGetLocal(locals_, &slot))
5693         return false;
5694 
5695     if (deadCode_)
5696         return true;
5697 
5698     // Local loads are pushed unresolved, ie, they may be deferred
5699     // until needed, until they may be affected by a store, or until a
5700     // sync.  This is intended to reduce register pressure.
5701 
5702     switch (locals_[slot]) {
5703       case ValType::I32:
5704         pushLocalI32(slot);
5705         break;
5706       case ValType::I64:
5707         pushLocalI64(slot);
5708         break;
5709       case ValType::F64:
5710         pushLocalF64(slot);
5711         break;
5712       case ValType::F32:
5713         pushLocalF32(slot);
5714         break;
5715       default:
5716         MOZ_CRASH("Local variable type");
5717     }
5718 
5719     return true;
5720 }
5721 
5722 bool
emitSetLocal()5723 BaseCompiler::emitSetLocal()
5724 {
5725     uint32_t slot;
5726     Nothing unused_value;
5727     if (!iter_.readSetLocal(locals_, &slot, &unused_value))
5728         return false;
5729 
5730     if (deadCode_)
5731         return true;
5732 
5733     switch (locals_[slot]) {
5734       case ValType::I32: {
5735         RegI32 rv = popI32();
5736         syncLocal(slot);
5737         storeToFrameI32(rv.reg, frameOffsetFromSlot(slot, MIRType::Int32));
5738         freeI32(rv);
5739         break;
5740       }
5741       case ValType::I64: {
5742         RegI64 rv = popI64();
5743         syncLocal(slot);
5744         storeToFrameI64(rv.reg, frameOffsetFromSlot(slot, MIRType::Int64));
5745         freeI64(rv);
5746         break;
5747       }
5748       case ValType::F64: {
5749         RegF64 rv = popF64();
5750         syncLocal(slot);
5751         storeToFrameF64(rv.reg, frameOffsetFromSlot(slot, MIRType::Double));
5752         freeF64(rv);
5753         break;
5754       }
5755       case ValType::F32: {
5756         RegF32 rv = popF32();
5757         syncLocal(slot);
5758         storeToFrameF32(rv.reg, frameOffsetFromSlot(slot, MIRType::Float32));
5759         freeF32(rv);
5760         break;
5761       }
5762       default:
5763         MOZ_CRASH("Local variable type");
5764     }
5765 
5766     return true;
5767 }
5768 
5769 bool
emitTeeLocal()5770 BaseCompiler::emitTeeLocal()
5771 {
5772     uint32_t slot;
5773     Nothing unused_value;
5774     if (!iter_.readTeeLocal(locals_, &slot, &unused_value))
5775         return false;
5776 
5777     if (deadCode_)
5778         return true;
5779 
5780     switch (locals_[slot]) {
5781       case ValType::I32: {
5782         RegI32 rv = popI32();
5783         syncLocal(slot);
5784         storeToFrameI32(rv.reg, frameOffsetFromSlot(slot, MIRType::Int32));
5785         pushI32(rv);
5786         break;
5787       }
5788       case ValType::I64: {
5789         RegI64 rv = popI64();
5790         syncLocal(slot);
5791         storeToFrameI64(rv.reg, frameOffsetFromSlot(slot, MIRType::Int64));
5792         pushI64(rv);
5793         break;
5794       }
5795       case ValType::F64: {
5796         RegF64 rv = popF64();
5797         syncLocal(slot);
5798         storeToFrameF64(rv.reg, frameOffsetFromSlot(slot, MIRType::Double));
5799         pushF64(rv);
5800         break;
5801       }
5802       case ValType::F32: {
5803         RegF32 rv = popF32();
5804         syncLocal(slot);
5805         storeToFrameF32(rv.reg, frameOffsetFromSlot(slot, MIRType::Float32));
5806         pushF32(rv);
5807         break;
5808       }
5809       default:
5810         MOZ_CRASH("Local variable type");
5811     }
5812 
5813     return true;
5814 }
5815 
5816 bool
emitGetGlobal()5817 BaseCompiler::emitGetGlobal()
5818 {
5819     uint32_t id;
5820     if (!iter_.readGetGlobal(mg_.globals, &id))
5821         return false;
5822 
5823     if (deadCode_)
5824         return true;
5825 
5826     const GlobalDesc& global = mg_.globals[id];
5827 
5828     if (global.isConstant()) {
5829         Val value = global.constantValue();
5830         switch (value.type()) {
5831           case ValType::I32:
5832             pushI32(value.i32());
5833             break;
5834           case ValType::I64:
5835             pushI64(value.i64());
5836             break;
5837           case ValType::F32:
5838             pushF32(value.f32());
5839             break;
5840           case ValType::F64:
5841             pushF64(value.f64());
5842             break;
5843           default:
5844             MOZ_CRASH("Global constant type");
5845         }
5846         return true;
5847     }
5848 
5849     switch (global.type()) {
5850       case ValType::I32: {
5851         RegI32 rv = needI32();
5852         loadGlobalVarI32(global.offset(), rv);
5853         pushI32(rv);
5854         break;
5855       }
5856       case ValType::I64: {
5857         RegI64 rv = needI64();
5858         loadGlobalVarI64(global.offset(), rv);
5859         pushI64(rv);
5860         break;
5861       }
5862       case ValType::F32: {
5863         RegF32 rv = needF32();
5864         loadGlobalVarF32(global.offset(), rv);
5865         pushF32(rv);
5866         break;
5867       }
5868       case ValType::F64: {
5869         RegF64 rv = needF64();
5870         loadGlobalVarF64(global.offset(), rv);
5871         pushF64(rv);
5872         break;
5873       }
5874       default:
5875         MOZ_CRASH("Global variable type");
5876         break;
5877     }
5878     return true;
5879 }
5880 
5881 bool
emitSetGlobal()5882 BaseCompiler::emitSetGlobal()
5883 {
5884     uint32_t id;
5885     Nothing unused_value;
5886     if (!iter_.readSetGlobal(mg_.globals, &id, &unused_value))
5887         return false;
5888 
5889     if (deadCode_)
5890         return true;
5891 
5892     const GlobalDesc& global = mg_.globals[id];
5893 
5894     switch (global.type()) {
5895       case ValType::I32: {
5896         RegI32 rv = popI32();
5897         storeGlobalVarI32(global.offset(), rv);
5898         freeI32(rv);
5899         break;
5900       }
5901       case ValType::I64: {
5902         RegI64 rv = popI64();
5903         storeGlobalVarI64(global.offset(), rv);
5904         freeI64(rv);
5905         break;
5906       }
5907       case ValType::F32: {
5908         RegF32 rv = popF32();
5909         storeGlobalVarF32(global.offset(), rv);
5910         freeF32(rv);
5911         break;
5912       }
5913       case ValType::F64: {
5914         RegF64 rv = popF64();
5915         storeGlobalVarF64(global.offset(), rv);
5916         freeF64(rv);
5917         break;
5918       }
5919       default:
5920         MOZ_CRASH("Global variable type");
5921         break;
5922     }
5923     return true;
5924 }
5925 
5926 bool
emitTeeGlobal()5927 BaseCompiler::emitTeeGlobal()
5928 {
5929     uint32_t id;
5930     Nothing unused_value;
5931     if (!iter_.readTeeGlobal(mg_.globals, &id, &unused_value))
5932         return false;
5933 
5934     if (deadCode_)
5935         return true;
5936 
5937     const GlobalDesc& global = mg_.globals[id];
5938 
5939     switch (global.type()) {
5940       case ValType::I32: {
5941         RegI32 rv = popI32();
5942         storeGlobalVarI32(global.offset(), rv);
5943         pushI32(rv);
5944         break;
5945       }
5946       case ValType::I64: {
5947         RegI64 rv = popI64();
5948         storeGlobalVarI64(global.offset(), rv);
5949         pushI64(rv);
5950         break;
5951       }
5952       case ValType::F32: {
5953         RegF32 rv = popF32();
5954         storeGlobalVarF32(global.offset(), rv);
5955         pushF32(rv);
5956         break;
5957       }
5958       case ValType::F64: {
5959         RegF64 rv = popF64();
5960         storeGlobalVarF64(global.offset(), rv);
5961         pushF64(rv);
5962         break;
5963       }
5964       default:
5965         MOZ_CRASH("Global variable type");
5966         break;
5967     }
5968     return true;
5969 }
5970 
5971 bool
emitLoad(ValType type,Scalar::Type viewType)5972 BaseCompiler::emitLoad(ValType type, Scalar::Type viewType)
5973 {
5974     LinearMemoryAddress<Nothing> addr;
5975     if (!iter_.readLoad(type, Scalar::byteSize(viewType), &addr))
5976         return false;
5977 
5978     if (deadCode_)
5979         return true;
5980 
5981     // TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
5982     // accesses below the minimum heap length.
5983 
5984     MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
5985 
5986     size_t temps = loadStoreTemps(access);
5987     RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
5988     RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
5989 
5990     switch (type) {
5991       case ValType::I32: {
5992         RegI32 rp = popI32();
5993 #ifdef JS_CODEGEN_ARM
5994         RegI32 rv = access.isUnaligned() ? needI32() : rp;
5995 #else
5996         RegI32 rv = rp;
5997 #endif
5998         if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
5999             return false;
6000         pushI32(rv);
6001         if (rp != rv)
6002             freeI32(rp);
6003         break;
6004       }
6005       case ValType::I64: {
6006         RegI64 rv;
6007         RegI32 rp;
6008 #ifdef JS_CODEGEN_X86
6009         rv = abiReturnRegI64;
6010         needI64(rv);
6011         rp = popI32();
6012 #else
6013         rp = popI32();
6014         rv = needI64();
6015 #endif
6016         if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
6017             return false;
6018         pushI64(rv);
6019         freeI32(rp);
6020         break;
6021       }
6022       case ValType::F32: {
6023         RegI32 rp = popI32();
6024         RegF32 rv = needF32();
6025         if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
6026             return false;
6027         pushF32(rv);
6028         freeI32(rp);
6029         break;
6030       }
6031       case ValType::F64: {
6032         RegI32 rp = popI32();
6033         RegF64 rv = needF64();
6034         if (!load(access, rp, AnyReg(rv), tmp1, tmp2))
6035             return false;
6036         pushF64(rv);
6037         freeI32(rp);
6038         break;
6039       }
6040       default:
6041         MOZ_CRASH("load type");
6042         break;
6043     }
6044 
6045     if (temps >= 1)
6046         freeI32(tmp1);
6047     if (temps >= 2)
6048         freeI32(tmp2);
6049 
6050     return true;
6051 }
6052 
6053 bool
emitStore(ValType resultType,Scalar::Type viewType)6054 BaseCompiler::emitStore(ValType resultType, Scalar::Type viewType)
6055 {
6056     LinearMemoryAddress<Nothing> addr;
6057     Nothing unused_value;
6058     if (!iter_.readStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
6059         return false;
6060 
6061     if (deadCode_)
6062         return true;
6063 
6064     // TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
6065     // accesses below the minimum heap length.
6066 
6067     MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
6068 
6069     size_t temps = loadStoreTemps(access);
6070     RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
6071     RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
6072 
6073     switch (resultType) {
6074       case ValType::I32: {
6075         RegI32 rp, rv;
6076         pop2xI32(&rp, &rv);
6077         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6078             return false;
6079         freeI32(rp);
6080         freeI32(rv);
6081         break;
6082       }
6083       case ValType::I64: {
6084         RegI64 rv = popI64();
6085         RegI32 rp = popI32();
6086         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6087             return false;
6088         freeI32(rp);
6089         freeI64(rv);
6090         break;
6091       }
6092       case ValType::F32: {
6093         RegF32 rv = popF32();
6094         RegI32 rp = popI32();
6095         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6096             return false;
6097         freeI32(rp);
6098         freeF32(rv);
6099         break;
6100       }
6101       case ValType::F64: {
6102         RegF64 rv = popF64();
6103         RegI32 rp = popI32();
6104         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6105             return false;
6106         freeI32(rp);
6107         freeF64(rv);
6108         break;
6109       }
6110       default:
6111         MOZ_CRASH("store type");
6112         break;
6113     }
6114 
6115     if (temps >= 1)
6116         freeI32(tmp1);
6117     if (temps >= 2)
6118         freeI32(tmp2);
6119 
6120     return true;
6121 }
6122 
6123 bool
emitTeeStore(ValType resultType,Scalar::Type viewType)6124 BaseCompiler::emitTeeStore(ValType resultType, Scalar::Type viewType)
6125 {
6126     LinearMemoryAddress<Nothing> addr;
6127     Nothing unused_value;
6128     if (!iter_.readTeeStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
6129         return false;
6130 
6131     if (deadCode_)
6132         return true;
6133 
6134     // TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
6135     // accesses below the minimum heap length.
6136 
6137     MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
6138 
6139     size_t temps = loadStoreTemps(access);
6140     RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
6141     RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
6142 
6143     switch (resultType) {
6144       case ValType::I32: {
6145         RegI32 rp, rv;
6146         pop2xI32(&rp, &rv);
6147         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6148             return false;
6149         freeI32(rp);
6150         pushI32(rv);
6151         break;
6152       }
6153       case ValType::I64: {
6154         RegI64 rv = popI64();
6155         RegI32 rp = popI32();
6156         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6157             return false;
6158         freeI32(rp);
6159         pushI64(rv);
6160         break;
6161       }
6162       case ValType::F32: {
6163         RegF32 rv = popF32();
6164         RegI32 rp = popI32();
6165         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6166             return false;
6167         freeI32(rp);
6168         pushF32(rv);
6169         break;
6170       }
6171       case ValType::F64: {
6172         RegF64 rv = popF64();
6173         RegI32 rp = popI32();
6174         if (!store(access, rp, AnyReg(rv), tmp1, tmp2))
6175             return false;
6176         freeI32(rp);
6177         pushF64(rv);
6178         break;
6179       }
6180       default:
6181         MOZ_CRASH("store type");
6182         break;
6183     }
6184 
6185     if (temps >= 1)
6186         freeI32(tmp1);
6187     if (temps >= 2)
6188         freeI32(tmp2);
6189 
6190     return true;
6191 }
6192 
6193 bool
emitSelect()6194 BaseCompiler::emitSelect()
6195 {
6196     ValType type;
6197     Nothing unused_trueValue;
6198     Nothing unused_falseValue;
6199     Nothing unused_condition;
6200     if (!iter_.readSelect(&type, &unused_trueValue, &unused_falseValue, &unused_condition))
6201         return false;
6202 
6203     if (deadCode_)
6204         return true;
6205 
6206     // I32 condition on top, then false, then true.
6207 
6208     RegI32 rc = popI32();
6209     switch (type) {
6210       case ValType::I32: {
6211         Label done;
6212         RegI32 r0, r1;
6213         pop2xI32(&r0, &r1);
6214         masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
6215         moveI32(r1, r0);
6216         masm.bind(&done);
6217         freeI32(r1);
6218         pushI32(r0);
6219         break;
6220       }
6221       case ValType::I64: {
6222         Label done;
6223         RegI64 r0, r1;
6224         pop2xI64(&r0, &r1);
6225         masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
6226         moveI64(r1, r0);
6227         masm.bind(&done);
6228         freeI64(r1);
6229         pushI64(r0);
6230         break;
6231       }
6232       case ValType::F32: {
6233         Label done;
6234         RegF32 r0, r1;
6235         pop2xF32(&r0, &r1);
6236         masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
6237         moveF32(r1, r0);
6238         masm.bind(&done);
6239         freeF32(r1);
6240         pushF32(r0);
6241         break;
6242       }
6243       case ValType::F64: {
6244         Label done;
6245         RegF64 r0, r1;
6246         pop2xF64(&r0, &r1);
6247         masm.branch32(Assembler::NotEqual, rc.reg, Imm32(0), &done);
6248         moveF64(r1, r0);
6249         masm.bind(&done);
6250         freeF64(r1);
6251         pushF64(r0);
6252         break;
6253       }
6254       default: {
6255         MOZ_CRASH("select type");
6256       }
6257     }
6258     freeI32(rc);
6259 
6260     return true;
6261 }
6262 
6263 void
emitCompareI32(JSOp compareOp,MCompare::CompareType compareType)6264 BaseCompiler::emitCompareI32(JSOp compareOp, MCompare::CompareType compareType)
6265 {
6266     // TODO / OPTIMIZE (bug 1286816): if we want to generate good code for
6267     // boolean operators for control it is possible to delay generating code
6268     // here by pushing a compare operation on the stack, after all it is
6269     // side-effect free.  The popping code for br_if will handle it differently,
6270     // but other popI32() will just force code generation.
6271     //
6272     // TODO / OPTIMIZE (bug 1286816): Comparisons against constants using the
6273     // same popConstant pattern as for add().
6274 
6275     MOZ_ASSERT(compareType == MCompare::Compare_Int32 || compareType == MCompare::Compare_UInt32);
6276     RegI32 r0, r1;
6277     pop2xI32(&r0, &r1);
6278     bool u = compareType == MCompare::Compare_UInt32;
6279     switch (compareOp) {
6280       case JSOP_EQ:
6281         masm.cmp32Set(Assembler::Equal, r0.reg, r1.reg, r0.reg);
6282         break;
6283       case JSOP_NE:
6284         masm.cmp32Set(Assembler::NotEqual, r0.reg, r1.reg, r0.reg);
6285         break;
6286       case JSOP_LE:
6287         masm.cmp32Set(u ? Assembler::BelowOrEqual : Assembler::LessThanOrEqual, r0.reg, r1.reg, r0.reg);
6288         break;
6289       case JSOP_LT:
6290         masm.cmp32Set(u ? Assembler::Below : Assembler::LessThan, r0.reg, r1.reg, r0.reg);
6291         break;
6292       case JSOP_GE:
6293         masm.cmp32Set(u ? Assembler::AboveOrEqual : Assembler::GreaterThanOrEqual, r0.reg, r1.reg, r0.reg);
6294         break;
6295       case JSOP_GT:
6296         masm.cmp32Set(u ? Assembler::Above : Assembler::GreaterThan, r0.reg, r1.reg, r0.reg);
6297         break;
6298       default:
6299         MOZ_CRASH("Compiler bug: Unexpected compare opcode");
6300     }
6301     freeI32(r1);
6302     pushI32(r0);
6303 }
6304 
6305 void
emitCompareI64(JSOp compareOp,MCompare::CompareType compareType)6306 BaseCompiler::emitCompareI64(JSOp compareOp, MCompare::CompareType compareType)
6307 {
6308     MOZ_ASSERT(compareType == MCompare::Compare_Int64 || compareType == MCompare::Compare_UInt64);
6309     RegI64 r0, r1;
6310     pop2xI64(&r0, &r1);
6311     RegI32 i0(fromI64(r0));
6312     bool u = compareType == MCompare::Compare_UInt64;
6313     switch (compareOp) {
6314       case JSOP_EQ:
6315         cmp64Set(Assembler::Equal, r0, r1, i0);
6316         break;
6317       case JSOP_NE:
6318         cmp64Set(Assembler::NotEqual, r0, r1, i0);
6319         break;
6320       case JSOP_LE:
6321         cmp64Set(u ? Assembler::BelowOrEqual : Assembler::LessThanOrEqual, r0, r1, i0);
6322         break;
6323       case JSOP_LT:
6324         cmp64Set(u ? Assembler::Below : Assembler::LessThan, r0, r1, i0);
6325         break;
6326       case JSOP_GE:
6327         cmp64Set(u ? Assembler::AboveOrEqual : Assembler::GreaterThanOrEqual, r0, r1, i0);
6328         break;
6329       case JSOP_GT:
6330         cmp64Set(u ? Assembler::Above : Assembler::GreaterThan, r0, r1, i0);
6331         break;
6332       default:
6333         MOZ_CRASH("Compiler bug: Unexpected compare opcode");
6334     }
6335     freeI64(r1);
6336     freeI64Except(r0, i0);
6337     pushI32(i0);
6338 }
6339 
6340 void
emitCompareF32(JSOp compareOp,MCompare::CompareType compareType)6341 BaseCompiler::emitCompareF32(JSOp compareOp, MCompare::CompareType compareType)
6342 {
6343     MOZ_ASSERT(compareType == MCompare::Compare_Float32);
6344     Label across;
6345     RegF32 r0, r1;
6346     pop2xF32(&r0, &r1);
6347     RegI32 i0 = needI32();
6348     masm.mov(ImmWord(1), i0.reg);
6349     switch (compareOp) {
6350       case JSOP_EQ:
6351         masm.branchFloat(Assembler::DoubleEqual, r0.reg, r1.reg, &across);
6352         break;
6353       case JSOP_NE:
6354         masm.branchFloat(Assembler::DoubleNotEqualOrUnordered, r0.reg, r1.reg, &across);
6355         break;
6356       case JSOP_LE:
6357         masm.branchFloat(Assembler::DoubleLessThanOrEqual, r0.reg, r1.reg, &across);
6358         break;
6359       case JSOP_LT:
6360         masm.branchFloat(Assembler::DoubleLessThan, r0.reg, r1.reg, &across);
6361         break;
6362       case JSOP_GE:
6363         masm.branchFloat(Assembler::DoubleGreaterThanOrEqual, r0.reg, r1.reg, &across);
6364         break;
6365       case JSOP_GT:
6366         masm.branchFloat(Assembler::DoubleGreaterThan, r0.reg, r1.reg, &across);
6367         break;
6368       default:
6369         MOZ_CRASH("Compiler bug: Unexpected compare opcode");
6370     }
6371     masm.mov(ImmWord(0), i0.reg);
6372     masm.bind(&across);
6373     freeF32(r0);
6374     freeF32(r1);
6375     pushI32(i0);
6376 }
6377 
6378 void
emitCompareF64(JSOp compareOp,MCompare::CompareType compareType)6379 BaseCompiler::emitCompareF64(JSOp compareOp, MCompare::CompareType compareType)
6380 {
6381     MOZ_ASSERT(compareType == MCompare::Compare_Double);
6382     Label across;
6383     RegF64 r0, r1;
6384     pop2xF64(&r0, &r1);
6385     RegI32 i0 = needI32();
6386     masm.mov(ImmWord(1), i0.reg);
6387     switch (compareOp) {
6388       case JSOP_EQ:
6389         masm.branchDouble(Assembler::DoubleEqual, r0.reg, r1.reg, &across);
6390         break;
6391       case JSOP_NE:
6392         masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, r0.reg, r1.reg, &across);
6393         break;
6394       case JSOP_LE:
6395         masm.branchDouble(Assembler::DoubleLessThanOrEqual, r0.reg, r1.reg, &across);
6396         break;
6397       case JSOP_LT:
6398         masm.branchDouble(Assembler::DoubleLessThan, r0.reg, r1.reg, &across);
6399         break;
6400       case JSOP_GE:
6401         masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, r0.reg, r1.reg, &across);
6402         break;
6403       case JSOP_GT:
6404         masm.branchDouble(Assembler::DoubleGreaterThan, r0.reg, r1.reg, &across);
6405         break;
6406       default:
6407         MOZ_CRASH("Compiler bug: Unexpected compare opcode");
6408     }
6409     masm.mov(ImmWord(0), i0.reg);
6410     masm.bind(&across);
6411     freeF64(r0);
6412     freeF64(r1);
6413     pushI32(i0);
6414 }
6415 
6416 bool
emitTeeStoreWithCoercion(ValType resultType,Scalar::Type viewType)6417 BaseCompiler::emitTeeStoreWithCoercion(ValType resultType, Scalar::Type viewType)
6418 {
6419     LinearMemoryAddress<Nothing> addr;
6420     Nothing unused_value;
6421     if (!iter_.readTeeStore(resultType, Scalar::byteSize(viewType), &addr, &unused_value))
6422         return false;
6423 
6424     if (deadCode_)
6425         return true;
6426 
6427     // TODO / OPTIMIZE (bug 1316831): Disable bounds checking on constant
6428     // accesses below the minimum heap length.
6429 
6430     MemoryAccessDesc access(viewType, addr.align, addr.offset, trapIfNotAsmJS());
6431 
6432     size_t temps = loadStoreTemps(access);
6433     RegI32 tmp1 = temps >= 1 ? needI32() : invalidI32();
6434     RegI32 tmp2 = temps >= 2 ? needI32() : invalidI32();
6435 
6436     if (resultType == ValType::F32 && viewType == Scalar::Float64) {
6437         RegF32 rv = popF32();
6438         RegF64 rw = needF64();
6439         masm.convertFloat32ToDouble(rv.reg, rw.reg);
6440         RegI32 rp = popI32();
6441         if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
6442             return false;
6443         pushF32(rv);
6444         freeI32(rp);
6445         freeF64(rw);
6446     }
6447     else if (resultType == ValType::F64 && viewType == Scalar::Float32) {
6448         RegF64 rv = popF64();
6449         RegF32 rw = needF32();
6450         masm.convertDoubleToFloat32(rv.reg, rw.reg);
6451         RegI32 rp = popI32();
6452         if (!store(access, rp, AnyReg(rw), tmp1, tmp2))
6453             return false;
6454         pushF64(rv);
6455         freeI32(rp);
6456         freeF32(rw);
6457     }
6458     else
6459         MOZ_CRASH("unexpected coerced store");
6460 
6461     if (temps >= 1)
6462         freeI32(tmp1);
6463     if (temps >= 2)
6464         freeI32(tmp2);
6465 
6466     return true;
6467 }
6468 
6469 bool
emitGrowMemory()6470 BaseCompiler::emitGrowMemory()
6471 {
6472     uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
6473 
6474     Nothing arg;
6475     if (!iter_.readGrowMemory(&arg))
6476         return false;
6477 
6478     if (deadCode_)
6479         return true;
6480 
6481     sync();
6482 
6483     uint32_t numArgs = 1;
6484     size_t stackSpace = stackConsumed(numArgs);
6485 
6486     FunctionCall baselineCall(lineOrBytecode);
6487     beginCall(baselineCall, UseABI::System, InterModule::True);
6488 
6489     ABIArg instanceArg = reservePointerArgument(baselineCall);
6490 
6491     startCallArgs(baselineCall, stackArgAreaSize(SigI_));
6492     passArg(baselineCall, ValType::I32, peek(0));
6493     builtinInstanceMethodCall(SymbolicAddress::GrowMemory, instanceArg, baselineCall);
6494     endCall(baselineCall);
6495 
6496     popValueStackBy(numArgs);
6497     masm.freeStack(stackSpace);
6498 
6499     pushReturned(baselineCall, ExprType::I32);
6500 
6501     return true;
6502 }
6503 
6504 bool
emitCurrentMemory()6505 BaseCompiler::emitCurrentMemory()
6506 {
6507     uint32_t lineOrBytecode = readCallSiteLineOrBytecode();
6508 
6509     if (!iter_.readCurrentMemory())
6510         return false;
6511 
6512     if (deadCode_)
6513         return true;
6514 
6515     sync();
6516 
6517     FunctionCall baselineCall(lineOrBytecode);
6518     beginCall(baselineCall, UseABI::System, InterModule::False);
6519 
6520     ABIArg instanceArg = reservePointerArgument(baselineCall);
6521 
6522     startCallArgs(baselineCall, stackArgAreaSize(Sig_));
6523     builtinInstanceMethodCall(SymbolicAddress::CurrentMemory, instanceArg, baselineCall);
6524     endCall(baselineCall);
6525 
6526     pushReturned(baselineCall, ExprType::I32);
6527 
6528     return true;
6529 }
6530 
6531 bool
emitBody()6532 BaseCompiler::emitBody()
6533 {
6534     uint32_t overhead = 0;
6535 
6536     for (;;) {
6537 
6538         Nothing unused_a, unused_b;
6539 
6540 #define emitBinary(doEmit, type) \
6541         iter_.readBinary(type, &unused_a, &unused_b) && (deadCode_ || (doEmit(), true))
6542 
6543 #define emitUnary(doEmit, type) \
6544         iter_.readUnary(type, &unused_a) && (deadCode_ || (doEmit(), true))
6545 
6546 #define emitComparison(doEmit, operandType, compareOp, compareType) \
6547         iter_.readComparison(operandType, &unused_a, &unused_b) && \
6548             (deadCode_ || (doEmit(compareOp, compareType), true))
6549 
6550 #define emitConversion(doEmit, inType, outType) \
6551         iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || (doEmit(), true))
6552 
6553 #define emitConversionOOM(doEmit, inType, outType) \
6554         iter_.readConversion(inType, outType, &unused_a) && (deadCode_ || doEmit())
6555 
6556 #define emitCalloutConversionOOM(doEmit, symbol, inType, outType) \
6557         iter_.readConversion(inType, outType, &unused_a) && \
6558             (deadCode_ || doEmit(symbol, inType, outType))
6559 
6560 #define CHECK(E)      if (!(E)) goto done
6561 #define NEXT()        continue
6562 #define CHECK_NEXT(E) if (!(E)) goto done; continue
6563 
6564         // TODO / EVALUATE (bug 1316845): Not obvious that this attempt at
6565         // reducing overhead is really paying off relative to making the check
6566         // every iteration.
6567 
6568         if (overhead == 0) {
6569             // Check every 50 expressions -- a happy medium between
6570             // memory usage and checking overhead.
6571             overhead = 50;
6572 
6573             // Checking every 50 expressions should be safe, as the
6574             // baseline JIT does very little allocation per expression.
6575             CHECK(alloc_.ensureBallast());
6576 
6577             // The pushiest opcode is LOOP, which pushes two values
6578             // per instance.
6579             CHECK(stk_.reserve(stk_.length() + overhead * 2));
6580         }
6581 
6582         overhead--;
6583 
6584         if (done())
6585             return true;
6586 
6587         uint16_t op;
6588         CHECK(iter_.readOp(&op));
6589 
6590         switch (op) {
6591           // Control opcodes
6592           case uint16_t(Op::Nop):
6593             CHECK(iter_.readNop());
6594             NEXT();
6595           case uint16_t(Op::Drop):
6596             CHECK_NEXT(emitDrop());
6597           case uint16_t(Op::Block):
6598             CHECK_NEXT(emitBlock());
6599           case uint16_t(Op::Loop):
6600             CHECK_NEXT(emitLoop());
6601           case uint16_t(Op::If):
6602             CHECK_NEXT(emitIf());
6603           case uint16_t(Op::Else):
6604             CHECK_NEXT(emitElse());
6605           case uint16_t(Op::End):
6606             CHECK_NEXT(emitEnd());
6607           case uint16_t(Op::Br):
6608             CHECK_NEXT(emitBr());
6609           case uint16_t(Op::BrIf):
6610             CHECK_NEXT(emitBrIf());
6611           case uint16_t(Op::BrTable):
6612             CHECK_NEXT(emitBrTable());
6613           case uint16_t(Op::Return):
6614             CHECK_NEXT(emitReturn());
6615           case uint16_t(Op::Unreachable):
6616             CHECK(iter_.readUnreachable());
6617             if (!deadCode_) {
6618                 unreachableTrap();
6619                 deadCode_ = true;
6620                 popValueStackTo(ctl_.back().stackSize);
6621             }
6622             NEXT();
6623 
6624           // Calls
6625           case uint16_t(Op::Call):
6626             CHECK_NEXT(emitCall());
6627           case uint16_t(Op::CallIndirect):
6628             CHECK_NEXT(emitCallIndirect(/* oldStyle = */ false));
6629           case uint16_t(Op::OldCallIndirect):
6630             CHECK_NEXT(emitCallIndirect(/* oldStyle = */ true));
6631 
6632           // Locals and globals
6633           case uint16_t(Op::GetLocal):
6634             CHECK_NEXT(emitGetLocal());
6635           case uint16_t(Op::SetLocal):
6636             CHECK_NEXT(emitSetLocal());
6637           case uint16_t(Op::TeeLocal):
6638             CHECK_NEXT(emitTeeLocal());
6639           case uint16_t(Op::GetGlobal):
6640             CHECK_NEXT(emitGetGlobal());
6641           case uint16_t(Op::SetGlobal):
6642             CHECK_NEXT(emitSetGlobal());
6643           case uint16_t(Op::TeeGlobal):
6644             CHECK_NEXT(emitTeeGlobal());
6645 
6646           // Select
6647           case uint16_t(Op::Select):
6648             CHECK_NEXT(emitSelect());
6649 
6650           // I32
6651           case uint16_t(Op::I32Const): {
6652             int32_t i32;
6653             CHECK(iter_.readI32Const(&i32));
6654             if (!deadCode_)
6655                 pushI32(i32);
6656             NEXT();
6657           }
6658           case uint16_t(Op::I32Add):
6659             CHECK_NEXT(emitBinary(emitAddI32, ValType::I32));
6660           case uint16_t(Op::I32Sub):
6661             CHECK_NEXT(emitBinary(emitSubtractI32, ValType::I32));
6662           case uint16_t(Op::I32Mul):
6663             CHECK_NEXT(emitBinary(emitMultiplyI32, ValType::I32));
6664           case uint16_t(Op::I32DivS):
6665             CHECK_NEXT(emitBinary(emitQuotientI32, ValType::I32));
6666           case uint16_t(Op::I32DivU):
6667             CHECK_NEXT(emitBinary(emitQuotientU32, ValType::I32));
6668           case uint16_t(Op::I32RemS):
6669             CHECK_NEXT(emitBinary(emitRemainderI32, ValType::I32));
6670           case uint16_t(Op::I32RemU):
6671             CHECK_NEXT(emitBinary(emitRemainderU32, ValType::I32));
6672           case uint16_t(Op::I32Min):
6673             CHECK_NEXT(emitBinary(emitMinI32, ValType::I32));
6674           case uint16_t(Op::I32Max):
6675             CHECK_NEXT(emitBinary(emitMaxI32, ValType::I32));
6676           case uint16_t(Op::I32Eqz):
6677             CHECK_NEXT(emitConversion(emitEqzI32, ValType::I32, ValType::I32));
6678           case uint16_t(Op::I32TruncSF32):
6679             CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI32<false>, ValType::F32, ValType::I32));
6680           case uint16_t(Op::I32TruncUF32):
6681             CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI32<true>, ValType::F32, ValType::I32));
6682           case uint16_t(Op::I32TruncSF64):
6683             CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI32<false>, ValType::F64, ValType::I32));
6684           case uint16_t(Op::I32TruncUF64):
6685             CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI32<true>, ValType::F64, ValType::I32));
6686           case uint16_t(Op::I32WrapI64):
6687             CHECK_NEXT(emitConversion(emitWrapI64ToI32, ValType::I64, ValType::I32));
6688           case uint16_t(Op::I32ReinterpretF32):
6689             CHECK_NEXT(emitConversion(emitReinterpretF32AsI32, ValType::F32, ValType::I32));
6690           case uint16_t(Op::I32Clz):
6691             CHECK_NEXT(emitUnary(emitClzI32, ValType::I32));
6692           case uint16_t(Op::I32Ctz):
6693             CHECK_NEXT(emitUnary(emitCtzI32, ValType::I32));
6694           case uint16_t(Op::I32Popcnt):
6695             CHECK_NEXT(emitUnary(emitPopcntI32, ValType::I32));
6696           case uint16_t(Op::I32Abs):
6697             CHECK_NEXT(emitUnary(emitAbsI32, ValType::I32));
6698           case uint16_t(Op::I32Neg):
6699             CHECK_NEXT(emitUnary(emitNegateI32, ValType::I32));
6700           case uint16_t(Op::I32Or):
6701             CHECK_NEXT(emitBinary(emitOrI32, ValType::I32));
6702           case uint16_t(Op::I32And):
6703             CHECK_NEXT(emitBinary(emitAndI32, ValType::I32));
6704           case uint16_t(Op::I32Xor):
6705             CHECK_NEXT(emitBinary(emitXorI32, ValType::I32));
6706           case uint16_t(Op::I32Shl):
6707             CHECK_NEXT(emitBinary(emitShlI32, ValType::I32));
6708           case uint16_t(Op::I32ShrS):
6709             CHECK_NEXT(emitBinary(emitShrI32, ValType::I32));
6710           case uint16_t(Op::I32ShrU):
6711             CHECK_NEXT(emitBinary(emitShrU32, ValType::I32));
6712           case uint16_t(Op::I32BitNot):
6713             CHECK_NEXT(emitUnary(emitBitNotI32, ValType::I32));
6714           case uint16_t(Op::I32Load8S):
6715             CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int8));
6716           case uint16_t(Op::I32Load8U):
6717             CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint8));
6718           case uint16_t(Op::I32Load16S):
6719             CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int16));
6720           case uint16_t(Op::I32Load16U):
6721             CHECK_NEXT(emitLoad(ValType::I32, Scalar::Uint16));
6722           case uint16_t(Op::I32Load):
6723             CHECK_NEXT(emitLoad(ValType::I32, Scalar::Int32));
6724           case uint16_t(Op::I32Store8):
6725             CHECK_NEXT(emitStore(ValType::I32, Scalar::Int8));
6726           case uint16_t(Op::I32TeeStore8):
6727             CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int8));
6728           case uint16_t(Op::I32Store16):
6729             CHECK_NEXT(emitStore(ValType::I32, Scalar::Int16));
6730           case uint16_t(Op::I32TeeStore16):
6731             CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int16));
6732           case uint16_t(Op::I32Store):
6733             CHECK_NEXT(emitStore(ValType::I32, Scalar::Int32));
6734           case uint16_t(Op::I32TeeStore):
6735             CHECK_NEXT(emitTeeStore(ValType::I32, Scalar::Int32));
6736           case uint16_t(Op::I32Rotr):
6737             CHECK_NEXT(emitBinary(emitRotrI32, ValType::I32));
6738           case uint16_t(Op::I32Rotl):
6739             CHECK_NEXT(emitBinary(emitRotlI32, ValType::I32));
6740 
6741           // I64
6742           case uint16_t(Op::I64Const): {
6743             int64_t i64;
6744             CHECK(iter_.readI64Const(&i64));
6745             if (!deadCode_)
6746                 pushI64(i64);
6747             NEXT();
6748           }
6749           case uint16_t(Op::I64Add):
6750             CHECK_NEXT(emitBinary(emitAddI64, ValType::I64));
6751           case uint16_t(Op::I64Sub):
6752             CHECK_NEXT(emitBinary(emitSubtractI64, ValType::I64));
6753           case uint16_t(Op::I64Mul):
6754             CHECK_NEXT(emitBinary(emitMultiplyI64, ValType::I64));
6755           case uint16_t(Op::I64DivS):
6756 #ifdef INT_DIV_I64_CALLOUT
6757             CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::DivI64, ValType::I64));
6758 #else
6759             CHECK_NEXT(emitBinary(emitQuotientI64, ValType::I64));
6760 #endif
6761           case uint16_t(Op::I64DivU):
6762 #ifdef INT_DIV_I64_CALLOUT
6763             CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::UDivI64, ValType::I64));
6764 #else
6765             CHECK_NEXT(emitBinary(emitQuotientU64, ValType::I64));
6766 #endif
6767           case uint16_t(Op::I64RemS):
6768 #ifdef INT_DIV_I64_CALLOUT
6769             CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::ModI64, ValType::I64));
6770 #else
6771             CHECK_NEXT(emitBinary(emitRemainderI64, ValType::I64));
6772 #endif
6773           case uint16_t(Op::I64RemU):
6774 #ifdef INT_DIV_I64_CALLOUT
6775             CHECK_NEXT(emitDivOrModI64BuiltinCall(SymbolicAddress::UModI64, ValType::I64));
6776 #else
6777             CHECK_NEXT(emitBinary(emitRemainderU64, ValType::I64));
6778 #endif
6779           case uint16_t(Op::I64TruncSF32):
6780 #ifdef FLOAT_TO_I64_CALLOUT
6781             CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
6782                                                 SymbolicAddress::TruncateDoubleToInt64,
6783                                                 ValType::F32, ValType::I64));
6784 #else
6785             CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI64<false>, ValType::F32, ValType::I64));
6786 #endif
6787           case uint16_t(Op::I64TruncUF32):
6788 #ifdef FLOAT_TO_I64_CALLOUT
6789             CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
6790                                                 SymbolicAddress::TruncateDoubleToUint64,
6791                                                 ValType::F32, ValType::I64));
6792 #else
6793             CHECK_NEXT(emitConversionOOM(emitTruncateF32ToI64<true>, ValType::F32, ValType::I64));
6794 #endif
6795           case uint16_t(Op::I64TruncSF64):
6796 #ifdef FLOAT_TO_I64_CALLOUT
6797             CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
6798                                                 SymbolicAddress::TruncateDoubleToInt64,
6799                                                 ValType::F64, ValType::I64));
6800 #else
6801             CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI64<false>, ValType::F64, ValType::I64));
6802 #endif
6803           case uint16_t(Op::I64TruncUF64):
6804 #ifdef FLOAT_TO_I64_CALLOUT
6805             CHECK_NEXT(emitCalloutConversionOOM(emitConvertFloatingToInt64Callout,
6806                                                 SymbolicAddress::TruncateDoubleToUint64,
6807                                                 ValType::F64, ValType::I64));
6808 #else
6809             CHECK_NEXT(emitConversionOOM(emitTruncateF64ToI64<true>, ValType::F64, ValType::I64));
6810 #endif
6811           case uint16_t(Op::I64ExtendSI32):
6812             CHECK_NEXT(emitConversion(emitExtendI32ToI64, ValType::I32, ValType::I64));
6813           case uint16_t(Op::I64ExtendUI32):
6814             CHECK_NEXT(emitConversion(emitExtendU32ToI64, ValType::I32, ValType::I64));
6815           case uint16_t(Op::I64ReinterpretF64):
6816             CHECK_NEXT(emitConversion(emitReinterpretF64AsI64, ValType::F64, ValType::I64));
6817           case uint16_t(Op::I64Or):
6818             CHECK_NEXT(emitBinary(emitOrI64, ValType::I64));
6819           case uint16_t(Op::I64And):
6820             CHECK_NEXT(emitBinary(emitAndI64, ValType::I64));
6821           case uint16_t(Op::I64Xor):
6822             CHECK_NEXT(emitBinary(emitXorI64, ValType::I64));
6823           case uint16_t(Op::I64Shl):
6824             CHECK_NEXT(emitBinary(emitShlI64, ValType::I64));
6825           case uint16_t(Op::I64ShrS):
6826             CHECK_NEXT(emitBinary(emitShrI64, ValType::I64));
6827           case uint16_t(Op::I64ShrU):
6828             CHECK_NEXT(emitBinary(emitShrU64, ValType::I64));
6829           case uint16_t(Op::I64Rotr):
6830             CHECK_NEXT(emitBinary(emitRotrI64, ValType::I64));
6831           case uint16_t(Op::I64Rotl):
6832             CHECK_NEXT(emitBinary(emitRotlI64, ValType::I64));
6833           case uint16_t(Op::I64Clz):
6834             CHECK_NEXT(emitUnary(emitClzI64, ValType::I64));
6835           case uint16_t(Op::I64Ctz):
6836             CHECK_NEXT(emitUnary(emitCtzI64, ValType::I64));
6837           case uint16_t(Op::I64Popcnt):
6838             CHECK_NEXT(emitUnary(emitPopcntI64, ValType::I64));
6839           case uint16_t(Op::I64Eqz):
6840             CHECK_NEXT(emitConversion(emitEqzI64, ValType::I64, ValType::I32));
6841           case uint16_t(Op::I64Load8S):
6842             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int8));
6843           case uint16_t(Op::I64Load16S):
6844             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int16));
6845           case uint16_t(Op::I64Load32S):
6846             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int32));
6847           case uint16_t(Op::I64Load8U):
6848             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint8));
6849           case uint16_t(Op::I64Load16U):
6850             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint16));
6851           case uint16_t(Op::I64Load32U):
6852             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Uint32));
6853           case uint16_t(Op::I64Load):
6854             CHECK_NEXT(emitLoad(ValType::I64, Scalar::Int64));
6855           case uint16_t(Op::I64Store8):
6856             CHECK_NEXT(emitStore(ValType::I64, Scalar::Int8));
6857           case uint16_t(Op::I64TeeStore8):
6858             CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int8));
6859           case uint16_t(Op::I64Store16):
6860             CHECK_NEXT(emitStore(ValType::I64, Scalar::Int16));
6861           case uint16_t(Op::I64TeeStore16):
6862             CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int16));
6863           case uint16_t(Op::I64Store32):
6864             CHECK_NEXT(emitStore(ValType::I64, Scalar::Int32));
6865           case uint16_t(Op::I64TeeStore32):
6866             CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int32));
6867           case uint16_t(Op::I64Store):
6868             CHECK_NEXT(emitStore(ValType::I64, Scalar::Int64));
6869           case uint16_t(Op::I64TeeStore):
6870             CHECK_NEXT(emitTeeStore(ValType::I64, Scalar::Int64));
6871 
6872           // F32
6873           case uint16_t(Op::F32Const): {
6874             RawF32 f32;
6875             CHECK(iter_.readF32Const(&f32));
6876             if (!deadCode_)
6877                 pushF32(f32);
6878             NEXT();
6879           }
6880           case uint16_t(Op::F32Add):
6881             CHECK_NEXT(emitBinary(emitAddF32, ValType::F32));
6882           case uint16_t(Op::F32Sub):
6883             CHECK_NEXT(emitBinary(emitSubtractF32, ValType::F32));
6884           case uint16_t(Op::F32Mul):
6885             CHECK_NEXT(emitBinary(emitMultiplyF32, ValType::F32));
6886           case uint16_t(Op::F32Div):
6887             CHECK_NEXT(emitBinary(emitDivideF32, ValType::F32));
6888           case uint16_t(Op::F32Min):
6889             CHECK_NEXT(emitBinary(emitMinF32, ValType::F32));
6890           case uint16_t(Op::F32Max):
6891             CHECK_NEXT(emitBinary(emitMaxF32, ValType::F32));
6892           case uint16_t(Op::F32Neg):
6893             CHECK_NEXT(emitUnary(emitNegateF32, ValType::F32));
6894           case uint16_t(Op::F32Abs):
6895             CHECK_NEXT(emitUnary(emitAbsF32, ValType::F32));
6896           case uint16_t(Op::F32Sqrt):
6897             CHECK_NEXT(emitUnary(emitSqrtF32, ValType::F32));
6898           case uint16_t(Op::F32Ceil):
6899             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CeilF, ValType::F32));
6900           case uint16_t(Op::F32Floor):
6901             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::FloorF, ValType::F32));
6902           case uint16_t(Op::F32DemoteF64):
6903             CHECK_NEXT(emitConversion(emitConvertF64ToF32, ValType::F64, ValType::F32));
6904           case uint16_t(Op::F32ConvertSI32):
6905             CHECK_NEXT(emitConversion(emitConvertI32ToF32, ValType::I32, ValType::F32));
6906           case uint16_t(Op::F32ConvertUI32):
6907             CHECK_NEXT(emitConversion(emitConvertU32ToF32, ValType::I32, ValType::F32));
6908           case uint16_t(Op::F32ConvertSI64):
6909 #ifdef I64_TO_FLOAT_CALLOUT
6910             CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
6911                                                 SymbolicAddress::Int64ToFloatingPoint,
6912                                                 ValType::I64, ValType::F32));
6913 #else
6914             CHECK_NEXT(emitConversion(emitConvertI64ToF32, ValType::I64, ValType::F32));
6915 #endif
6916           case uint16_t(Op::F32ConvertUI64):
6917 #ifdef I64_TO_FLOAT_CALLOUT
6918             CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
6919                                                 SymbolicAddress::Uint64ToFloatingPoint,
6920                                                 ValType::I64, ValType::F32));
6921 #else
6922             CHECK_NEXT(emitConversion(emitConvertU64ToF32, ValType::I64, ValType::F32));
6923 #endif
6924           case uint16_t(Op::F32ReinterpretI32):
6925             CHECK_NEXT(emitConversion(emitReinterpretI32AsF32, ValType::I32, ValType::F32));
6926           case uint16_t(Op::F32Load):
6927             CHECK_NEXT(emitLoad(ValType::F32, Scalar::Float32));
6928           case uint16_t(Op::F32Store):
6929             CHECK_NEXT(emitStore(ValType::F32, Scalar::Float32));
6930           case uint16_t(Op::F32TeeStore):
6931             CHECK_NEXT(emitTeeStore(ValType::F32, Scalar::Float32));
6932           case uint16_t(Op::F32TeeStoreF64):
6933             CHECK_NEXT(emitTeeStoreWithCoercion(ValType::F32, Scalar::Float64));
6934           case uint16_t(Op::F32CopySign):
6935             CHECK_NEXT(emitBinary(emitCopysignF32, ValType::F32));
6936           case uint16_t(Op::F32Nearest):
6937             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntF, ValType::F32));
6938           case uint16_t(Op::F32Trunc):
6939             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TruncF, ValType::F32));
6940 
6941           // F64
6942           case uint16_t(Op::F64Const): {
6943             RawF64 f64;
6944             CHECK(iter_.readF64Const(&f64));
6945             if (!deadCode_)
6946                 pushF64(f64);
6947             NEXT();
6948           }
6949           case uint16_t(Op::F64Add):
6950             CHECK_NEXT(emitBinary(emitAddF64, ValType::F64));
6951           case uint16_t(Op::F64Sub):
6952             CHECK_NEXT(emitBinary(emitSubtractF64, ValType::F64));
6953           case uint16_t(Op::F64Mul):
6954             CHECK_NEXT(emitBinary(emitMultiplyF64, ValType::F64));
6955           case uint16_t(Op::F64Div):
6956             CHECK_NEXT(emitBinary(emitDivideF64, ValType::F64));
6957           case uint16_t(Op::F64Mod):
6958             CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::ModD, ValType::F64));
6959           case uint16_t(Op::F64Min):
6960             CHECK_NEXT(emitBinary(emitMinF64, ValType::F64));
6961           case uint16_t(Op::F64Max):
6962             CHECK_NEXT(emitBinary(emitMaxF64, ValType::F64));
6963           case uint16_t(Op::F64Neg):
6964             CHECK_NEXT(emitUnary(emitNegateF64, ValType::F64));
6965           case uint16_t(Op::F64Abs):
6966             CHECK_NEXT(emitUnary(emitAbsF64, ValType::F64));
6967           case uint16_t(Op::F64Sqrt):
6968             CHECK_NEXT(emitUnary(emitSqrtF64, ValType::F64));
6969           case uint16_t(Op::F64Ceil):
6970             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CeilD, ValType::F64));
6971           case uint16_t(Op::F64Floor):
6972             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::FloorD, ValType::F64));
6973           case uint16_t(Op::F64Sin):
6974             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::SinD, ValType::F64));
6975           case uint16_t(Op::F64Cos):
6976             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::CosD, ValType::F64));
6977           case uint16_t(Op::F64Tan):
6978             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TanD, ValType::F64));
6979           case uint16_t(Op::F64Asin):
6980             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ASinD, ValType::F64));
6981           case uint16_t(Op::F64Acos):
6982             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ACosD, ValType::F64));
6983           case uint16_t(Op::F64Atan):
6984             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ATanD, ValType::F64));
6985           case uint16_t(Op::F64Exp):
6986             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::ExpD, ValType::F64));
6987           case uint16_t(Op::F64Log):
6988             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::LogD, ValType::F64));
6989           case uint16_t(Op::F64Pow):
6990             CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::PowD, ValType::F64));
6991           case uint16_t(Op::F64Atan2):
6992             CHECK_NEXT(emitBinaryMathBuiltinCall(SymbolicAddress::ATan2D, ValType::F64));
6993           case uint16_t(Op::F64PromoteF32):
6994             CHECK_NEXT(emitConversion(emitConvertF32ToF64, ValType::F32, ValType::F64));
6995           case uint16_t(Op::F64ConvertSI32):
6996             CHECK_NEXT(emitConversion(emitConvertI32ToF64, ValType::I32, ValType::F64));
6997           case uint16_t(Op::F64ConvertUI32):
6998             CHECK_NEXT(emitConversion(emitConvertU32ToF64, ValType::I32, ValType::F64));
6999           case uint16_t(Op::F64ConvertSI64):
7000 #ifdef I64_TO_FLOAT_CALLOUT
7001             CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
7002                                                 SymbolicAddress::Int64ToFloatingPoint,
7003                                                 ValType::I64, ValType::F64));
7004 #else
7005             CHECK_NEXT(emitConversion(emitConvertI64ToF64, ValType::I64, ValType::F64));
7006 #endif
7007           case uint16_t(Op::F64ConvertUI64):
7008 #ifdef I64_TO_FLOAT_CALLOUT
7009             CHECK_NEXT(emitCalloutConversionOOM(emitConvertInt64ToFloatingCallout,
7010                                                 SymbolicAddress::Uint64ToFloatingPoint,
7011                                                 ValType::I64, ValType::F64));
7012 #else
7013             CHECK_NEXT(emitConversion(emitConvertU64ToF64, ValType::I64, ValType::F64));
7014 #endif
7015           case uint16_t(Op::F64Load):
7016             CHECK_NEXT(emitLoad(ValType::F64, Scalar::Float64));
7017           case uint16_t(Op::F64Store):
7018             CHECK_NEXT(emitStore(ValType::F64, Scalar::Float64));
7019           case uint16_t(Op::F64TeeStore):
7020             CHECK_NEXT(emitTeeStore(ValType::F64, Scalar::Float64));
7021           case uint16_t(Op::F64TeeStoreF32):
7022             CHECK_NEXT(emitTeeStoreWithCoercion(ValType::F64, Scalar::Float32));
7023           case uint16_t(Op::F64ReinterpretI64):
7024             CHECK_NEXT(emitConversion(emitReinterpretI64AsF64, ValType::I64, ValType::F64));
7025           case uint16_t(Op::F64CopySign):
7026             CHECK_NEXT(emitBinary(emitCopysignF64, ValType::F64));
7027           case uint16_t(Op::F64Nearest):
7028             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::NearbyIntD, ValType::F64));
7029           case uint16_t(Op::F64Trunc):
7030             CHECK_NEXT(emitUnaryMathBuiltinCall(SymbolicAddress::TruncD, ValType::F64));
7031 
7032           // Comparisons
7033           case uint16_t(Op::I32Eq):
7034             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_EQ, MCompare::Compare_Int32));
7035           case uint16_t(Op::I32Ne):
7036             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_NE, MCompare::Compare_Int32));
7037           case uint16_t(Op::I32LtS):
7038             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LT, MCompare::Compare_Int32));
7039           case uint16_t(Op::I32LeS):
7040             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LE, MCompare::Compare_Int32));
7041           case uint16_t(Op::I32GtS):
7042             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GT, MCompare::Compare_Int32));
7043           case uint16_t(Op::I32GeS):
7044             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GE, MCompare::Compare_Int32));
7045           case uint16_t(Op::I32LtU):
7046             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LT, MCompare::Compare_UInt32));
7047           case uint16_t(Op::I32LeU):
7048             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_LE, MCompare::Compare_UInt32));
7049           case uint16_t(Op::I32GtU):
7050             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GT, MCompare::Compare_UInt32));
7051           case uint16_t(Op::I32GeU):
7052             CHECK_NEXT(emitComparison(emitCompareI32, ValType::I32, JSOP_GE, MCompare::Compare_UInt32));
7053           case uint16_t(Op::I64Eq):
7054             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_EQ, MCompare::Compare_Int64));
7055           case uint16_t(Op::I64Ne):
7056             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_NE, MCompare::Compare_Int64));
7057           case uint16_t(Op::I64LtS):
7058             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LT, MCompare::Compare_Int64));
7059           case uint16_t(Op::I64LeS):
7060             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LE, MCompare::Compare_Int64));
7061           case uint16_t(Op::I64GtS):
7062             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GT, MCompare::Compare_Int64));
7063           case uint16_t(Op::I64GeS):
7064             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GE, MCompare::Compare_Int64));
7065           case uint16_t(Op::I64LtU):
7066             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LT, MCompare::Compare_UInt64));
7067           case uint16_t(Op::I64LeU):
7068             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_LE, MCompare::Compare_UInt64));
7069           case uint16_t(Op::I64GtU):
7070             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GT, MCompare::Compare_UInt64));
7071           case uint16_t(Op::I64GeU):
7072             CHECK_NEXT(emitComparison(emitCompareI64, ValType::I64, JSOP_GE, MCompare::Compare_UInt64));
7073           case uint16_t(Op::F32Eq):
7074             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_EQ, MCompare::Compare_Float32));
7075           case uint16_t(Op::F32Ne):
7076             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_NE, MCompare::Compare_Float32));
7077           case uint16_t(Op::F32Lt):
7078             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_LT, MCompare::Compare_Float32));
7079           case uint16_t(Op::F32Le):
7080             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_LE, MCompare::Compare_Float32));
7081           case uint16_t(Op::F32Gt):
7082             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_GT, MCompare::Compare_Float32));
7083           case uint16_t(Op::F32Ge):
7084             CHECK_NEXT(emitComparison(emitCompareF32, ValType::F32, JSOP_GE, MCompare::Compare_Float32));
7085           case uint16_t(Op::F64Eq):
7086             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_EQ, MCompare::Compare_Double));
7087           case uint16_t(Op::F64Ne):
7088             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_NE, MCompare::Compare_Double));
7089           case uint16_t(Op::F64Lt):
7090             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_LT, MCompare::Compare_Double));
7091           case uint16_t(Op::F64Le):
7092             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_LE, MCompare::Compare_Double));
7093           case uint16_t(Op::F64Gt):
7094             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_GT, MCompare::Compare_Double));
7095           case uint16_t(Op::F64Ge):
7096             CHECK_NEXT(emitComparison(emitCompareF64, ValType::F64, JSOP_GE, MCompare::Compare_Double));
7097 
7098           // SIMD
7099 #define CASE(TYPE, OP, SIGN) \
7100           case uint16_t(Op::TYPE##OP): \
7101             MOZ_CRASH("Unimplemented SIMD");
7102 #define I8x16CASE(OP) CASE(I8x16, OP, SimdSign::Signed)
7103 #define I16x8CASE(OP) CASE(I16x8, OP, SimdSign::Signed)
7104 #define I32x4CASE(OP) CASE(I32x4, OP, SimdSign::Signed)
7105 #define F32x4CASE(OP) CASE(F32x4, OP, SimdSign::NotApplicable)
7106 #define B8x16CASE(OP) CASE(B8x16, OP, SimdSign::NotApplicable)
7107 #define B16x8CASE(OP) CASE(B16x8, OP, SimdSign::NotApplicable)
7108 #define B32x4CASE(OP) CASE(B32x4, OP, SimdSign::NotApplicable)
7109 #define ENUMERATE(TYPE, FORALL, DO) \
7110           case uint16_t(Op::TYPE##Constructor): \
7111             FORALL(DO)
7112 
7113           ENUMERATE(I8x16, FORALL_INT8X16_ASMJS_OP, I8x16CASE)
7114           ENUMERATE(I16x8, FORALL_INT16X8_ASMJS_OP, I16x8CASE)
7115           ENUMERATE(I32x4, FORALL_INT32X4_ASMJS_OP, I32x4CASE)
7116           ENUMERATE(F32x4, FORALL_FLOAT32X4_ASMJS_OP, F32x4CASE)
7117           ENUMERATE(B8x16, FORALL_BOOL_SIMD_OP, B8x16CASE)
7118           ENUMERATE(B16x8, FORALL_BOOL_SIMD_OP, B16x8CASE)
7119           ENUMERATE(B32x4, FORALL_BOOL_SIMD_OP, B32x4CASE)
7120 
7121 #undef CASE
7122 #undef I8x16CASE
7123 #undef I16x8CASE
7124 #undef I32x4CASE
7125 #undef F32x4CASE
7126 #undef B8x16CASE
7127 #undef B16x8CASE
7128 #undef B32x4CASE
7129 #undef ENUMERATE
7130 
7131           case uint16_t(Op::I8x16Const):
7132           case uint16_t(Op::I16x8Const):
7133           case uint16_t(Op::I32x4Const):
7134           case uint16_t(Op::F32x4Const):
7135           case uint16_t(Op::B8x16Const):
7136           case uint16_t(Op::B16x8Const):
7137           case uint16_t(Op::B32x4Const):
7138           case uint16_t(Op::I32x4shiftRightByScalarU):
7139           case uint16_t(Op::I8x16addSaturateU):
7140           case uint16_t(Op::I8x16subSaturateU):
7141           case uint16_t(Op::I8x16shiftRightByScalarU):
7142           case uint16_t(Op::I8x16lessThanU):
7143           case uint16_t(Op::I8x16lessThanOrEqualU):
7144           case uint16_t(Op::I8x16greaterThanU):
7145           case uint16_t(Op::I8x16greaterThanOrEqualU):
7146           case uint16_t(Op::I8x16extractLaneU):
7147           case uint16_t(Op::I16x8addSaturateU):
7148           case uint16_t(Op::I16x8subSaturateU):
7149           case uint16_t(Op::I16x8shiftRightByScalarU):
7150           case uint16_t(Op::I16x8lessThanU):
7151           case uint16_t(Op::I16x8lessThanOrEqualU):
7152           case uint16_t(Op::I16x8greaterThanU):
7153           case uint16_t(Op::I16x8greaterThanOrEqualU):
7154           case uint16_t(Op::I16x8extractLaneU):
7155           case uint16_t(Op::I32x4lessThanU):
7156           case uint16_t(Op::I32x4lessThanOrEqualU):
7157           case uint16_t(Op::I32x4greaterThanU):
7158           case uint16_t(Op::I32x4greaterThanOrEqualU):
7159           case uint16_t(Op::I32x4fromFloat32x4U):
7160             MOZ_CRASH("Unimplemented SIMD");
7161 
7162           // Atomics
7163           case uint16_t(Op::I32AtomicsLoad):
7164           case uint16_t(Op::I32AtomicsStore):
7165           case uint16_t(Op::I32AtomicsBinOp):
7166           case uint16_t(Op::I32AtomicsCompareExchange):
7167           case uint16_t(Op::I32AtomicsExchange):
7168             MOZ_CRASH("Unimplemented Atomics");
7169 
7170           // Memory Related
7171           case uint16_t(Op::GrowMemory):
7172             CHECK_NEXT(emitGrowMemory());
7173           case uint16_t(Op::CurrentMemory):
7174             CHECK_NEXT(emitCurrentMemory());
7175         }
7176 
7177         MOZ_CRASH("unexpected wasm opcode");
7178 
7179 #undef CHECK
7180 #undef NEXT
7181 #undef CHECK_NEXT
7182 #undef emitBinary
7183 #undef emitUnary
7184 #undef emitComparison
7185 #undef emitConversion
7186 #undef emitConversionOOM
7187 #undef emitCalloutConversionOOM
7188     }
7189 
7190 done:
7191     return false;
7192 }
7193 
7194 bool
emitFunction()7195 BaseCompiler::emitFunction()
7196 {
7197     // emitBody() will ensure that there is enough memory reserved in the
7198     // vector for infallible allocation to succeed within the compiler, but we
7199     // need a little headroom for the initial pushControl(), which pushes a
7200     // void value onto the value stack.
7201 
7202     if (!stk_.reserve(8))
7203         return false;
7204 
7205     const Sig& sig = func_.sig();
7206 
7207     if (!iter_.readFunctionStart(sig.ret()))
7208         return false;
7209 
7210     beginFunction();
7211 
7212     UniquePooledLabel functionEnd(newLabel());
7213     if (!pushControl(&functionEnd))
7214         return false;
7215 
7216     if (!emitBody())
7217         return false;
7218 
7219     if (!iter_.readFunctionEnd())
7220         return false;
7221 
7222     if (!endFunction())
7223         return false;
7224 
7225     return true;
7226 }
7227 
BaseCompiler(const ModuleGeneratorData & mg,Decoder & decoder,const FuncBytes & func,const ValTypeVector & locals,FuncCompileResults & compileResults)7228 BaseCompiler::BaseCompiler(const ModuleGeneratorData& mg,
7229                            Decoder& decoder,
7230                            const FuncBytes& func,
7231                            const ValTypeVector& locals,
7232                            FuncCompileResults& compileResults)
7233     : mg_(mg),
7234       iter_(decoder, func.lineOrBytecode()),
7235       func_(func),
7236       lastReadCallSite_(0),
7237       alloc_(compileResults.alloc()),
7238       locals_(locals),
7239       localSize_(0),
7240       varLow_(0),
7241       varHigh_(0),
7242       maxFramePushed_(0),
7243       deadCode_(false),
7244       prologueTrapOffset_(trapOffset()),
7245       compileResults_(compileResults),
7246       masm(compileResults_.masm()),
7247       availGPR_(GeneralRegisterSet::All()),
7248       availFPU_(FloatRegisterSet::All()),
7249 #ifdef DEBUG
7250       scratchRegisterTaken_(false),
7251 #endif
7252       tlsSlot_(0),
7253 #ifdef JS_CODEGEN_X64
7254       specific_rax(RegI64(Register64(rax))),
7255       specific_rcx(RegI64(Register64(rcx))),
7256       specific_rdx(RegI64(Register64(rdx))),
7257 #endif
7258 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
7259       specific_eax(RegI32(eax)),
7260       specific_ecx(RegI32(ecx)),
7261       specific_edx(RegI32(edx)),
7262 #endif
7263 #ifdef JS_CODEGEN_X86
7264       singleByteRegs_(GeneralRegisterSet(Registers::SingleByteRegs)),
7265       abiReturnRegI64(RegI64(Register64(edx, eax))),
7266 #endif
7267 #ifdef JS_CODEGEN_ARM
7268       abiReturnRegI64(ReturnReg64),
7269 #endif
7270       joinRegI32(RegI32(ReturnReg)),
7271       joinRegI64(RegI64(ReturnReg64)),
7272       joinRegF32(RegF32(ReturnFloat32Reg)),
7273       joinRegF64(RegF64(ReturnDoubleReg))
7274 {
7275     // jit/RegisterAllocator.h: RegisterAllocator::RegisterAllocator()
7276 
7277 #if defined(JS_CODEGEN_X64)
7278     availGPR_.take(HeapReg);
7279 #elif defined(JS_CODEGEN_ARM)
7280     availGPR_.take(HeapReg);
7281     availGPR_.take(GlobalReg);
7282     availGPR_.take(ScratchRegARM);
7283 #elif defined(JS_CODEGEN_ARM64)
7284     availGPR_.take(HeapReg);
7285     availGPR_.take(HeapLenReg);
7286     availGPR_.take(GlobalReg);
7287 #elif defined(JS_CODEGEN_X86)
7288     availGPR_.take(ScratchRegX86);
7289 #elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
7290     availGPR_.take(HeapReg);
7291     availGPR_.take(GlobalReg);
7292 #endif
7293 
7294     labelPool_.setAllocator(alloc_);
7295 }
7296 
7297 bool
init()7298 BaseCompiler::init()
7299 {
7300     if (!SigDD_.append(ValType::F64) || !SigDD_.append(ValType::F64))
7301         return false;
7302     if (!SigD_.append(ValType::F64))
7303         return false;
7304     if (!SigF_.append(ValType::F32))
7305         return false;
7306     if (!SigI_.append(ValType::I32))
7307         return false;
7308     if (!SigI64I64_.append(ValType::I64) || !SigI64I64_.append(ValType::I64))
7309         return false;
7310 
7311     const ValTypeVector& args = func_.sig().args();
7312 
7313     // localInfo_ contains an entry for every local in locals_, followed by
7314     // entries for special locals. Currently the only special local is the TLS
7315     // pointer.
7316     tlsSlot_ = locals_.length();
7317     if (!localInfo_.resize(locals_.length() + 1))
7318         return false;
7319 
7320     localSize_ = 0;
7321 
7322     for (ABIArgIter<const ValTypeVector> i(args); !i.done(); i++) {
7323         Local& l = localInfo_[i.index()];
7324         switch (i.mirType()) {
7325           case MIRType::Int32:
7326             if (i->argInRegister())
7327                 l.init(MIRType::Int32, pushLocal(4));
7328             else
7329                 l.init(MIRType::Int32, -(i->offsetFromArgBase() + sizeof(Frame)));
7330             break;
7331           case MIRType::Int64:
7332             if (i->argInRegister())
7333                 l.init(MIRType::Int64, pushLocal(8));
7334             else
7335                 l.init(MIRType::Int64, -(i->offsetFromArgBase() + sizeof(Frame)));
7336             break;
7337           case MIRType::Double:
7338             if (i->argInRegister())
7339                 l.init(MIRType::Double, pushLocal(8));
7340             else
7341                 l.init(MIRType::Double, -(i->offsetFromArgBase() + sizeof(Frame)));
7342             break;
7343           case MIRType::Float32:
7344             if (i->argInRegister())
7345                 l.init(MIRType::Float32, pushLocal(4));
7346             else
7347                 l.init(MIRType::Float32, -(i->offsetFromArgBase() + sizeof(Frame)));
7348             break;
7349           default:
7350             MOZ_CRASH("Argument type");
7351         }
7352     }
7353 
7354     // Reserve a stack slot for the TLS pointer outside the varLow..varHigh
7355     // range so it isn't zero-filled like the normal locals.
7356     localInfo_[tlsSlot_].init(MIRType::Pointer, pushLocal(sizeof(void*)));
7357 
7358     varLow_ = localSize_;
7359 
7360     for (size_t i = args.length(); i < locals_.length(); i++) {
7361         Local& l = localInfo_[i];
7362         switch (locals_[i]) {
7363           case ValType::I32:
7364             l.init(MIRType::Int32, pushLocal(4));
7365             break;
7366           case ValType::F32:
7367             l.init(MIRType::Float32, pushLocal(4));
7368             break;
7369           case ValType::F64:
7370             l.init(MIRType::Double, pushLocal(8));
7371             break;
7372           case ValType::I64:
7373             l.init(MIRType::Int64, pushLocal(8));
7374             break;
7375           default:
7376             MOZ_CRASH("Compiler bug: Unexpected local type");
7377         }
7378     }
7379 
7380     varHigh_ = localSize_;
7381 
7382     localSize_ = AlignBytes(localSize_, 16u);
7383 
7384     addInterruptCheck();
7385 
7386     return true;
7387 }
7388 
7389 void
finish()7390 BaseCompiler::finish()
7391 {
7392     MOZ_ASSERT(done(), "all bytes must be consumed");
7393     MOZ_ASSERT(func_.callSiteLineNums().length() == lastReadCallSite_);
7394 
7395     masm.flushBuffer();
7396 }
7397 
7398 static LiveRegisterSet
volatileReturnGPR()7399 volatileReturnGPR()
7400 {
7401     GeneralRegisterSet rtn;
7402     rtn.addAllocatable(ReturnReg);
7403     return LiveRegisterSet(RegisterSet::VolatileNot(RegisterSet(rtn, FloatRegisterSet())));
7404 }
7405 
7406 LiveRegisterSet BaseCompiler::VolatileReturnGPR = volatileReturnGPR();
7407 
7408 } // wasm
7409 } // js
7410 
7411 bool
BaselineCanCompile(const FunctionGenerator * fg)7412 js::wasm::BaselineCanCompile(const FunctionGenerator* fg)
7413 {
7414     // On all platforms we require signals for AsmJS/Wasm.
7415     // If we made it this far we must have signals.
7416     MOZ_RELEASE_ASSERT(wasm::HaveSignalHandlers());
7417 
7418 #if defined(JS_CODEGEN_ARM)
7419     // Simplifying assumption: require SDIV and UDIV.
7420     //
7421     // I have no good data on ARM populations allowing me to say that
7422     // X% of devices in the market implement SDIV and UDIV.  However,
7423     // they are definitely implemented on the Cortex-A7 and Cortex-A15
7424     // and on all ARMv8 systems.
7425     if (!HasIDIV())
7426         return false;
7427 #endif
7428 
7429 #if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_ARM)
7430     if (fg->usesAtomics())
7431         return false;
7432 
7433     if (fg->usesSimd())
7434         return false;
7435 
7436     return true;
7437 #else
7438     return false;
7439 #endif
7440 }
7441 
7442 bool
BaselineCompileFunction(IonCompileTask * task)7443 js::wasm::BaselineCompileFunction(IonCompileTask* task)
7444 {
7445     MOZ_ASSERT(task->mode() == IonCompileTask::CompileMode::Baseline);
7446 
7447     const FuncBytes& func = task->func();
7448     FuncCompileResults& results = task->results();
7449 
7450     Decoder d(func.bytes());
7451 
7452     // Build the local types vector.
7453 
7454     ValTypeVector locals;
7455     if (!locals.appendAll(func.sig().args()))
7456         return false;
7457     if (!DecodeLocalEntries(d, task->mg().kind, &locals))
7458         return false;
7459 
7460     // The MacroAssembler will sometimes access the jitContext.
7461 
7462     JitContext jitContext(&results.alloc());
7463 
7464     // One-pass baseline compilation.
7465 
7466     BaseCompiler f(task->mg(), d, func, locals, results);
7467     if (!f.init())
7468         return false;
7469 
7470     if (!f.emitFunction())
7471         return false;
7472 
7473     f.finish();
7474 
7475     return true;
7476 }
7477 
7478 #undef INT_DIV_I64_CALLOUT
7479 #undef I64_TO_FLOAT_CALLOUT
7480 #undef FLOAT_TO_I64_CALLOUT
7481