1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_MacroAssembler_h
8 #define jit_MacroAssembler_h
9
10 #include "mozilla/EndianUtils.h"
11 #include "mozilla/MacroForEach.h"
12 #include "mozilla/MathAlgorithms.h"
13 #include "mozilla/Maybe.h"
14 #include "mozilla/Variant.h"
15
16 #if defined(JS_CODEGEN_X86)
17 # include "jit/x86/MacroAssembler-x86.h"
18 #elif defined(JS_CODEGEN_X64)
19 # include "jit/x64/MacroAssembler-x64.h"
20 #elif defined(JS_CODEGEN_ARM)
21 # include "jit/arm/MacroAssembler-arm.h"
22 #elif defined(JS_CODEGEN_ARM64)
23 # include "jit/arm64/MacroAssembler-arm64.h"
24 #elif defined(JS_CODEGEN_MIPS32)
25 # include "jit/mips32/MacroAssembler-mips32.h"
26 #elif defined(JS_CODEGEN_MIPS64)
27 # include "jit/mips64/MacroAssembler-mips64.h"
28 #elif defined(JS_CODEGEN_LOONG64)
29 # include "jit/loong64/MacroAssembler-loong64.h"
30 #elif defined(JS_CODEGEN_NONE)
31 # include "jit/none/MacroAssembler-none.h"
32 #else
33 # error "Unknown architecture!"
34 #endif
35 #include "jit/ABIArgGenerator.h"
36 #include "jit/ABIFunctions.h"
37 #include "jit/AtomicOp.h"
38 #include "jit/AutoJitContextAlloc.h"
39 #include "jit/IonTypes.h"
40 #include "jit/MoveResolver.h"
41 #include "jit/VMFunctions.h"
42 #include "js/ScalarType.h" // js::Scalar::Type
43 #include "util/Memory.h"
44 #include "vm/FunctionFlags.h"
45 #include "vm/Opcodes.h"
46 #include "wasm/WasmCodegenTypes.h"
47 #include "wasm/WasmFrame.h"
48
49 // [SMDOC] MacroAssembler multi-platform overview
50 //
51 // * How to read/write MacroAssembler method declarations:
52 //
53 // The following macros are made to avoid #ifdef around each method declarations
54 // of the Macro Assembler, and they are also used as an hint on the location of
55 // the implementations of each method. For example, the following declaration
56 //
57 // void Pop(FloatRegister t) DEFINED_ON(x86_shared, arm);
58 //
59 // suggests the MacroAssembler::Pop(FloatRegister) method is implemented in
60 // x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h.
61 //
62 // - If there is no annotation, then there is only one generic definition in
63 // MacroAssembler.cpp.
64 //
65 // - If the declaration is "inline", then the method definition(s) would be in
66 // the "-inl.h" variant of the same file(s).
67 //
68 // The script check_macroassembler_style.py (which runs on every build) is
69 // used to verify that method definitions match the annotation on the method
70 // declarations. If there is any difference, then you either forgot to define
71 // the method in one of the macro assembler, or you forgot to update the
72 // annotation of the macro assembler declaration.
73 //
74 // Some convenient short-cuts are used to avoid repeating the same list of
75 // architectures on each method declaration, such as PER_ARCH and
76 // PER_SHARED_ARCH.
77 //
78 // Functions that are architecture-agnostic and are the same for all
79 // architectures, that it's necessary to define inline *in this header* to
80 // avoid used-before-defined warnings/errors that would occur if the
81 // definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER
82 // marker at end of the declaration:
83 //
84 // inline uint32_t framePushed() const OOL_IN_HEADER;
85 //
86 // Such functions should then be defined immediately after MacroAssembler's
87 // definition, for example:
88 //
89 // //{{{ check_macroassembler_style
90 // inline uint32_t
91 // MacroAssembler::framePushed() const
92 // {
93 // return framePushed_;
94 // }
95 // ////}}} check_macroassembler_style
96
97 #define ALL_ARCH mips32, mips64, arm, arm64, x86, x64, loong64
98 #define ALL_SHARED_ARCH arm, arm64, loong64, x86_shared, mips_shared
99
100 // * How this macro works:
101 //
102 // DEFINED_ON is a macro which check if, for the current architecture, the
103 // method is defined on the macro assembler or not.
104 //
105 // For each architecture, we have a macro named DEFINED_ON_arch. This macro is
106 // empty if this is not the current architecture. Otherwise it must be either
107 // set to "define" or "crash" (only used for the none target so far).
108 //
109 // The DEFINED_ON macro maps the list of architecture names given as arguments
110 // to a list of macro names. For example,
111 //
112 // DEFINED_ON(arm, x86_shared)
113 //
114 // is expanded to
115 //
116 // DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared
117 //
118 // which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS
119 // to
120 //
121 // define
122 //
123 // or if the JIT is disabled or set to no architecture to
124 //
125 // crash
126 //
127 // or to nothing, if the current architecture is not listed in the list of
128 // arguments of DEFINED_ON. Note, only one of the DEFINED_ON_arch macro
129 // contributes to the non-empty result, which is the macro of the current
130 // architecture if it is listed in the arguments of DEFINED_ON.
131 //
132 // This result is appended to DEFINED_ON_RESULT_ before expanding the macro,
133 // which results in either no annotation, a MOZ_CRASH(), or a "= delete"
134 // annotation on the method declaration.
135
136 #define DEFINED_ON_x86
137 #define DEFINED_ON_x64
138 #define DEFINED_ON_x86_shared
139 #define DEFINED_ON_arm
140 #define DEFINED_ON_arm64
141 #define DEFINED_ON_mips32
142 #define DEFINED_ON_mips64
143 #define DEFINED_ON_mips_shared
144 #define DEFINED_ON_loong64
145 #define DEFINED_ON_none
146
147 // Specialize for each architecture.
148 #if defined(JS_CODEGEN_X86)
149 # undef DEFINED_ON_x86
150 # define DEFINED_ON_x86 define
151 # undef DEFINED_ON_x86_shared
152 # define DEFINED_ON_x86_shared define
153 #elif defined(JS_CODEGEN_X64)
154 # undef DEFINED_ON_x64
155 # define DEFINED_ON_x64 define
156 # undef DEFINED_ON_x86_shared
157 # define DEFINED_ON_x86_shared define
158 #elif defined(JS_CODEGEN_ARM)
159 # undef DEFINED_ON_arm
160 # define DEFINED_ON_arm define
161 #elif defined(JS_CODEGEN_ARM64)
162 # undef DEFINED_ON_arm64
163 # define DEFINED_ON_arm64 define
164 #elif defined(JS_CODEGEN_MIPS32)
165 # undef DEFINED_ON_mips32
166 # define DEFINED_ON_mips32 define
167 # undef DEFINED_ON_mips_shared
168 # define DEFINED_ON_mips_shared define
169 #elif defined(JS_CODEGEN_MIPS64)
170 # undef DEFINED_ON_mips64
171 # define DEFINED_ON_mips64 define
172 # undef DEFINED_ON_mips_shared
173 # define DEFINED_ON_mips_shared define
174 #elif defined(JS_CODEGEN_LOONG64)
175 # undef DEFINED_ON_loong64
176 # define DEFINED_ON_loong64 define
177 #elif defined(JS_CODEGEN_NONE)
178 # undef DEFINED_ON_none
179 # define DEFINED_ON_none crash
180 #else
181 # error "Unknown architecture!"
182 #endif
183
184 #define DEFINED_ON_RESULT_crash \
185 { MOZ_CRASH(); }
186 #define DEFINED_ON_RESULT_define
187 #define DEFINED_ON_RESULT_ = delete
188
189 #define DEFINED_ON_DISPATCH_RESULT_2(Macro, Result) Macro##Result
190 #define DEFINED_ON_DISPATCH_RESULT(...) \
191 DEFINED_ON_DISPATCH_RESULT_2(DEFINED_ON_RESULT_, __VA_ARGS__)
192
193 // We need to let the evaluation of MOZ_FOR_EACH terminates.
194 #define DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult) \
195 DEFINED_ON_DISPATCH_RESULT ParenResult
196 #define DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult) \
197 DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult)
198 #define DEFINED_ON_EXPAND_ARCH_RESULTS(ParenResult) \
199 DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult)
200
201 #define DEFINED_ON_FWDARCH(Arch) DEFINED_ON_##Arch
202 #define DEFINED_ON_MAP_ON_ARCHS(ArchList) \
203 DEFINED_ON_EXPAND_ARCH_RESULTS( \
204 (MOZ_FOR_EACH(DEFINED_ON_FWDARCH, (), ArchList)))
205
206 #define DEFINED_ON(...) DEFINED_ON_MAP_ON_ARCHS((none, __VA_ARGS__))
207
208 #define PER_ARCH DEFINED_ON(ALL_ARCH)
209 #define PER_SHARED_ARCH DEFINED_ON(ALL_SHARED_ARCH)
210 #define OOL_IN_HEADER
211
Imm32_16Adj(uint32_t x)212 constexpr int32_t Imm32_16Adj(uint32_t x) {
213 #if MOZ_LITTLE_ENDIAN()
214 return x << 16;
215 #else
216 return x;
217 #endif
218 }
219
220 namespace JS {
221 struct ExpandoAndGeneration;
222 }
223
224 namespace js {
225
226 class TypedArrayObject;
227
228 namespace wasm {
229 class CalleeDesc;
230 class CallSiteDesc;
231 class BytecodeOffset;
232 class MemoryAccessDesc;
233
234 struct ModuleEnvironment;
235
236 enum class FailureMode : uint8_t;
237 enum class SimdOp;
238 enum class SymbolicAddress;
239 enum class Trap;
240 } // namespace wasm
241
242 namespace jit {
243
244 // Defined in JitFrames.h
245 enum class ExitFrameType : uint8_t;
246
247 class AutoSaveLiveRegisters;
248 class CompileZone;
249 class TemplateNativeObject;
250 class TemplateObject;
251
252 enum class CheckUnsafeCallWithABI {
253 // Require the callee to use AutoUnsafeCallWithABI.
254 Check,
255
256 // We pushed an exit frame so this callWithABI can safely GC and walk the
257 // stack.
258 DontCheckHasExitFrame,
259
260 // Don't check this callWithABI uses AutoUnsafeCallWithABI, for instance
261 // because we're calling a simple helper function (like malloc or js_free)
262 // that we can't change and/or that we know won't GC.
263 DontCheckOther,
264 };
265
266 // This is a global function made to create the DynFn type in a controlled
267 // environment which would check if the function signature has been registered
268 // as an ABI function signature.
269 template <typename Sig>
270 static inline DynFn DynamicFunction(Sig fun);
271
272 enum class CharEncoding { Latin1, TwoByte };
273
274 constexpr uint32_t WasmCallerTlsOffsetBeforeCall =
275 wasm::FrameWithTls::callerTlsOffsetWithoutFrame();
276 constexpr uint32_t WasmCalleeTlsOffsetBeforeCall =
277 wasm::FrameWithTls::calleeTlsOffsetWithoutFrame();
278
279 // Allocation sites may be passed to GC thing allocation methods either via a
280 // register (for baseline compilation) or an enum indicating one of the
281 // catch-all allocation sites (for optimized compilation).
282 struct AllocSiteInput
283 : public mozilla::Variant<Register, gc::CatchAllAllocSite> {
284 using Base = mozilla::Variant<Register, gc::CatchAllAllocSite>;
AllocSiteInputAllocSiteInput285 AllocSiteInput() : Base(gc::CatchAllAllocSite::Unknown) {}
AllocSiteInputAllocSiteInput286 explicit AllocSiteInput(gc::CatchAllAllocSite catchAll) : Base(catchAll) {}
AllocSiteInputAllocSiteInput287 explicit AllocSiteInput(Register reg) : Base(reg) {}
288 };
289
290 // [SMDOC] Code generation invariants (incomplete)
291 //
292 // ## 64-bit GPRs carrying 32-bit values
293 //
294 // At least at the end of every JS or Wasm operation (= SpiderMonkey bytecode or
295 // Wasm bytecode; this is necessarily a little vague), if a 64-bit GPR has a
296 // 32-bit value, then the upper 32 bits of the register may be predictable in
297 // accordance with platform-specific rules, as follows.
298 //
299 // - On x64 and arm64, the upper bits are zero
300 // - On mips64 and loongarch64 the upper bits are the sign extension of the
301 // lower bits
302 // - (On risc-v we have no rule, having no port yet. Sign extension is the most
303 // likely rule, but "unpredictable" is an option.)
304 //
305 // In most cases no extra work needs to be done to maintain the invariant:
306 //
307 // - 32-bit operations on x64 and arm64 zero-extend the result to 64 bits.
308 // These operations ignore the upper bits of the inputs.
309 // - 32-bit operations on mips64 sign-extend the result to 64 bits (even many
310 // that are labeled as "unsigned", eg ADDU, though not all, eg LU).
311 // Additionally, the inputs to many 32-bit operations must be properly
312 // sign-extended to avoid "unpredictable" behavior, and our simulators check
313 // that inputs conform.
314 // - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but
315 // appear to ignore the upper bits of the inputs.)
316 //
317 // The upshot of these invariants is, among other things, that:
318 //
319 // - No code needs to be generated when a 32-bit value is extended to 64 bits
320 // or a 64-bit value is wrapped to 32 bits, if the upper bits are known to be
321 // correct because they resulted from an operation that produced them
322 // predictably.
323 // - Literal loads must be careful to avoid instructions that might extend the
324 // literal in the wrong way.
325 // - Code that produces values using intermediate values with non-canonical
326 // extensions must extend according to platform conventions before being
327 // "done".
328 //
329 // All optimizations are necessarily platform-specific and should only be used
330 // in platform-specific code. We may add architectures in the future that do
331 // not follow the patterns of the few architectures we already have.
332 //
333 // Also see MacroAssembler::debugAssertCanonicalInt32().
334
335 // The public entrypoint for emitting assembly. Note that a MacroAssembler can
336 // use cx->lifoAlloc, so take care not to interleave masm use with other
337 // lifoAlloc use if one will be destroyed before the other.
338 class MacroAssembler : public MacroAssemblerSpecific {
339 public:
340 mozilla::Maybe<JitContext> jitContext_;
341 mozilla::Maybe<AutoJitContextAlloc> alloc_;
342
343 private:
344 // Labels for handling exceptions and failures.
345 NonAssertingLabel failureLabel_;
346
347 protected:
348 // Constructors are protected. Use one of the derived classes!
349 MacroAssembler();
350
351 // This constructor should only be used when there is no JitContext active
352 // (for example when generating string and regexp stubs).
353 explicit MacroAssembler(JSContext* cx);
354
355 // wasm compilation handles its own JitContext-pushing
356 struct WasmToken {};
357 explicit MacroAssembler(WasmToken, TempAllocator& alloc);
358
359 public:
moveResolver()360 MoveResolver& moveResolver() {
361 // As an optimization, the MoveResolver is a persistent data structure
362 // shared between visitors in the CodeGenerator. This assertion
363 // checks that state is not leaking from visitor to visitor
364 // via an unresolved addMove().
365 MOZ_ASSERT(moveResolver_.hasNoPendingMoves());
366 return moveResolver_;
367 }
368
instructionsSize()369 size_t instructionsSize() const { return size(); }
370
371 #ifdef JS_HAS_HIDDEN_SP
372 void Push(RegisterOrSP reg);
373 #endif
374
375 #ifdef ENABLE_WASM_SIMD
376 // `op` should be a shift operation. Return true if a variable-width shift
377 // operation on this architecture should pre-mask the shift count, and if so,
378 // return the mask in `*mask`.
379 static bool MustMaskShiftCountSimd128(wasm::SimdOp op, int32_t* mask);
380 #endif
381
382 private:
383 // The value returned by GetMaxOffsetGuardLimit() in WasmTypes.h
384 uint32_t wasmMaxOffsetGuardLimit_;
385
386 public:
wasmMaxOffsetGuardLimit()387 uint32_t wasmMaxOffsetGuardLimit() const { return wasmMaxOffsetGuardLimit_; }
setWasmMaxOffsetGuardLimit(uint32_t limit)388 void setWasmMaxOffsetGuardLimit(uint32_t limit) {
389 wasmMaxOffsetGuardLimit_ = limit;
390 }
391
392 //{{{ check_macroassembler_decl_style
393 public:
394 // ===============================================================
395 // MacroAssembler high-level usage.
396
397 // Flushes the assembly buffer, on platforms that need it.
398 void flush() PER_SHARED_ARCH;
399
400 // Add a comment that is visible in the pretty printed assembly code.
401 void comment(const char* msg) PER_SHARED_ARCH;
402
403 // ===============================================================
404 // Frame manipulation functions.
405
406 inline uint32_t framePushed() const OOL_IN_HEADER;
407 inline void setFramePushed(uint32_t framePushed) OOL_IN_HEADER;
408 inline void adjustFrame(int32_t value) OOL_IN_HEADER;
409
410 // Adjust the frame, to account for implicit modification of the stack
411 // pointer, such that callee can remove arguments on the behalf of the
412 // caller.
413 inline void implicitPop(uint32_t bytes) OOL_IN_HEADER;
414
415 private:
416 // This field is used to statically (at compilation time) emulate a frame
417 // pointer by keeping track of stack manipulations.
418 //
419 // It is maintained by all stack manipulation functions below.
420 uint32_t framePushed_;
421
422 public:
423 // ===============================================================
424 // Stack manipulation functions -- sets of registers.
425
426 // Approximately speaking, the following routines must use the same memory
427 // layout. Any inconsistencies will certainly lead to crashing in generated
428 // code:
429 //
430 // MacroAssembler::PushRegsInMaskSizeInBytes
431 // MacroAssembler::PushRegsInMask
432 // MacroAssembler::storeRegsInMask
433 // MacroAssembler::PopRegsInMask
434 // MacroAssembler::PopRegsInMaskIgnore
435 // FloatRegister::getRegisterDumpOffsetInBytes
436 // (no class) PushRegisterDump
437 // (union) RegisterContent
438 //
439 // To be more exact, the invariants are:
440 //
441 // * The save area is conceptually viewed as starting at a highest address
442 // (really, at "highest address - 1") and working down to some lower
443 // address.
444 //
445 // * PushRegsInMask, storeRegsInMask and PopRegsInMask{Ignore} must use
446 // exactly the same memory layout, when starting from the abovementioned
447 // highest address.
448 //
449 // * PushRegsInMaskSizeInBytes must produce a value which is exactly equal
450 // to the change in the machine's stack pointer register as a result of
451 // calling PushRegsInMask or PopRegsInMask{Ignore}. This value must be at
452 // least uintptr_t-aligned on the target, and may be more aligned than that.
453 //
454 // * PushRegsInMaskSizeInBytes must produce a value which is greater than or
455 // equal to the amount of space used by storeRegsInMask.
456 //
457 // * Hence, regardless of whether the save area is created with
458 // storeRegsInMask or PushRegsInMask, it is guaranteed to fit inside an
459 // area of size calculated by PushRegsInMaskSizeInBytes.
460 //
461 // * For the `ignore` argument of PopRegsInMaskIgnore, equality checking
462 // for the floating point/SIMD registers is done on the basis of the
463 // underlying physical register, regardless of width. For example, if the
464 // to-restore set contains v17 (the SIMD register with encoding 17) and
465 // the ignore set contains d17 (the double register with encoding 17) then
466 // no part of the physical register with encoding 17 will be restored.
467 // (This is probably not true on arm32, since that has aliased float32
468 // registers; but none of our other targets do.)
469 //
470 // * {Push,store}RegsInMask/storeRegsInMask are further constrained as
471 // follows: when given the argument AllFloatRegisters, the resulting
472 // memory area must contain exactly all the SIMD/FP registers for the
473 // target at their widest width (that we care about). [We have no targets
474 // where the SIMD registers and FP register sets are disjoint.] They must
475 // be packed end-to-end with no holes, with the register with the lowest
476 // encoding number (0), as returned by FloatRegister::encoding(), at the
477 // abovementioned highest address, register 1 just below that, etc.
478 //
479 // Furthermore the sizeof(RegisterContent) must equal the size of a SIMD
480 // register in the abovementioned array.
481 //
482 // Furthermore the value returned by
483 // FloatRegister::getRegisterDumpOffsetInBytes must be a correct index
484 // into the abovementioned array. Given the constraints, the only correct
485 // value is `reg.encoding() * sizeof(RegisterContent)`.
486
487 // Regarding JitRuntime::generateInvalidator and the first two fields of of
488 // class InvalidationBailoutStack (`fpregs_` and `regs_`). These form their
489 // own layout-equivalence class. That is, they must be format-consistent.
490 // But they are not part of the equivalence class that PushRegsInMask et al
491 // belong to. JitRuntime::generateInvalidator may use PushRegsInMask to
492 // generate part of the layout, but that's only a happy coincidence; some
493 // targets roll their own save-code instead.
494 //
495 // Nevertheless, because some targets *do* call PushRegsInMask from
496 // JitRuntime::generateInvalidator, you should check carefully all of the
497 // ::generateInvalidator methods if you change the PushRegsInMask format.
498
499 // The size of the area used by PushRegsInMask.
500 size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set)
501 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
502
503 void PushRegsInMask(LiveRegisterSet set)
504 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
505 void PushRegsInMask(LiveGeneralRegisterSet set);
506
507 // Like PushRegsInMask, but instead of pushing the registers, store them to
508 // |dest|. |dest| should point to the end of the reserved space, so the
509 // first register will be stored at |dest.offset - sizeof(register)|. It is
510 // required that |dest.offset| is at least as large as the value computed by
511 // PushRegsInMaskSizeInBytes for this |set|. In other words, |dest.base|
512 // must point to either the lowest address in the save area, or some address
513 // below that.
514 void storeRegsInMask(LiveRegisterSet set, Address dest, Register scratch)
515 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
516
517 void PopRegsInMask(LiveRegisterSet set);
518 void PopRegsInMask(LiveGeneralRegisterSet set);
519 void PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore)
520 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
521
522 // ===============================================================
523 // Stack manipulation functions -- single registers/values.
524
525 void Push(const Operand op) DEFINED_ON(x86_shared);
526 void Push(Register reg) PER_SHARED_ARCH;
527 void Push(Register reg1, Register reg2, Register reg3, Register reg4)
528 DEFINED_ON(arm64);
529 void Push(const Imm32 imm) PER_SHARED_ARCH;
530 void Push(const ImmWord imm) PER_SHARED_ARCH;
531 void Push(const ImmPtr imm) PER_SHARED_ARCH;
532 void Push(const ImmGCPtr ptr) PER_SHARED_ARCH;
533 void Push(FloatRegister reg) PER_SHARED_ARCH;
534 void PushBoxed(FloatRegister reg) PER_ARCH;
535 void PushFlags() DEFINED_ON(x86_shared);
536 void Push(PropertyKey key, Register scratchReg);
537 void Push(const Address& addr);
538 void Push(TypedOrValueRegister v);
539 void Push(const ConstantOrRegister& v);
540 void Push(const ValueOperand& val);
541 void Push(const Value& val);
542 void Push(JSValueType type, Register reg);
543 void Push(const Register64 reg);
544 void PushEmptyRooted(VMFunctionData::RootType rootType);
545 inline CodeOffset PushWithPatch(ImmWord word);
546 inline CodeOffset PushWithPatch(ImmPtr imm);
547
548 void Pop(const Operand op) DEFINED_ON(x86_shared);
549 void Pop(Register reg) PER_SHARED_ARCH;
550 void Pop(FloatRegister t) PER_SHARED_ARCH;
551 void Pop(const ValueOperand& val) PER_SHARED_ARCH;
552 void PopFlags() DEFINED_ON(x86_shared);
553 void PopStackPtr() DEFINED_ON(arm, mips_shared, x86_shared, loong64);
554 void popRooted(VMFunctionData::RootType rootType, Register cellReg,
555 const ValueOperand& valueReg);
556
557 // Move the stack pointer based on the requested amount.
558 void adjustStack(int amount);
559 void freeStack(uint32_t amount);
560
561 // Warning: This method does not update the framePushed() counter.
562 void freeStack(Register amount);
563
564 private:
565 // ===============================================================
566 // Register allocation fields.
567 #ifdef DEBUG
568 friend AutoRegisterScope;
569 friend AutoFloatRegisterScope;
570 // Used to track register scopes for debug builds.
571 // Manipulated by the AutoGenericRegisterScope class.
572 AllocatableRegisterSet debugTrackedRegisters_;
573 #endif // DEBUG
574
575 public:
576 // ===============================================================
577 // Simple call functions.
578
579 // The returned CodeOffset is the assembler offset for the instruction
580 // immediately following the call; that is, for the return point.
581 CodeOffset call(Register reg) PER_SHARED_ARCH;
582 CodeOffset call(Label* label) PER_SHARED_ARCH;
583
584 void call(const Address& addr) PER_SHARED_ARCH;
585 void call(ImmWord imm) PER_SHARED_ARCH;
586 // Call a target native function, which is neither traceable nor movable.
587 void call(ImmPtr imm) PER_SHARED_ARCH;
588 CodeOffset call(wasm::SymbolicAddress imm) PER_SHARED_ARCH;
589 inline CodeOffset call(const wasm::CallSiteDesc& desc,
590 wasm::SymbolicAddress imm);
591
592 // Call a target JitCode, which must be traceable, and may be movable.
593 void call(JitCode* c) PER_SHARED_ARCH;
594
595 inline void call(TrampolinePtr code);
596
597 inline CodeOffset call(const wasm::CallSiteDesc& desc, const Register reg);
598 inline CodeOffset call(const wasm::CallSiteDesc& desc, uint32_t funcDefIndex);
599 inline void call(const wasm::CallSiteDesc& desc, wasm::Trap trap);
600
601 CodeOffset callWithPatch() PER_SHARED_ARCH;
602 void patchCall(uint32_t callerOffset, uint32_t calleeOffset) PER_SHARED_ARCH;
603
604 // Push the return address and make a call. On platforms where this function
605 // is not defined, push the link register (pushReturnAddress) at the entry
606 // point of the callee.
607 void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared);
608 void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared);
609
610 // These do not adjust framePushed().
611 void pushReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64);
612 void popReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64);
613
614 // Useful for dealing with two-valued returns.
615 void moveRegPair(Register src0, Register src1, Register dst0, Register dst1,
616 MoveOp::Type type = MoveOp::GENERAL);
617
618 public:
619 // ===============================================================
620 // Patchable near/far jumps.
621
622 // "Far jumps" provide the ability to jump to any uint32_t offset from any
623 // other uint32_t offset without using a constant pool (thus returning a
624 // simple CodeOffset instead of a CodeOffsetJump).
625 CodeOffset farJumpWithPatch() PER_SHARED_ARCH;
626 void patchFarJump(CodeOffset farJump, uint32_t targetOffset) PER_SHARED_ARCH;
627
628 // Emit a nop that can be patched to and from a nop and a call with int32
629 // relative displacement.
630 CodeOffset nopPatchableToCall() PER_SHARED_ARCH;
631 void nopPatchableToCall(const wasm::CallSiteDesc& desc);
632 static void patchNopToCall(uint8_t* callsite,
633 uint8_t* target) PER_SHARED_ARCH;
634 static void patchCallToNop(uint8_t* callsite) PER_SHARED_ARCH;
635
636 // These methods are like movWithPatch/PatchDataWithValueCheck but allow
637 // using pc-relative addressing on certain platforms (RIP-relative LEA on x64,
638 // ADR instruction on arm64).
639 //
640 // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is
641 // release-asserted).
642 CodeOffset moveNearAddressWithPatch(Register dest)
643 DEFINED_ON(x86, x64, arm, arm64, loong64, mips_shared);
644 static void patchNearAddressMove(CodeLocationLabel loc,
645 CodeLocationLabel target)
646 DEFINED_ON(x86, x64, arm, arm64, loong64, mips_shared);
647
648 public:
649 // ===============================================================
650 // [SMDOC] JIT-to-C++ Function Calls (callWithABI)
651 //
652 // callWithABI is used to make a call using the standard C/C++ system ABI.
653 //
654 // callWithABI is a low level interface for making calls, as such every call
655 // made with callWithABI should be organized with 6 steps: spilling live
656 // registers, aligning the stack, listing arguments of the called function,
657 // calling a function pointer, extracting the returned value and restoring
658 // live registers.
659 //
660 // A more detailed example of the six stages:
661 //
662 // 1) Saving of registers that are live. This will vary depending on which
663 // SpiderMonkey compiler you are working on. Registers that shouldn't be
664 // restored can be excluded.
665 //
666 // LiveRegisterSet volatileRegs(...);
667 // volatileRegs.take(scratch);
668 // masm.PushRegsInMask(volatileRegs);
669 //
670 // 2) Align the stack to perform the call with the correct stack alignment.
671 //
672 // When the stack pointer alignment is unknown and cannot be corrected
673 // when generating the code, setupUnalignedABICall must be used to
674 // dynamically align the stack pointer to the expectation of the ABI.
675 // When the stack pointer is known at JIT compilation time, the stack can
676 // be fixed manually and setupAlignedABICall and setupWasmABICall can be
677 // used.
678 //
679 // setupWasmABICall is a special case of setupAlignedABICall as
680 // SpiderMonkey's WebAssembly implementation mostly follow the system
681 // ABI, except for float/double arguments, which always use floating
682 // point registers, even if this is not supported by the system ABI.
683 //
684 // masm.setupUnalignedABICall(scratch);
685 //
686 // 3) Passing arguments. Arguments are passed left-to-right.
687 //
688 // masm.passABIArg(scratch);
689 // masm.passABIArg(FloatOp0, MoveOp::Double);
690 //
691 // Note how float register arguments are annotated with MoveOp::Double.
692 //
693 // Concerning stack-relative address, see the note on passABIArg.
694 //
695 // 4) Make the call:
696 //
697 // using Fn = int32_t (*)(int32_t)
698 // masm.callWithABI<Fn, Callee>();
699 //
700 // In the case where the call returns a double, that needs to be
701 // indicated to the callWithABI like this:
702 //
703 // using Fn = double (*)(int32_t)
704 // masm.callWithABI<Fn, Callee>(MoveOp::DOUBLE);
705 //
706 // There are overloads to allow calls to registers and addresses.
707 //
708 // 5) Take care of the ReturnReg or ReturnDoubleReg
709 //
710 // masm.mov(ReturnReg, scratch1);
711 //
712 // 6) Restore the potentially clobbered volatile registers
713 //
714 // masm.PopRegsInMask(volatileRegs);
715 //
716 // If expecting a returned value, this call should use
717 // PopRegsInMaskIgnore to filter out the registers which are containing
718 // the returned value.
719 //
720 // Unless an exit frame is pushed prior to the setupABICall, the callee
721 // should not GC. To ensure this is the case callWithABI is instrumented to
722 // make sure that in the default case callees are annotated with an
723 // AutoUnsafeCallWithABI on the stack.
724 //
725 // A callWithABI can opt out of checking, if for example it is known there
726 // is an exit frame, or the callee is known not to GC.
727 //
728 // If your callee needs to be able to GC, consider using a VMFunction, or
729 // create a fake exit frame, and instrument the TraceJitExitFrame
730 // accordingly.
731
732 // Setup a call to C/C++ code, given the assumption that the framePushed
733 // accruately define the state of the stack, and that the top of the stack
734 // was properly aligned. Note that this only supports cdecl.
735 void setupAlignedABICall(); // CRASH_ON(arm64)
736
737 // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass
738 // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls
739 // can be native, since we always know the stack alignment a priori.
740 void setupWasmABICall(); // CRASH_ON(arm64)
741
742 // Setup an ABI call for when the alignment is not known. This may need a
743 // scratch register.
744 void setupUnalignedABICall(Register scratch) PER_ARCH;
745
746 // Arguments must be assigned to a C/C++ call in order. They are moved
747 // in parallel immediately before performing the call. This process may
748 // temporarily use more stack, in which case esp-relative addresses will be
749 // automatically adjusted. It is extremely important that esp-relative
750 // addresses are computed *after* setupABICall(). Furthermore, no
751 // operations should be emitted while setting arguments.
752 void passABIArg(const MoveOperand& from, MoveOp::Type type);
753 inline void passABIArg(Register reg);
754 inline void passABIArg(FloatRegister reg, MoveOp::Type type);
755
756 inline void callWithABI(
757 DynFn fun, MoveOp::Type result = MoveOp::GENERAL,
758 CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
759 template <typename Sig, Sig fun>
760 inline void callWithABI(
761 MoveOp::Type result = MoveOp::GENERAL,
762 CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
763 inline void callWithABI(Register fun, MoveOp::Type result = MoveOp::GENERAL);
764 inline void callWithABI(const Address& fun,
765 MoveOp::Type result = MoveOp::GENERAL);
766
767 CodeOffset callWithABI(wasm::BytecodeOffset offset, wasm::SymbolicAddress fun,
768 mozilla::Maybe<int32_t> tlsOffset,
769 MoveOp::Type result = MoveOp::GENERAL);
770 void callDebugWithABI(wasm::SymbolicAddress fun,
771 MoveOp::Type result = MoveOp::GENERAL);
772
773 private:
774 // Reinitialize the variables which have to be cleared before making a call
775 // with callWithABI.
776 template <class ABIArgGeneratorT>
777 void setupABICallHelper();
778
779 // Reinitialize the variables which have to be cleared before making a call
780 // with native abi.
781 void setupNativeABICall();
782
783 // Reserve the stack and resolve the arguments move.
784 void callWithABIPre(uint32_t* stackAdjust,
785 bool callFromWasm = false) PER_ARCH;
786
787 // Emits a call to a C/C++ function, resolving all argument moves.
788 void callWithABINoProfiler(void* fun, MoveOp::Type result,
789 CheckUnsafeCallWithABI check);
790 void callWithABINoProfiler(Register fun, MoveOp::Type result) PER_ARCH;
791 void callWithABINoProfiler(const Address& fun, MoveOp::Type result) PER_ARCH;
792
793 // Restore the stack to its state before the setup function call.
794 void callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
795 bool callFromWasm = false) PER_ARCH;
796
797 // Create the signature to be able to decode the arguments of a native
798 // function, when calling a function within the simulator.
799 inline void appendSignatureType(MoveOp::Type type);
800 inline ABIFunctionType signature() const;
801
802 // Private variables used to handle moves between registers given as
803 // arguments to passABIArg and the list of ABI registers expected for the
804 // signature of the function.
805 MoveResolver moveResolver_;
806
807 // Architecture specific implementation which specify how registers & stack
808 // offsets are used for calling a function.
809 ABIArgGenerator abiArgs_;
810
811 #ifdef DEBUG
812 // Flag use to assert that we use ABI function in the right context.
813 bool inCall_;
814 #endif
815
816 // If set by setupUnalignedABICall then callWithABI will pop the stack
817 // register which is on the stack.
818 bool dynamicAlignment_;
819
820 #ifdef JS_SIMULATOR
821 // The signature is used to accumulate all types of arguments which are used
822 // by the caller. This is used by the simulators to decode the arguments
823 // properly, and cast the function pointer to the right type.
824 uint32_t signature_;
825 #endif
826
827 public:
828 // ===============================================================
829 // Jit Frames.
830 //
831 // These functions are used to build the content of the Jit frames. See
832 // CommonFrameLayout class, and all its derivatives. The content should be
833 // pushed in the opposite order as the fields of the structures, such that
834 // the structures can be used to interpret the content of the stack.
835
836 // Call the Jit function, and push the return address (or let the callee
837 // push the return address).
838 //
839 // These functions return the offset of the return address, in order to use
840 // the return address to index the safepoints, which are used to list all
841 // live registers.
842 inline uint32_t callJitNoProfiler(Register callee);
843 inline uint32_t callJit(Register callee);
844 inline uint32_t callJit(JitCode* code);
845 inline uint32_t callJit(TrampolinePtr code);
846 inline uint32_t callJit(ImmPtr callee);
847
848 // The frame descriptor is the second field of all Jit frames, pushed before
849 // calling the Jit function. It is a composite value defined in JitFrames.h
850 inline void makeFrameDescriptor(Register frameSizeReg, FrameType type,
851 uint32_t headerSize);
852
853 // Push the frame descriptor, based on the statically known framePushed.
854 inline void pushStaticFrameDescriptor(FrameType type, uint32_t headerSize);
855
856 // Push the callee token of a JSFunction which pointer is stored in the
857 // |callee| register. The callee token is packed with a |constructing| flag
858 // which correspond to the fact that the JS function is called with "new" or
859 // not.
860 inline void PushCalleeToken(Register callee, bool constructing);
861
862 // Unpack a callee token located at the |token| address, and return the
863 // JSFunction pointer in the |dest| register.
864 inline void loadFunctionFromCalleeToken(Address token, Register dest);
865
866 // This function emulates a call by pushing an exit frame on the stack,
867 // except that the fake-function is inlined within the body of the caller.
868 //
869 // This function assumes that the current frame is an IonJS frame.
870 //
871 // This function returns the offset of the /fake/ return address, in order to
872 // use the return address to index the safepoints, which are used to list all
873 // live registers.
874 //
875 // This function should be balanced with a call to adjustStack, to pop the
876 // exit frame and emulate the return statement of the inlined function.
877 inline uint32_t buildFakeExitFrame(Register scratch);
878
879 private:
880 // This function is used by buildFakeExitFrame to push a fake return address
881 // on the stack. This fake return address should never be used for resuming
882 // any execution, and can even be an invalid pointer into the instruction
883 // stream, as long as it does not alias any other.
884 uint32_t pushFakeReturnAddress(Register scratch) PER_SHARED_ARCH;
885
886 public:
887 // ===============================================================
888 // Exit frame footer.
889 //
890 // When calling outside the Jit we push an exit frame. To mark the stack
891 // correctly, we have to push additional information, called the Exit frame
892 // footer, which is used to identify how the stack is marked.
893 //
894 // See JitFrames.h, and TraceJitExitFrame in JitFrames.cpp.
895
896 // Push stub code and the VMFunctionData pointer.
897 inline void enterExitFrame(Register cxreg, Register scratch,
898 const VMFunctionData* f);
899
900 // Push an exit frame token to identify which fake exit frame this footer
901 // corresponds to.
902 inline void enterFakeExitFrame(Register cxreg, Register scratch,
903 ExitFrameType type);
904
905 // Push an exit frame token for a native call.
906 inline void enterFakeExitFrameForNative(Register cxreg, Register scratch,
907 bool isConstructing);
908
909 // Pop ExitFrame footer in addition to the extra frame.
910 inline void leaveExitFrame(size_t extraFrame = 0);
911
912 private:
913 // Save the top of the stack into JitActivation::packedExitFP of the
914 // current thread, which should be the location of the latest exit frame.
915 void linkExitFrame(Register cxreg, Register scratch);
916
917 public:
918 // ===============================================================
919 // Move instructions
920
921 inline void move64(Imm64 imm, Register64 dest) PER_ARCH;
922 inline void move64(Register64 src, Register64 dest) PER_ARCH;
923
924 inline void moveFloat32ToGPR(FloatRegister src,
925 Register dest) PER_SHARED_ARCH;
926 inline void moveGPRToFloat32(Register src,
927 FloatRegister dest) PER_SHARED_ARCH;
928
929 inline void moveDoubleToGPR64(FloatRegister src, Register64 dest) PER_ARCH;
930 inline void moveGPR64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;
931
932 inline void move8SignExtend(Register src, Register dest) PER_SHARED_ARCH;
933 inline void move16SignExtend(Register src, Register dest) PER_SHARED_ARCH;
934
935 // move64To32 will clear the high bits of `dest` on 64-bit systems.
936 inline void move64To32(Register64 src, Register dest) PER_ARCH;
937
938 inline void move32To64ZeroExtend(Register src, Register64 dest) PER_ARCH;
939
940 inline void move8To64SignExtend(Register src, Register64 dest) PER_ARCH;
941 inline void move16To64SignExtend(Register src, Register64 dest) PER_ARCH;
942 inline void move32To64SignExtend(Register src, Register64 dest) PER_ARCH;
943
944 inline void move32SignExtendToPtr(Register src, Register dest) PER_ARCH;
945 inline void move32ZeroExtendToPtr(Register src, Register dest) PER_ARCH;
946
947 // Copy a constant, typed-register, or a ValueOperand into a ValueOperand
948 // destination.
949 inline void moveValue(const ConstantOrRegister& src,
950 const ValueOperand& dest);
951 void moveValue(const TypedOrValueRegister& src,
952 const ValueOperand& dest) PER_ARCH;
953 void moveValue(const ValueOperand& src, const ValueOperand& dest) PER_ARCH;
954 void moveValue(const Value& src, const ValueOperand& dest) PER_ARCH;
955
956 void movePropertyKey(PropertyKey key, Register dest);
957
958 // ===============================================================
959 // Load instructions
960
961 inline void load32SignExtendToPtr(const Address& src, Register dest) PER_ARCH;
962
963 inline void loadAbiReturnAddress(Register dest) PER_SHARED_ARCH;
964
965 public:
966 // ===============================================================
967 // Logical instructions
968
969 inline void not32(Register reg) PER_SHARED_ARCH;
970 inline void notPtr(Register reg) PER_ARCH;
971
972 inline void and32(Register src, Register dest) PER_SHARED_ARCH;
973 inline void and32(Imm32 imm, Register dest) PER_SHARED_ARCH;
974 inline void and32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64);
975 inline void and32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
976 inline void and32(const Address& src, Register dest) PER_SHARED_ARCH;
977
978 inline void andPtr(Register src, Register dest) PER_ARCH;
979 inline void andPtr(Imm32 imm, Register dest) PER_ARCH;
980
981 inline void and64(Imm64 imm, Register64 dest) PER_ARCH;
982 inline void or64(Imm64 imm, Register64 dest) PER_ARCH;
983 inline void xor64(Imm64 imm, Register64 dest) PER_ARCH;
984
985 inline void or32(Register src, Register dest) PER_SHARED_ARCH;
986 inline void or32(Imm32 imm, Register dest) PER_SHARED_ARCH;
987 inline void or32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
988
989 inline void orPtr(Register src, Register dest) PER_ARCH;
990 inline void orPtr(Imm32 imm, Register dest) PER_ARCH;
991
992 inline void and64(Register64 src, Register64 dest) PER_ARCH;
993 inline void or64(Register64 src, Register64 dest) PER_ARCH;
994 inline void xor64(Register64 src, Register64 dest) PER_ARCH;
995
996 inline void xor32(Register src, Register dest) PER_SHARED_ARCH;
997 inline void xor32(Imm32 imm, Register dest) PER_SHARED_ARCH;
998 inline void xor32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
999 inline void xor32(const Address& src, Register dest) PER_SHARED_ARCH;
1000
1001 inline void xorPtr(Register src, Register dest) PER_ARCH;
1002 inline void xorPtr(Imm32 imm, Register dest) PER_ARCH;
1003
1004 inline void and64(const Operand& src, Register64 dest)
1005 DEFINED_ON(x64, mips64, loong64);
1006 inline void or64(const Operand& src, Register64 dest)
1007 DEFINED_ON(x64, mips64, loong64);
1008 inline void xor64(const Operand& src, Register64 dest)
1009 DEFINED_ON(x64, mips64, loong64);
1010
1011 // ===============================================================
1012 // Swap instructions
1013
1014 // Swap the two lower bytes and sign extend the result to 32-bit.
1015 inline void byteSwap16SignExtend(Register reg) PER_SHARED_ARCH;
1016
1017 // Swap the two lower bytes and zero extend the result to 32-bit.
1018 inline void byteSwap16ZeroExtend(Register reg) PER_SHARED_ARCH;
1019
1020 // Swap all four bytes in a 32-bit integer.
1021 inline void byteSwap32(Register reg) PER_SHARED_ARCH;
1022
1023 // Swap all eight bytes in a 64-bit integer.
1024 inline void byteSwap64(Register64 reg) PER_ARCH;
1025
1026 // ===============================================================
1027 // Arithmetic functions
1028
1029 inline void add32(Register src, Register dest) PER_SHARED_ARCH;
1030 inline void add32(Imm32 imm, Register dest) PER_SHARED_ARCH;
1031 inline void add32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
1032 inline void add32(Imm32 imm, const AbsoluteAddress& dest)
1033 DEFINED_ON(x86_shared);
1034
1035 inline void addPtr(Register src, Register dest) PER_ARCH;
1036 inline void addPtr(Register src1, Register src2, Register dest)
1037 DEFINED_ON(arm64);
1038 inline void addPtr(Imm32 imm, Register dest) PER_ARCH;
1039 inline void addPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64);
1040 inline void addPtr(ImmWord imm, Register dest) PER_ARCH;
1041 inline void addPtr(ImmPtr imm, Register dest);
1042 inline void addPtr(Imm32 imm, const Address& dest)
1043 DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64);
1044 inline void addPtr(Imm32 imm, const AbsoluteAddress& dest)
1045 DEFINED_ON(x86, x64);
1046 inline void addPtr(const Address& src, Register dest)
1047 DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64);
1048
1049 inline void add64(Register64 src, Register64 dest) PER_ARCH;
1050 inline void add64(Imm32 imm, Register64 dest) PER_ARCH;
1051 inline void add64(Imm64 imm, Register64 dest) PER_ARCH;
1052 inline void add64(const Operand& src, Register64 dest)
1053 DEFINED_ON(x64, mips64, loong64);
1054
1055 inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1056
1057 // Compute dest=SP-imm where dest is a pointer registers and not SP. The
1058 // offset returned from sub32FromStackPtrWithPatch() must be passed to
1059 // patchSub32FromStackPtr().
1060 inline CodeOffset sub32FromStackPtrWithPatch(Register dest) PER_ARCH;
1061 inline void patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) PER_ARCH;
1062
1063 inline void addDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1064 inline void addConstantDouble(double d, FloatRegister dest) DEFINED_ON(x86);
1065
1066 inline void sub32(const Address& src, Register dest) PER_SHARED_ARCH;
1067 inline void sub32(Register src, Register dest) PER_SHARED_ARCH;
1068 inline void sub32(Imm32 imm, Register dest) PER_SHARED_ARCH;
1069
1070 inline void subPtr(Register src, Register dest) PER_ARCH;
1071 inline void subPtr(Register src, const Address& dest)
1072 DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64);
1073 inline void subPtr(Imm32 imm, Register dest) PER_ARCH;
1074 inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x64);
1075 inline void subPtr(const Address& addr, Register dest)
1076 DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64);
1077
1078 inline void sub64(Register64 src, Register64 dest) PER_ARCH;
1079 inline void sub64(Imm64 imm, Register64 dest) PER_ARCH;
1080 inline void sub64(const Operand& src, Register64 dest)
1081 DEFINED_ON(x64, mips64, loong64);
1082
1083 inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1084
1085 inline void subDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1086
1087 inline void mul32(Register rhs, Register srcDest) PER_SHARED_ARCH;
1088 inline void mul32(Imm32 imm, Register srcDest) PER_SHARED_ARCH;
1089
1090 inline void mul32(Register src1, Register src2, Register dest, Label* onOver)
1091 DEFINED_ON(arm64);
1092
1093 inline void mulPtr(Register rhs, Register srcDest) PER_ARCH;
1094
1095 inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64);
1096 inline void mul64(const Operand& src, const Register64& dest,
1097 const Register temp) DEFINED_ON(x64, mips64, loong64);
1098 inline void mul64(Imm64 imm, const Register64& dest) PER_ARCH;
1099 inline void mul64(Imm64 imm, const Register64& dest, const Register temp)
1100 DEFINED_ON(x86, x64, arm, mips32, mips64, loong64);
1101 inline void mul64(const Register64& src, const Register64& dest,
1102 const Register temp) PER_ARCH;
1103 inline void mul64(const Register64& src1, const Register64& src2,
1104 const Register64& dest) DEFINED_ON(arm64);
1105 inline void mul64(Imm64 src1, const Register64& src2, const Register64& dest)
1106 DEFINED_ON(arm64);
1107
1108 inline void mulBy3(Register src, Register dest) PER_ARCH;
1109
1110 inline void mulFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1111 inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1112
1113 inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest)
1114 DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64);
1115
1116 // Perform an integer division, returning the integer part rounded toward
1117 // zero. rhs must not be zero, and the division must not overflow.
1118 //
1119 // On ARM, the chip must have hardware division instructions.
1120 inline void quotient32(Register rhs, Register srcDest, bool isUnsigned)
1121 DEFINED_ON(mips_shared, arm, arm64, loong64);
1122
1123 // As above, but srcDest must be eax and tempEdx must be edx.
1124 inline void quotient32(Register rhs, Register srcDest, Register tempEdx,
1125 bool isUnsigned) DEFINED_ON(x86_shared);
1126
1127 // Perform an integer division, returning the remainder part.
1128 // rhs must not be zero, and the division must not overflow.
1129 //
1130 // On ARM, the chip must have hardware division instructions.
1131 inline void remainder32(Register rhs, Register srcDest, bool isUnsigned)
1132 DEFINED_ON(mips_shared, arm, arm64, loong64);
1133
1134 // As above, but srcDest must be eax and tempEdx must be edx.
1135 inline void remainder32(Register rhs, Register srcDest, Register tempEdx,
1136 bool isUnsigned) DEFINED_ON(x86_shared);
1137
1138 // Perform an integer division, returning the integer part rounded toward
1139 // zero. rhs must not be zero, and the division must not overflow.
1140 //
1141 // This variant preserves registers, and doesn't require hardware division
1142 // instructions on ARM (will call out to a runtime routine).
1143 //
1144 // rhs is preserved, srdDest is clobbered.
1145 void flexibleRemainder32(Register rhs, Register srcDest, bool isUnsigned,
1146 const LiveRegisterSet& volatileLiveRegs)
1147 DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64);
1148
1149 // Perform an integer division, returning the integer part rounded toward
1150 // zero. rhs must not be zero, and the division must not overflow.
1151 //
1152 // This variant preserves registers, and doesn't require hardware division
1153 // instructions on ARM (will call out to a runtime routine).
1154 //
1155 // rhs is preserved, srdDest is clobbered.
1156 void flexibleQuotient32(Register rhs, Register srcDest, bool isUnsigned,
1157 const LiveRegisterSet& volatileLiveRegs)
1158 DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64);
1159
1160 // Perform an integer division, returning the integer part rounded toward
1161 // zero. rhs must not be zero, and the division must not overflow. The
1162 // remainder is stored into the third argument register here.
1163 //
1164 // This variant preserves registers, and doesn't require hardware division
1165 // instructions on ARM (will call out to a runtime routine).
1166 //
1167 // rhs is preserved, srdDest and remOutput are clobbered.
1168 void flexibleDivMod32(Register rhs, Register srcDest, Register remOutput,
1169 bool isUnsigned,
1170 const LiveRegisterSet& volatileLiveRegs)
1171 DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64);
1172
1173 inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1174 inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1175
1176 inline void inc64(AbsoluteAddress dest) PER_ARCH;
1177
1178 inline void neg32(Register reg) PER_SHARED_ARCH;
1179 inline void neg64(Register64 reg) PER_ARCH;
1180 inline void negPtr(Register reg) PER_ARCH;
1181
1182 inline void negateFloat(FloatRegister reg) PER_SHARED_ARCH;
1183
1184 inline void negateDouble(FloatRegister reg) PER_SHARED_ARCH;
1185
1186 inline void abs32(Register src, Register dest) PER_SHARED_ARCH;
1187 inline void absFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1188 inline void absDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1189
1190 inline void sqrtFloat32(FloatRegister src,
1191 FloatRegister dest) PER_SHARED_ARCH;
1192 inline void sqrtDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
1193
1194 void floorFloat32ToInt32(FloatRegister src, Register dest,
1195 Label* fail) PER_SHARED_ARCH;
1196 void floorDoubleToInt32(FloatRegister src, Register dest,
1197 Label* fail) PER_SHARED_ARCH;
1198
1199 void ceilFloat32ToInt32(FloatRegister src, Register dest,
1200 Label* fail) PER_SHARED_ARCH;
1201 void ceilDoubleToInt32(FloatRegister src, Register dest,
1202 Label* fail) PER_SHARED_ARCH;
1203
1204 void roundFloat32ToInt32(FloatRegister src, Register dest, FloatRegister temp,
1205 Label* fail) PER_SHARED_ARCH;
1206 void roundDoubleToInt32(FloatRegister src, Register dest, FloatRegister temp,
1207 Label* fail) PER_SHARED_ARCH;
1208
1209 void truncFloat32ToInt32(FloatRegister src, Register dest,
1210 Label* fail) PER_SHARED_ARCH;
1211 void truncDoubleToInt32(FloatRegister src, Register dest,
1212 Label* fail) PER_SHARED_ARCH;
1213
1214 void nearbyIntDouble(RoundingMode mode, FloatRegister src,
1215 FloatRegister dest) PER_SHARED_ARCH;
1216 void nearbyIntFloat32(RoundingMode mode, FloatRegister src,
1217 FloatRegister dest) PER_SHARED_ARCH;
1218
1219 void signInt32(Register input, Register output);
1220 void signDouble(FloatRegister input, FloatRegister output);
1221 void signDoubleToInt32(FloatRegister input, Register output,
1222 FloatRegister temp, Label* fail);
1223
1224 void copySignDouble(FloatRegister lhs, FloatRegister rhs,
1225 FloatRegister output) PER_SHARED_ARCH;
1226 void copySignFloat32(FloatRegister lhs, FloatRegister rhs,
1227 FloatRegister output) DEFINED_ON(x86_shared, arm64);
1228
1229 // Returns a random double in range [0, 1) in |dest|. The |rng| register must
1230 // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG.
1231 void randomDouble(Register rng, FloatRegister dest, Register64 temp0,
1232 Register64 temp1);
1233
1234 // srcDest = {min,max}{Float32,Double}(srcDest, other)
1235 // For min and max, handle NaN specially if handleNaN is true.
1236
1237 inline void minFloat32(FloatRegister other, FloatRegister srcDest,
1238 bool handleNaN) PER_SHARED_ARCH;
1239 inline void minDouble(FloatRegister other, FloatRegister srcDest,
1240 bool handleNaN) PER_SHARED_ARCH;
1241
1242 inline void maxFloat32(FloatRegister other, FloatRegister srcDest,
1243 bool handleNaN) PER_SHARED_ARCH;
1244 inline void maxDouble(FloatRegister other, FloatRegister srcDest,
1245 bool handleNaN) PER_SHARED_ARCH;
1246
1247 void minMaxArrayInt32(Register array, Register result, Register temp1,
1248 Register temp2, Register temp3, bool isMax,
1249 Label* fail);
1250 void minMaxArrayNumber(Register array, FloatRegister result,
1251 FloatRegister floatTemp, Register temp1,
1252 Register temp2, bool isMax, Label* fail);
1253
1254 // Compute |pow(base, power)| and store the result in |dest|. If the result
1255 // exceeds the int32 range, jumps to |onOver|.
1256 // |base| and |power| are preserved, the other input registers are clobbered.
1257 void pow32(Register base, Register power, Register dest, Register temp1,
1258 Register temp2, Label* onOver);
1259
1260 void sameValueDouble(FloatRegister left, FloatRegister right,
1261 FloatRegister temp, Register dest);
1262
1263 void branchIfNotRegExpPrototypeOptimizable(Register proto, Register temp,
1264 Label* label);
1265 void branchIfNotRegExpInstanceOptimizable(Register regexp, Register temp,
1266 Label* label);
1267
1268 // ===============================================================
1269 // Shift functions
1270
1271 // For shift-by-register there may be platform-specific variations, for
1272 // example, x86 will perform the shift mod 32 but ARM will perform the shift
1273 // mod 256.
1274 //
1275 // For shift-by-immediate the platform assembler may restrict the immediate,
1276 // for example, the ARM assembler requires the count for 32-bit shifts to be
1277 // in the range [0,31].
1278
1279 inline void lshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
1280 inline void rshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
1281 inline void rshift32Arithmetic(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
1282
1283 inline void lshiftPtr(Imm32 imm, Register dest) PER_ARCH;
1284 inline void rshiftPtr(Imm32 imm, Register dest) PER_ARCH;
1285 inline void rshiftPtr(Imm32 imm, Register src, Register dest)
1286 DEFINED_ON(arm64);
1287 inline void rshiftPtrArithmetic(Imm32 imm, Register dest) PER_ARCH;
1288
1289 inline void lshift64(Imm32 imm, Register64 dest) PER_ARCH;
1290 inline void rshift64(Imm32 imm, Register64 dest) PER_ARCH;
1291 inline void rshift64Arithmetic(Imm32 imm, Register64 dest) PER_ARCH;
1292
1293 // On x86_shared these have the constraint that shift must be in CL.
1294 inline void lshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
1295 inline void rshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
1296 inline void rshift32Arithmetic(Register shift,
1297 Register srcDest) PER_SHARED_ARCH;
1298 inline void lshiftPtr(Register shift, Register srcDest) PER_ARCH;
1299 inline void rshiftPtr(Register shift, Register srcDest) PER_ARCH;
1300
1301 // These variants do not have the above constraint, but may emit some extra
1302 // instructions on x86_shared. They also handle shift >= 32 consistently by
1303 // masking with 0x1F (either explicitly or relying on the hardware to do
1304 // that).
1305 inline void flexibleLshift32(Register shift,
1306 Register srcDest) PER_SHARED_ARCH;
1307 inline void flexibleRshift32(Register shift,
1308 Register srcDest) PER_SHARED_ARCH;
1309 inline void flexibleRshift32Arithmetic(Register shift,
1310 Register srcDest) PER_SHARED_ARCH;
1311
1312 inline void lshift64(Register shift, Register64 srcDest) PER_ARCH;
1313 inline void rshift64(Register shift, Register64 srcDest) PER_ARCH;
1314 inline void rshift64Arithmetic(Register shift, Register64 srcDest) PER_ARCH;
1315
1316 // ===============================================================
1317 // Rotation functions
1318 // Note: - on x86 and x64 the count register must be in CL.
1319 // - on x64 the temp register should be InvalidReg.
1320
1321 inline void rotateLeft(Imm32 count, Register input,
1322 Register dest) PER_SHARED_ARCH;
1323 inline void rotateLeft(Register count, Register input,
1324 Register dest) PER_SHARED_ARCH;
1325 inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest)
1326 DEFINED_ON(x64);
1327 inline void rotateLeft64(Register count, Register64 input, Register64 dest)
1328 DEFINED_ON(x64);
1329 inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest,
1330 Register temp) PER_ARCH;
1331 inline void rotateLeft64(Register count, Register64 input, Register64 dest,
1332 Register temp) PER_ARCH;
1333
1334 inline void rotateRight(Imm32 count, Register input,
1335 Register dest) PER_SHARED_ARCH;
1336 inline void rotateRight(Register count, Register input,
1337 Register dest) PER_SHARED_ARCH;
1338 inline void rotateRight64(Imm32 count, Register64 input, Register64 dest)
1339 DEFINED_ON(x64);
1340 inline void rotateRight64(Register count, Register64 input, Register64 dest)
1341 DEFINED_ON(x64);
1342 inline void rotateRight64(Imm32 count, Register64 input, Register64 dest,
1343 Register temp) PER_ARCH;
1344 inline void rotateRight64(Register count, Register64 input, Register64 dest,
1345 Register temp) PER_ARCH;
1346
1347 // ===============================================================
1348 // Bit counting functions
1349
1350 // knownNotZero may be true only if the src is known not to be zero.
1351 inline void clz32(Register src, Register dest,
1352 bool knownNotZero) PER_SHARED_ARCH;
1353 inline void ctz32(Register src, Register dest,
1354 bool knownNotZero) PER_SHARED_ARCH;
1355
1356 inline void clz64(Register64 src, Register dest) PER_ARCH;
1357 inline void ctz64(Register64 src, Register dest) PER_ARCH;
1358
1359 // On x86_shared, temp may be Invalid only if the chip has the POPCNT
1360 // instruction. On ARM, temp may never be Invalid.
1361 inline void popcnt32(Register src, Register dest,
1362 Register temp) PER_SHARED_ARCH;
1363
1364 // temp may be invalid only if the chip has the POPCNT instruction.
1365 inline void popcnt64(Register64 src, Register64 dest, Register temp) PER_ARCH;
1366
1367 // ===============================================================
1368 // Condition functions
1369
1370 inline void cmp8Set(Condition cond, Address lhs, Imm32 rhs,
1371 Register dest) PER_SHARED_ARCH;
1372
1373 inline void cmp16Set(Condition cond, Address lhs, Imm32 rhs,
1374 Register dest) PER_SHARED_ARCH;
1375
1376 template <typename T1, typename T2>
1377 inline void cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest)
1378 DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64);
1379
1380 // Only the NotEqual and Equal conditions are allowed.
1381 inline void cmp64Set(Condition cond, Address lhs, Imm64 rhs,
1382 Register dest) PER_ARCH;
1383
1384 template <typename T1, typename T2>
1385 inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) PER_ARCH;
1386
1387 // ===============================================================
1388 // Branch functions
1389
1390 inline void branch8(Condition cond, const Address& lhs, Imm32 rhs,
1391 Label* label) PER_SHARED_ARCH;
1392
1393 inline void branch16(Condition cond, const Address& lhs, Imm32 rhs,
1394 Label* label) PER_SHARED_ARCH;
1395
1396 template <class L>
1397 inline void branch32(Condition cond, Register lhs, Register rhs,
1398 L label) PER_SHARED_ARCH;
1399 template <class L>
1400 inline void branch32(Condition cond, Register lhs, Imm32 rhs,
1401 L label) PER_SHARED_ARCH;
1402
1403 inline void branch32(Condition cond, Register lhs, const Address& rhs,
1404 Label* label) DEFINED_ON(arm64);
1405
1406 inline void branch32(Condition cond, const Address& lhs, Register rhs,
1407 Label* label) PER_SHARED_ARCH;
1408 inline void branch32(Condition cond, const Address& lhs, Imm32 rhs,
1409 Label* label) PER_SHARED_ARCH;
1410
1411 inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs,
1412 Label* label)
1413 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1414 inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs,
1415 Label* label)
1416 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1417
1418 inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs,
1419 Label* label) DEFINED_ON(arm, x86_shared);
1420 inline void branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
1421 Label* label) PER_SHARED_ARCH;
1422
1423 inline void branch32(Condition cond, const Operand& lhs, Register rhs,
1424 Label* label) DEFINED_ON(x86_shared);
1425 inline void branch32(Condition cond, const Operand& lhs, Imm32 rhs,
1426 Label* label) DEFINED_ON(x86_shared);
1427
1428 inline void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs,
1429 Label* label)
1430 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1431
1432 // The supported condition are Equal, NotEqual, LessThan(orEqual),
1433 // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label
1434 // is not defined it will fall through to next instruction, else jump to the
1435 // fail label.
1436 inline void branch64(Condition cond, Register64 lhs, Imm64 val,
1437 Label* success, Label* fail = nullptr) PER_ARCH;
1438 inline void branch64(Condition cond, Register64 lhs, Register64 rhs,
1439 Label* success, Label* fail = nullptr) PER_ARCH;
1440 // Only the NotEqual and Equal conditions are allowed for the branch64
1441 // variants with Address as lhs.
1442 inline void branch64(Condition cond, const Address& lhs, Imm64 val,
1443 Label* label) PER_ARCH;
1444 inline void branch64(Condition cond, const Address& lhs, Register64 rhs,
1445 Label* label) PER_ARCH;
1446
1447 // Compare the value at |lhs| with the value at |rhs|. The scratch
1448 // register *must not* be the base of |lhs| or |rhs|.
1449 inline void branch64(Condition cond, const Address& lhs, const Address& rhs,
1450 Register scratch, Label* label) PER_ARCH;
1451
1452 template <class L>
1453 inline void branchPtr(Condition cond, Register lhs, Register rhs,
1454 L label) PER_SHARED_ARCH;
1455 inline void branchPtr(Condition cond, Register lhs, Imm32 rhs,
1456 Label* label) PER_SHARED_ARCH;
1457 inline void branchPtr(Condition cond, Register lhs, ImmPtr rhs,
1458 Label* label) PER_SHARED_ARCH;
1459 inline void branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
1460 Label* label) PER_SHARED_ARCH;
1461 inline void branchPtr(Condition cond, Register lhs, ImmWord rhs,
1462 Label* label) PER_SHARED_ARCH;
1463
1464 template <class L>
1465 inline void branchPtr(Condition cond, const Address& lhs, Register rhs,
1466 L label) PER_SHARED_ARCH;
1467 inline void branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
1468 Label* label) PER_SHARED_ARCH;
1469 inline void branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
1470 Label* label) PER_SHARED_ARCH;
1471 inline void branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
1472 Label* label) PER_SHARED_ARCH;
1473
1474 inline void branchPtr(Condition cond, const BaseIndex& lhs, ImmWord rhs,
1475 Label* label) PER_SHARED_ARCH;
1476 inline void branchPtr(Condition cond, const BaseIndex& lhs, Register rhs,
1477 Label* label) PER_SHARED_ARCH;
1478
1479 inline void branchPtr(Condition cond, const AbsoluteAddress& lhs,
1480 Register rhs, Label* label)
1481 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1482 inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs,
1483 Label* label)
1484 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1485
1486 inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs,
1487 Label* label)
1488 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1489
1490 // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
1491 // chunk header, or nullptr if it is in the tenured heap.
1492 void loadStoreBuffer(Register ptr, Register buffer) PER_ARCH;
1493
1494 void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
1495 Label* label)
1496 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1497 void branchPtrInNurseryChunk(Condition cond, const Address& address,
1498 Register temp, Label* label) DEFINED_ON(x86);
1499 void branchValueIsNurseryCell(Condition cond, const Address& address,
1500 Register temp, Label* label) PER_ARCH;
1501 void branchValueIsNurseryCell(Condition cond, ValueOperand value,
1502 Register temp, Label* label) PER_ARCH;
1503
1504 // This function compares a Value (lhs) which is having a private pointer
1505 // boxed inside a js::Value, with a raw pointer (rhs).
1506 inline void branchPrivatePtr(Condition cond, const Address& lhs, Register rhs,
1507 Label* label) PER_ARCH;
1508
1509 inline void branchFloat(DoubleCondition cond, FloatRegister lhs,
1510 FloatRegister rhs, Label* label) PER_SHARED_ARCH;
1511
1512 // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will
1513 // jump to the failure label. This particular variant is allowed to return the
1514 // value module 2**32, which isn't implemented on all architectures. E.g. the
1515 // x64 variants will do this only in the int64_t range.
1516 inline void branchTruncateFloat32MaybeModUint32(FloatRegister src,
1517 Register dest, Label* fail)
1518 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1519 inline void branchTruncateDoubleMaybeModUint32(FloatRegister src,
1520 Register dest, Label* fail)
1521 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1522
1523 // Truncate a double/float32 to intptr and when it doesn't fit jump to the
1524 // failure label.
1525 inline void branchTruncateFloat32ToPtr(FloatRegister src, Register dest,
1526 Label* fail) DEFINED_ON(x86, x64);
1527 inline void branchTruncateDoubleToPtr(FloatRegister src, Register dest,
1528 Label* fail) DEFINED_ON(x86, x64);
1529
1530 // Truncate a double/float32 to int32 and when it doesn't fit jump to the
1531 // failure label.
1532 inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest,
1533 Label* fail)
1534 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1535 inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest,
1536 Label* fail)
1537 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1538
1539 inline void branchDouble(DoubleCondition cond, FloatRegister lhs,
1540 FloatRegister rhs, Label* label) PER_SHARED_ARCH;
1541
1542 inline void branchDoubleNotInInt64Range(Address src, Register temp,
1543 Label* fail);
1544 inline void branchDoubleNotInUInt64Range(Address src, Register temp,
1545 Label* fail);
1546 inline void branchFloat32NotInInt64Range(Address src, Register temp,
1547 Label* fail);
1548 inline void branchFloat32NotInUInt64Range(Address src, Register temp,
1549 Label* fail);
1550
1551 template <typename T>
1552 inline void branchAdd32(Condition cond, T src, Register dest,
1553 Label* label) PER_SHARED_ARCH;
1554 template <typename T>
1555 inline void branchSub32(Condition cond, T src, Register dest,
1556 Label* label) PER_SHARED_ARCH;
1557 template <typename T>
1558 inline void branchMul32(Condition cond, T src, Register dest,
1559 Label* label) PER_SHARED_ARCH;
1560 template <typename T>
1561 inline void branchRshift32(Condition cond, T src, Register dest,
1562 Label* label) PER_SHARED_ARCH;
1563
1564 inline void branchNeg32(Condition cond, Register reg,
1565 Label* label) PER_SHARED_ARCH;
1566
1567 inline void branchAdd64(Condition cond, Imm64 imm, Register64 dest,
1568 Label* label) DEFINED_ON(x86, arm);
1569
1570 template <typename T>
1571 inline void branchAddPtr(Condition cond, T src, Register dest,
1572 Label* label) PER_SHARED_ARCH;
1573
1574 template <typename T>
1575 inline void branchSubPtr(Condition cond, T src, Register dest,
1576 Label* label) PER_SHARED_ARCH;
1577
1578 inline void branchMulPtr(Condition cond, Register src, Register dest,
1579 Label* label) PER_SHARED_ARCH;
1580
1581 inline void decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
1582 Label* label) PER_SHARED_ARCH;
1583
1584 template <class L>
1585 inline void branchTest32(Condition cond, Register lhs, Register rhs,
1586 L label) PER_SHARED_ARCH;
1587 template <class L>
1588 inline void branchTest32(Condition cond, Register lhs, Imm32 rhs,
1589 L label) PER_SHARED_ARCH;
1590 inline void branchTest32(Condition cond, const Address& lhs, Imm32 rhh,
1591 Label* label) PER_SHARED_ARCH;
1592 inline void branchTest32(Condition cond, const AbsoluteAddress& lhs,
1593 Imm32 rhs, Label* label)
1594 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
1595
1596 template <class L>
1597 inline void branchTestPtr(Condition cond, Register lhs, Register rhs,
1598 L label) PER_SHARED_ARCH;
1599 inline void branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
1600 Label* label) PER_SHARED_ARCH;
1601 inline void branchTestPtr(Condition cond, const Address& lhs, Imm32 rhs,
1602 Label* label) PER_SHARED_ARCH;
1603
1604 template <class L>
1605 inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs,
1606 Register temp, L label) PER_ARCH;
1607
1608 // Branches to |label| if |reg| is false. |reg| should be a C++ bool.
1609 template <class L>
1610 inline void branchIfFalseBool(Register reg, L label);
1611
1612 // Branches to |label| if |reg| is true. |reg| should be a C++ bool.
1613 inline void branchIfTrueBool(Register reg, Label* label);
1614
1615 inline void branchIfRope(Register str, Label* label);
1616 inline void branchIfNotRope(Register str, Label* label);
1617
1618 inline void branchLatin1String(Register string, Label* label);
1619 inline void branchTwoByteString(Register string, Label* label);
1620
1621 inline void branchIfBigIntIsNegative(Register bigInt, Label* label);
1622 inline void branchIfBigIntIsNonNegative(Register bigInt, Label* label);
1623 inline void branchIfBigIntIsZero(Register bigInt, Label* label);
1624 inline void branchIfBigIntIsNonZero(Register bigInt, Label* label);
1625
1626 inline void branchTestFunctionFlags(Register fun, uint32_t flags,
1627 Condition cond, Label* label);
1628
1629 inline void branchIfNotFunctionIsNonBuiltinCtor(Register fun,
1630 Register scratch,
1631 Label* label);
1632
1633 inline void branchIfFunctionHasNoJitEntry(Register fun, bool isConstructing,
1634 Label* label);
1635 inline void branchIfFunctionHasJitEntry(Register fun, bool isConstructing,
1636 Label* label);
1637
1638 inline void branchIfScriptHasJitScript(Register script, Label* label);
1639 inline void branchIfScriptHasNoJitScript(Register script, Label* label);
1640 inline void loadJitScript(Register script, Register dest);
1641
1642 // Loads the function length. This handles interpreted, native, and bound
1643 // functions. The caller is responsible for checking that INTERPRETED_LAZY and
1644 // RESOLVED_LENGTH flags are not set.
1645 void loadFunctionLength(Register func, Register funFlagsAndArgCount,
1646 Register output, Label* slowPath);
1647
1648 // Loads the function name. This handles interpreted, native, and bound
1649 // functions.
1650 void loadFunctionName(Register func, Register output, ImmGCPtr emptyString,
1651 Label* slowPath);
1652
1653 inline void branchFunctionKind(Condition cond,
1654 FunctionFlags::FunctionKind kind, Register fun,
1655 Register scratch, Label* label);
1656
1657 inline void branchIfObjectEmulatesUndefined(Register objReg, Register scratch,
1658 Label* slowCheck, Label* label);
1659
1660 // For all methods below: spectreRegToZero is a register that will be zeroed
1661 // on speculatively executed code paths (when the branch should be taken but
1662 // branch prediction speculates it isn't). Usually this will be the object
1663 // register but the caller may pass a different register.
1664
1665 inline void branchTestObjClass(Condition cond, Register obj,
1666 const JSClass* clasp, Register scratch,
1667 Register spectreRegToZero, Label* label);
1668 inline void branchTestObjClassNoSpectreMitigations(Condition cond,
1669 Register obj,
1670 const JSClass* clasp,
1671 Register scratch,
1672 Label* label);
1673
1674 inline void branchTestObjClass(Condition cond, Register obj,
1675 const Address& clasp, Register scratch,
1676 Register spectreRegToZero, Label* label);
1677 inline void branchTestObjClassNoSpectreMitigations(Condition cond,
1678 Register obj,
1679 const Address& clasp,
1680 Register scratch,
1681 Label* label);
1682
1683 inline void branchTestObjClass(Condition cond, Register obj, Register clasp,
1684 Register scratch, Register spectreRegToZero,
1685 Label* label);
1686
1687 inline void branchTestObjShape(Condition cond, Register obj,
1688 const Shape* shape, Register scratch,
1689 Register spectreRegToZero, Label* label);
1690 inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
1691 Register obj,
1692 const Shape* shape,
1693 Label* label);
1694
1695 inline void branchTestClassIsFunction(Condition cond, Register clasp,
1696 Label* label);
1697 inline void branchTestObjIsFunction(Condition cond, Register obj,
1698 Register scratch,
1699 Register spectreRegToZero, Label* label);
1700 inline void branchTestObjIsFunctionNoSpectreMitigations(Condition cond,
1701 Register obj,
1702 Register scratch,
1703 Label* label);
1704
1705 inline void branchTestObjShape(Condition cond, Register obj, Register shape,
1706 Register scratch, Register spectreRegToZero,
1707 Label* label);
1708 inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
1709 Register obj,
1710 Register shape,
1711 Label* label);
1712
1713 // TODO: audit/fix callers to be Spectre safe.
1714 inline void branchTestObjShapeUnsafe(Condition cond, Register obj,
1715 Register shape, Label* label);
1716
1717 void branchTestObjCompartment(Condition cond, Register obj,
1718 const Address& compartment, Register scratch,
1719 Label* label);
1720 void branchTestObjCompartment(Condition cond, Register obj,
1721 const JS::Compartment* compartment,
1722 Register scratch, Label* label);
1723
1724 void branchIfNonNativeObj(Register obj, Register scratch, Label* label);
1725
1726 void branchIfObjectNotExtensible(Register obj, Register scratch,
1727 Label* label);
1728
1729 inline void branchTestClassIsProxy(bool proxy, Register clasp, Label* label);
1730
1731 inline void branchTestObjectIsProxy(bool proxy, Register object,
1732 Register scratch, Label* label);
1733
1734 inline void branchTestProxyHandlerFamily(Condition cond, Register proxy,
1735 Register scratch,
1736 const void* handlerp, Label* label);
1737
1738 inline void branchTestNeedsIncrementalBarrier(Condition cond, Label* label);
1739 inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond,
1740 Label* label,
1741 Register scratch);
1742
1743 // Perform a type-test on a tag of a Value (32bits boxing), or the tagged
1744 // value (64bits boxing).
1745 inline void branchTestUndefined(Condition cond, Register tag,
1746 Label* label) PER_SHARED_ARCH;
1747 inline void branchTestInt32(Condition cond, Register tag,
1748 Label* label) PER_SHARED_ARCH;
1749 inline void branchTestDouble(Condition cond, Register tag, Label* label)
1750 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1751 inline void branchTestNumber(Condition cond, Register tag,
1752 Label* label) PER_SHARED_ARCH;
1753 inline void branchTestBoolean(Condition cond, Register tag,
1754 Label* label) PER_SHARED_ARCH;
1755 inline void branchTestString(Condition cond, Register tag,
1756 Label* label) PER_SHARED_ARCH;
1757 inline void branchTestSymbol(Condition cond, Register tag,
1758 Label* label) PER_SHARED_ARCH;
1759 inline void branchTestBigInt(Condition cond, Register tag,
1760 Label* label) PER_SHARED_ARCH;
1761 inline void branchTestNull(Condition cond, Register tag,
1762 Label* label) PER_SHARED_ARCH;
1763 inline void branchTestObject(Condition cond, Register tag,
1764 Label* label) PER_SHARED_ARCH;
1765 inline void branchTestPrimitive(Condition cond, Register tag,
1766 Label* label) PER_SHARED_ARCH;
1767 inline void branchTestMagic(Condition cond, Register tag,
1768 Label* label) PER_SHARED_ARCH;
1769 void branchTestType(Condition cond, Register tag, JSValueType type,
1770 Label* label);
1771
1772 // Perform a type-test on a Value, addressed by Address or BaseIndex, or
1773 // loaded into ValueOperand.
1774 // BaseIndex and ValueOperand variants clobber the ScratchReg on x64.
1775 // All Variants clobber the ScratchReg on arm64.
1776 inline void branchTestUndefined(Condition cond, const Address& address,
1777 Label* label) PER_SHARED_ARCH;
1778 inline void branchTestUndefined(Condition cond, const BaseIndex& address,
1779 Label* label) PER_SHARED_ARCH;
1780 inline void branchTestUndefined(Condition cond, const ValueOperand& value,
1781 Label* label)
1782 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1783
1784 inline void branchTestInt32(Condition cond, const Address& address,
1785 Label* label) PER_SHARED_ARCH;
1786 inline void branchTestInt32(Condition cond, const BaseIndex& address,
1787 Label* label) PER_SHARED_ARCH;
1788 inline void branchTestInt32(Condition cond, const ValueOperand& value,
1789 Label* label)
1790 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1791
1792 inline void branchTestDouble(Condition cond, const Address& address,
1793 Label* label) PER_SHARED_ARCH;
1794 inline void branchTestDouble(Condition cond, const BaseIndex& address,
1795 Label* label) PER_SHARED_ARCH;
1796 inline void branchTestDouble(Condition cond, const ValueOperand& value,
1797 Label* label)
1798 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1799
1800 inline void branchTestNumber(Condition cond, const ValueOperand& value,
1801 Label* label)
1802 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1803
1804 inline void branchTestBoolean(Condition cond, const Address& address,
1805 Label* label) PER_SHARED_ARCH;
1806 inline void branchTestBoolean(Condition cond, const BaseIndex& address,
1807 Label* label) PER_SHARED_ARCH;
1808 inline void branchTestBoolean(Condition cond, const ValueOperand& value,
1809 Label* label)
1810 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1811
1812 inline void branchTestString(Condition cond, const Address& address,
1813 Label* label) PER_SHARED_ARCH;
1814 inline void branchTestString(Condition cond, const BaseIndex& address,
1815 Label* label) PER_SHARED_ARCH;
1816 inline void branchTestString(Condition cond, const ValueOperand& value,
1817 Label* label)
1818 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1819
1820 inline void branchTestSymbol(Condition cond, const Address& address,
1821 Label* label) PER_SHARED_ARCH;
1822 inline void branchTestSymbol(Condition cond, const BaseIndex& address,
1823 Label* label) PER_SHARED_ARCH;
1824 inline void branchTestSymbol(Condition cond, const ValueOperand& value,
1825 Label* label)
1826 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1827
1828 inline void branchTestBigInt(Condition cond, const Address& address,
1829 Label* label) PER_SHARED_ARCH;
1830 inline void branchTestBigInt(Condition cond, const BaseIndex& address,
1831 Label* label) PER_SHARED_ARCH;
1832 inline void branchTestBigInt(Condition cond, const ValueOperand& value,
1833 Label* label)
1834 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1835
1836 inline void branchTestNull(Condition cond, const Address& address,
1837 Label* label) PER_SHARED_ARCH;
1838 inline void branchTestNull(Condition cond, const BaseIndex& address,
1839 Label* label) PER_SHARED_ARCH;
1840 inline void branchTestNull(Condition cond, const ValueOperand& value,
1841 Label* label)
1842 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1843
1844 // Clobbers the ScratchReg on x64.
1845 inline void branchTestObject(Condition cond, const Address& address,
1846 Label* label) PER_SHARED_ARCH;
1847 inline void branchTestObject(Condition cond, const BaseIndex& address,
1848 Label* label) PER_SHARED_ARCH;
1849 inline void branchTestObject(Condition cond, const ValueOperand& value,
1850 Label* label)
1851 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1852
1853 inline void branchTestGCThing(Condition cond, const Address& address,
1854 Label* label) PER_SHARED_ARCH;
1855 inline void branchTestGCThing(Condition cond, const BaseIndex& address,
1856 Label* label) PER_SHARED_ARCH;
1857 inline void branchTestGCThing(Condition cond, const ValueOperand& value,
1858 Label* label) PER_SHARED_ARCH;
1859
1860 inline void branchTestPrimitive(Condition cond, const ValueOperand& value,
1861 Label* label)
1862 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1863
1864 inline void branchTestMagic(Condition cond, const Address& address,
1865 Label* label) PER_SHARED_ARCH;
1866 inline void branchTestMagic(Condition cond, const BaseIndex& address,
1867 Label* label) PER_SHARED_ARCH;
1868 template <class L>
1869 inline void branchTestMagic(Condition cond, const ValueOperand& value,
1870 L label)
1871 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1872
1873 inline void branchTestMagic(Condition cond, const Address& valaddr,
1874 JSWhyMagic why, Label* label) PER_ARCH;
1875
1876 inline void branchTestMagicValue(Condition cond, const ValueOperand& val,
1877 JSWhyMagic why, Label* label);
1878
1879 void branchTestValue(Condition cond, const ValueOperand& lhs,
1880 const Value& rhs, Label* label) PER_ARCH;
1881
1882 inline void branchTestValue(Condition cond, const BaseIndex& lhs,
1883 const ValueOperand& rhs, Label* label) PER_ARCH;
1884
1885 // Checks if given Value is evaluated to true or false in a condition.
1886 // The type of the value should match the type of the method.
1887 inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value,
1888 Label* label)
1889 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1890 inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg,
1891 Label* label) PER_SHARED_ARCH;
1892 inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value,
1893 Label* label) PER_ARCH;
1894 inline void branchTestStringTruthy(bool truthy, const ValueOperand& value,
1895 Label* label)
1896 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1897 inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value,
1898 Label* label)
1899 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared);
1900
1901 // Create an unconditional branch to the address given as argument.
1902 inline void branchToComputedAddress(const BaseIndex& address) PER_ARCH;
1903
1904 private:
1905 template <typename T, typename S, typename L>
1906 inline void branchPtrImpl(Condition cond, const T& lhs, const S& rhs, L label)
1907 DEFINED_ON(x86_shared);
1908
1909 void branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label)
1910 DEFINED_ON(x86);
1911 template <typename T>
1912 void branchValueIsNurseryCellImpl(Condition cond, const T& value,
1913 Register temp, Label* label)
1914 DEFINED_ON(arm64, x64, mips64, loong64);
1915
1916 template <typename T>
1917 inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label)
1918 DEFINED_ON(arm, arm64, x86_shared);
1919 template <typename T>
1920 inline void branchTestInt32Impl(Condition cond, const T& t, Label* label)
1921 DEFINED_ON(arm, arm64, x86_shared);
1922 template <typename T>
1923 inline void branchTestDoubleImpl(Condition cond, const T& t, Label* label)
1924 DEFINED_ON(arm, arm64, x86_shared);
1925 template <typename T>
1926 inline void branchTestNumberImpl(Condition cond, const T& t, Label* label)
1927 DEFINED_ON(arm, arm64, x86_shared);
1928 template <typename T>
1929 inline void branchTestBooleanImpl(Condition cond, const T& t, Label* label)
1930 DEFINED_ON(arm, arm64, x86_shared);
1931 template <typename T>
1932 inline void branchTestStringImpl(Condition cond, const T& t, Label* label)
1933 DEFINED_ON(arm, arm64, x86_shared);
1934 template <typename T>
1935 inline void branchTestSymbolImpl(Condition cond, const T& t, Label* label)
1936 DEFINED_ON(arm, arm64, x86_shared);
1937 template <typename T>
1938 inline void branchTestBigIntImpl(Condition cond, const T& t, Label* label)
1939 DEFINED_ON(arm, arm64, x86_shared);
1940 template <typename T>
1941 inline void branchTestNullImpl(Condition cond, const T& t, Label* label)
1942 DEFINED_ON(arm, arm64, x86_shared);
1943 template <typename T>
1944 inline void branchTestObjectImpl(Condition cond, const T& t, Label* label)
1945 DEFINED_ON(arm, arm64, x86_shared);
1946 template <typename T>
1947 inline void branchTestGCThingImpl(Condition cond, const T& t,
1948 Label* label) PER_SHARED_ARCH;
1949 template <typename T>
1950 inline void branchTestPrimitiveImpl(Condition cond, const T& t, Label* label)
1951 DEFINED_ON(arm, arm64, x86_shared);
1952 template <typename T, class L>
1953 inline void branchTestMagicImpl(Condition cond, const T& t, L label)
1954 DEFINED_ON(arm, arm64, x86_shared);
1955
1956 public:
1957 template <typename T>
1958 inline void testNumberSet(Condition cond, const T& src,
1959 Register dest) PER_SHARED_ARCH;
1960 template <typename T>
1961 inline void testBooleanSet(Condition cond, const T& src,
1962 Register dest) PER_SHARED_ARCH;
1963 template <typename T>
1964 inline void testStringSet(Condition cond, const T& src,
1965 Register dest) PER_SHARED_ARCH;
1966 template <typename T>
1967 inline void testSymbolSet(Condition cond, const T& src,
1968 Register dest) PER_SHARED_ARCH;
1969 template <typename T>
1970 inline void testBigIntSet(Condition cond, const T& src,
1971 Register dest) PER_SHARED_ARCH;
1972
1973 public:
1974 // The fallibleUnbox* methods below combine a Value type check with an unbox.
1975 // Especially on 64-bit platforms this can be implemented more efficiently
1976 // than a separate branch + unbox.
1977 //
1978 // |src| and |dest| can be the same register, but |dest| may hold garbage on
1979 // failure.
1980 inline void fallibleUnboxPtr(const ValueOperand& src, Register dest,
1981 JSValueType type, Label* fail) PER_ARCH;
1982 inline void fallibleUnboxPtr(const Address& src, Register dest,
1983 JSValueType type, Label* fail) PER_ARCH;
1984 inline void fallibleUnboxPtr(const BaseIndex& src, Register dest,
1985 JSValueType type, Label* fail) PER_ARCH;
1986 template <typename T>
1987 inline void fallibleUnboxInt32(const T& src, Register dest, Label* fail);
1988 template <typename T>
1989 inline void fallibleUnboxBoolean(const T& src, Register dest, Label* fail);
1990 template <typename T>
1991 inline void fallibleUnboxObject(const T& src, Register dest, Label* fail);
1992 template <typename T>
1993 inline void fallibleUnboxString(const T& src, Register dest, Label* fail);
1994 template <typename T>
1995 inline void fallibleUnboxSymbol(const T& src, Register dest, Label* fail);
1996 template <typename T>
1997 inline void fallibleUnboxBigInt(const T& src, Register dest, Label* fail);
1998
1999 inline void cmp32Move32(Condition cond, Register lhs, Register rhs,
2000 Register src, Register dest)
2001 DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
2002
2003 inline void cmp32Move32(Condition cond, Register lhs, const Address& rhs,
2004 Register src, Register dest)
2005 DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
2006
2007 inline void cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
2008 Register src, Register dest) PER_ARCH;
2009
2010 inline void cmpPtrMovePtr(Condition cond, Register lhs, const Address& rhs,
2011 Register src, Register dest) PER_ARCH;
2012
2013 inline void cmp32Load32(Condition cond, Register lhs, const Address& rhs,
2014 const Address& src, Register dest)
2015 DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
2016
2017 inline void cmp32Load32(Condition cond, Register lhs, Register rhs,
2018 const Address& src, Register dest)
2019 DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
2020
2021 inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
2022 const Address& src, Register dest)
2023 DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64);
2024
2025 inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
2026 Register src, Register dest)
2027 DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64);
2028
2029 inline void test32LoadPtr(Condition cond, const Address& addr, Imm32 mask,
2030 const Address& src, Register dest)
2031 DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64);
2032
2033 inline void test32MovePtr(Condition cond, const Address& addr, Imm32 mask,
2034 Register src, Register dest)
2035 DEFINED_ON(arm, arm64, loong64, mips_shared, x86, x64);
2036
2037 // Conditional move for Spectre mitigations.
2038 inline void spectreMovePtr(Condition cond, Register src, Register dest)
2039 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
2040
2041 // Zeroes dest if the condition is true.
2042 inline void spectreZeroRegister(Condition cond, Register scratch,
2043 Register dest)
2044 DEFINED_ON(arm, arm64, mips_shared, x86_shared, loong64);
2045
2046 // Performs a bounds check and zeroes the index register if out-of-bounds
2047 // (to mitigate Spectre).
2048 private:
2049 inline void spectreBoundsCheck32(Register index, const Operand& length,
2050 Register maybeScratch, Label* failure)
2051 DEFINED_ON(x86);
2052
2053 public:
2054 inline void spectreBoundsCheck32(Register index, Register length,
2055 Register maybeScratch, Label* failure)
2056 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
2057 inline void spectreBoundsCheck32(Register index, const Address& length,
2058 Register maybeScratch, Label* failure)
2059 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
2060
2061 inline void spectreBoundsCheckPtr(Register index, Register length,
2062 Register maybeScratch, Label* failure)
2063 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
2064 inline void spectreBoundsCheckPtr(Register index, const Address& length,
2065 Register maybeScratch, Label* failure)
2066 DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64);
2067
2068 // ========================================================================
2069 // Canonicalization primitives.
2070 inline void canonicalizeDouble(FloatRegister reg);
2071 inline void canonicalizeDoubleIfDeterministic(FloatRegister reg);
2072
2073 inline void canonicalizeFloat(FloatRegister reg);
2074 inline void canonicalizeFloatIfDeterministic(FloatRegister reg);
2075
2076 public:
2077 // ========================================================================
2078 // Memory access primitives.
2079 inline void storeUncanonicalizedDouble(FloatRegister src, const Address& dest)
2080 DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64);
2081 inline void storeUncanonicalizedDouble(FloatRegister src,
2082 const BaseIndex& dest)
2083 DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64);
2084 inline void storeUncanonicalizedDouble(FloatRegister src, const Operand& dest)
2085 DEFINED_ON(x86_shared);
2086
2087 template <class T>
2088 inline void storeDouble(FloatRegister src, const T& dest);
2089
2090 template <class T>
2091 inline void boxDouble(FloatRegister src, const T& dest);
2092
2093 using MacroAssemblerSpecific::boxDouble;
2094
2095 inline void storeUncanonicalizedFloat32(FloatRegister src,
2096 const Address& dest)
2097 DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64);
2098 inline void storeUncanonicalizedFloat32(FloatRegister src,
2099 const BaseIndex& dest)
2100 DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64);
2101 inline void storeUncanonicalizedFloat32(FloatRegister src,
2102 const Operand& dest)
2103 DEFINED_ON(x86_shared);
2104
2105 template <class T>
2106 inline void storeFloat32(FloatRegister src, const T& dest);
2107
2108 template <typename T>
2109 void storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType,
2110 const T& dest, MIRType slotType) PER_ARCH;
2111
2112 inline void memoryBarrier(MemoryBarrierBits barrier) PER_SHARED_ARCH;
2113
2114 public:
2115 // ========================================================================
2116 // Wasm SIMD
2117 //
2118 // Naming is "operationSimd128" when operate on the whole vector, otherwise
2119 // it's "operation<Type><Size>x<Lanes>".
2120 //
2121 // For microarchitectural reasons we can in principle get a performance win by
2122 // using int or float specific instructions in the operationSimd128 case when
2123 // we know that subsequent operations on the result are int or float oriented.
2124 // In practice, we don't care about that yet.
2125 //
2126 // The order of operations here follows those in the SIMD overview document,
2127 // https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md.
2128 //
2129 // Since we must target Intel SSE indefinitely and SSE is one-address or
2130 // two-address, the x86 porting interfaces are nearly all one-address or
2131 // two-address. Likewise there are two-address ARM64 interfaces to support
2132 // the baseline compiler. But there are also three-address ARM64 interfaces
2133 // as the ARM64 Ion back-end can use those. In the future, they may support
2134 // AVX2 or similar for x86.
2135 //
2136 // Conventions for argument order and naming and semantics:
2137 // - Condition codes come first.
2138 // - Other immediates (masks, shift counts) come next.
2139 // - Operands come next:
2140 // - For a binary two-address operator where the left-hand-side has the
2141 // same type as the result, one register parameter is normally named
2142 // `lhsDest` and is both the left-hand side and destination; the other
2143 // parameter is named `rhs` and is the right-hand side. `rhs` comes
2144 // first, `lhsDest` second. `rhs` and `lhsDest` may be the same register
2145 // (if rhs is a register).
2146 // - For a binary three-address operator the order is `lhs`, `rhs`, `dest`,
2147 // and generally these registers may be the same.
2148 // - For a unary operator, the input is named `src` and the output is named
2149 // `dest`. `src` comes first, `dest` second. `src` and `dest` may be
2150 // the same register (if `src` is a register).
2151 // - Temp registers follow operands and are named `temp` if there's only one,
2152 // otherwise `temp1`, `temp2`, etc regardless of type. GPR temps precede
2153 // FPU temps. If there are several temps then they must be distinct
2154 // registers, and they must be distinct from the operand registers unless
2155 // noted.
2156
2157 // Moves
2158
2159 inline void moveSimd128(FloatRegister src, FloatRegister dest)
2160 DEFINED_ON(x86_shared, arm64);
2161
2162 // Constants
2163
2164 inline void zeroSimd128(FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2165
2166 inline void loadConstantSimd128(const SimdConstant& v, FloatRegister dest)
2167 DEFINED_ON(x86_shared, arm64);
2168
2169 // Splat
2170
2171 inline void splatX16(Register src, FloatRegister dest)
2172 DEFINED_ON(x86_shared, arm64);
2173
2174 inline void splatX16(uint32_t srcLane, FloatRegister src, FloatRegister dest)
2175 DEFINED_ON(arm64);
2176
2177 inline void splatX8(Register src, FloatRegister dest)
2178 DEFINED_ON(x86_shared, arm64);
2179
2180 inline void splatX8(uint32_t srcLane, FloatRegister src, FloatRegister dest)
2181 DEFINED_ON(arm64);
2182
2183 inline void splatX4(Register src, FloatRegister dest)
2184 DEFINED_ON(x86_shared, arm64);
2185
2186 inline void splatX4(FloatRegister src, FloatRegister dest)
2187 DEFINED_ON(x86_shared, arm64);
2188
2189 inline void splatX2(Register64 src, FloatRegister dest)
2190 DEFINED_ON(x86, x64, arm64);
2191
2192 inline void splatX2(FloatRegister src, FloatRegister dest)
2193 DEFINED_ON(x86_shared, arm64);
2194
2195 // Extract lane as scalar. Float extraction does not canonicalize the value.
2196
2197 inline void extractLaneInt8x16(uint32_t lane, FloatRegister src,
2198 Register dest) DEFINED_ON(x86_shared, arm64);
2199
2200 inline void unsignedExtractLaneInt8x16(uint32_t lane, FloatRegister src,
2201 Register dest)
2202 DEFINED_ON(x86_shared, arm64);
2203
2204 inline void extractLaneInt16x8(uint32_t lane, FloatRegister src,
2205 Register dest) DEFINED_ON(x86_shared, arm64);
2206
2207 inline void unsignedExtractLaneInt16x8(uint32_t lane, FloatRegister src,
2208 Register dest)
2209 DEFINED_ON(x86_shared, arm64);
2210
2211 inline void extractLaneInt32x4(uint32_t lane, FloatRegister src,
2212 Register dest) DEFINED_ON(x86_shared, arm64);
2213
2214 inline void extractLaneInt64x2(uint32_t lane, FloatRegister src,
2215 Register64 dest) DEFINED_ON(x86, x64, arm64);
2216
2217 inline void extractLaneFloat32x4(uint32_t lane, FloatRegister src,
2218 FloatRegister dest)
2219 DEFINED_ON(x86_shared, arm64);
2220
2221 inline void extractLaneFloat64x2(uint32_t lane, FloatRegister src,
2222 FloatRegister dest)
2223 DEFINED_ON(x86_shared, arm64);
2224
2225 // Replace lane value
2226
2227 inline void replaceLaneInt8x16(unsigned lane, Register rhs,
2228 FloatRegister lhsDest)
2229 DEFINED_ON(x86_shared, arm64);
2230
2231 inline void replaceLaneInt16x8(unsigned lane, Register rhs,
2232 FloatRegister lhsDest)
2233 DEFINED_ON(x86_shared, arm64);
2234
2235 inline void replaceLaneInt32x4(unsigned lane, Register rhs,
2236 FloatRegister lhsDest)
2237 DEFINED_ON(x86_shared, arm64);
2238
2239 inline void replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
2240 Register64 rhs, FloatRegister dest)
2241 DEFINED_ON(x86, x64);
2242
2243 inline void replaceLaneInt64x2(unsigned lane, Register64 rhs,
2244 FloatRegister lhsDest)
2245 DEFINED_ON(x86, x64, arm64);
2246
2247 inline void replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
2248 FloatRegister lhsDest)
2249 DEFINED_ON(x86_shared, arm64);
2250
2251 inline void replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
2252 FloatRegister lhsDest)
2253 DEFINED_ON(x86_shared, arm64);
2254
2255 // Shuffle - blend and permute with immediate indices, and its many
2256 // specializations. Lane values other than those mentioned are illegal.
2257
2258 // lane values 0..31
2259 inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
2260 FloatRegister lhsDest)
2261 DEFINED_ON(x86_shared, arm64);
2262
2263 inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
2264 FloatRegister rhs, FloatRegister dest)
2265 DEFINED_ON(x86_shared, arm64);
2266
2267 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2268 // The behavior is undefined for lane values that are neither 0 nor FF.
2269 // on x86_shared: it is required that lhs == dest.
2270 inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
2271 FloatRegister rhs, FloatRegister dest,
2272 FloatRegister temp) DEFINED_ON(x86_shared);
2273
2274 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2275 // The behavior is undefined for lane values that are neither 0 nor FF.
2276 inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
2277 FloatRegister rhs, FloatRegister dest)
2278 DEFINED_ON(arm64);
2279
2280 // Lane values must be 0 (select from lhs) or FFFF (select from rhs).
2281 // The behavior is undefined for lane values that are neither 0 nor FFFF.
2282 // on x86_shared: it is required that lhs == dest.
2283 inline void blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
2284 FloatRegister rhs, FloatRegister dest)
2285 DEFINED_ON(x86_shared, arm64);
2286
2287 // Mask lane values must be ~0 or 0. The former selects from lhs and the
2288 // latter from rhs.
2289 // The implementation works effectively for I8x16, I16x8, I32x4, and I64x2.
2290 inline void laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
2291 FloatRegister rhs, FloatRegister dest)
2292 DEFINED_ON(x86_shared, arm64);
2293
2294 inline void interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
2295 FloatRegister dest)
2296 DEFINED_ON(x86_shared, arm64);
2297
2298 inline void interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
2299 FloatRegister dest)
2300 DEFINED_ON(x86_shared, arm64);
2301
2302 inline void interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
2303 FloatRegister dest)
2304 DEFINED_ON(x86_shared, arm64);
2305
2306 inline void interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
2307 FloatRegister dest)
2308 DEFINED_ON(x86_shared, arm64);
2309
2310 inline void interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
2311 FloatRegister dest)
2312 DEFINED_ON(x86_shared, arm64);
2313
2314 inline void interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
2315 FloatRegister dest)
2316 DEFINED_ON(x86_shared, arm64);
2317
2318 inline void interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
2319 FloatRegister dest)
2320 DEFINED_ON(x86_shared, arm64);
2321
2322 inline void interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
2323 FloatRegister dest)
2324 DEFINED_ON(x86_shared, arm64);
2325
2326 // Permute - permute with immediate indices.
2327
2328 // lane values 0..15
2329 inline void permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
2330 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2331
2332 // lane values 0..7
2333 inline void permuteInt16x8(const uint16_t lanes[8], FloatRegister src,
2334 FloatRegister dest) DEFINED_ON(arm64);
2335
2336 // lane values 0..3 [sic].
2337 inline void permuteHighInt16x8(const uint16_t lanes[4], FloatRegister src,
2338 FloatRegister dest) DEFINED_ON(x86_shared);
2339
2340 // lane values 0..3.
2341 inline void permuteLowInt16x8(const uint16_t lanes[4], FloatRegister src,
2342 FloatRegister dest) DEFINED_ON(x86_shared);
2343
2344 // lane values 0..3
2345 inline void permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
2346 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2347
2348 // Funnel shift by immediate count:
2349 // low_16_bytes_of((lhs ++ rhs) >> shift*8), shift must be < 16
2350 inline void concatAndRightShiftSimd128(FloatRegister lhs, FloatRegister rhs,
2351 FloatRegister dest, uint32_t shift)
2352 DEFINED_ON(x86_shared, arm64);
2353
2354 // Rotate right by immediate count:
2355 // low_16_bytes_of((src ++ src) >> shift*8), shift must be < 16
2356 inline void rotateRightSimd128(FloatRegister src, FloatRegister dest,
2357 uint32_t shift) DEFINED_ON(arm64);
2358
2359 // Shift bytes with immediate count, shifting in zeroes. Shift count 0..15.
2360
2361 inline void leftShiftSimd128(Imm32 count, FloatRegister src,
2362 FloatRegister dest)
2363 DEFINED_ON(x86_shared, arm64);
2364
2365 inline void rightShiftSimd128(Imm32 count, FloatRegister src,
2366 FloatRegister dest)
2367 DEFINED_ON(x86_shared, arm64);
2368
2369 // Reverse bytes in lanes.
2370
2371 inline void reverseInt16x8(FloatRegister src, FloatRegister dest)
2372 DEFINED_ON(x86_shared, arm64);
2373
2374 inline void reverseInt32x4(FloatRegister src, FloatRegister dest)
2375 DEFINED_ON(x86_shared, arm64);
2376
2377 inline void reverseInt64x2(FloatRegister src, FloatRegister dest)
2378 DEFINED_ON(x86_shared, arm64);
2379
2380 // Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
2381
2382 inline void swizzleInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2383 DEFINED_ON(x86_shared, arm64);
2384
2385 inline void swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
2386 FloatRegister dest) DEFINED_ON(arm64);
2387
2388 inline void swizzleInt8x16Relaxed(FloatRegister rhs, FloatRegister lhsDest)
2389 DEFINED_ON(x86_shared, arm64);
2390
2391 inline void swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
2392 FloatRegister dest) DEFINED_ON(arm64);
2393
2394 // Integer Add
2395
2396 inline void addInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2397 DEFINED_ON(x86_shared, arm64);
2398
2399 inline void addInt8x16(FloatRegister lhs, FloatRegister rhs,
2400 FloatRegister dest) DEFINED_ON(arm64);
2401
2402 inline void addInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2403 FloatRegister dest) DEFINED_ON(x86_shared);
2404
2405 inline void addInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2406 DEFINED_ON(x86_shared, arm64);
2407
2408 inline void addInt16x8(FloatRegister lhs, FloatRegister rhs,
2409 FloatRegister dest) DEFINED_ON(arm64);
2410
2411 inline void addInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2412 FloatRegister dest) DEFINED_ON(x86_shared);
2413
2414 inline void addInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2415 DEFINED_ON(x86_shared, arm64);
2416
2417 inline void addInt32x4(FloatRegister lhs, FloatRegister rhs,
2418 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2419
2420 inline void addInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2421 FloatRegister dest) DEFINED_ON(x86_shared);
2422
2423 inline void addInt64x2(FloatRegister rhs, FloatRegister lhsDest)
2424 DEFINED_ON(x86_shared, arm64);
2425
2426 inline void addInt64x2(FloatRegister lhs, FloatRegister rhs,
2427 FloatRegister dest) DEFINED_ON(arm64);
2428
2429 inline void addInt64x2(FloatRegister lhs, const SimdConstant& rhs,
2430 FloatRegister dest) DEFINED_ON(x86_shared);
2431
2432 // Integer Subtract
2433
2434 inline void subInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2435 DEFINED_ON(x86_shared, arm64);
2436
2437 inline void subInt8x16(FloatRegister lhs, FloatRegister rhs,
2438 FloatRegister dest) DEFINED_ON(arm64);
2439
2440 inline void subInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2441 FloatRegister dest) DEFINED_ON(x86_shared);
2442
2443 inline void subInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2444 DEFINED_ON(x86_shared, arm64);
2445
2446 inline void subInt16x8(FloatRegister lhs, FloatRegister rhs,
2447 FloatRegister dest) DEFINED_ON(arm64);
2448
2449 inline void subInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2450 FloatRegister dest) DEFINED_ON(x86_shared);
2451
2452 inline void subInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2453 DEFINED_ON(x86_shared, arm64);
2454
2455 inline void subInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2456 FloatRegister dest) DEFINED_ON(x86_shared);
2457
2458 inline void subInt32x4(FloatRegister lhs, FloatRegister rhs,
2459 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2460
2461 inline void subInt64x2(FloatRegister rhs, FloatRegister lhsDest)
2462 DEFINED_ON(x86_shared, arm64);
2463
2464 inline void subInt64x2(FloatRegister lhs, const SimdConstant& rhs,
2465 FloatRegister dest) DEFINED_ON(x86_shared);
2466
2467 inline void subInt64x2(FloatRegister lhs, FloatRegister rhs,
2468 FloatRegister dest) DEFINED_ON(arm64);
2469
2470 // Integer Multiply
2471
2472 inline void mulInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2473 DEFINED_ON(x86_shared, arm64);
2474
2475 inline void mulInt16x8(FloatRegister lhs, FloatRegister rhs,
2476 FloatRegister dest) DEFINED_ON(arm64);
2477
2478 inline void mulInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2479 FloatRegister dest) DEFINED_ON(x86_shared);
2480
2481 inline void mulInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2482 DEFINED_ON(x86_shared, arm64);
2483
2484 inline void mulInt32x4(FloatRegister lhs, FloatRegister rhs,
2485 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
2486
2487 inline void mulInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2488 FloatRegister dest) DEFINED_ON(x86_shared);
2489
2490 // On x86_shared, it is required lhs == dest
2491 inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
2492 FloatRegister dest, FloatRegister temp)
2493 DEFINED_ON(x86_shared);
2494
2495 inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
2496 FloatRegister dest, FloatRegister temp1,
2497 FloatRegister temp2) DEFINED_ON(arm64);
2498
2499 // Note for the extMul opcodes, the NxM designation is for the input lanes;
2500 // the output lanes are twice as wide.
2501 inline void extMulLowInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2502 DEFINED_ON(x86_shared, arm64);
2503
2504 inline void extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
2505 FloatRegister dest) DEFINED_ON(arm64);
2506
2507 inline void extMulHighInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2508 DEFINED_ON(x86_shared, arm64);
2509
2510 inline void extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
2511 FloatRegister dest) DEFINED_ON(arm64);
2512
2513 inline void unsignedExtMulLowInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2514 DEFINED_ON(x86_shared, arm64);
2515
2516 inline void unsignedExtMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
2517 FloatRegister dest) DEFINED_ON(arm64);
2518
2519 inline void unsignedExtMulHighInt8x16(FloatRegister rhs,
2520 FloatRegister lhsDest)
2521 DEFINED_ON(x86_shared, arm64);
2522
2523 inline void unsignedExtMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
2524 FloatRegister dest) DEFINED_ON(arm64);
2525
2526 inline void extMulLowInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2527 DEFINED_ON(x86_shared, arm64);
2528
2529 inline void extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
2530 FloatRegister dest) DEFINED_ON(arm64);
2531
2532 inline void extMulHighInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2533 DEFINED_ON(x86_shared, arm64);
2534
2535 inline void extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
2536 FloatRegister dest) DEFINED_ON(arm64);
2537
2538 inline void unsignedExtMulLowInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2539 DEFINED_ON(x86_shared, arm64);
2540
2541 inline void unsignedExtMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
2542 FloatRegister dest) DEFINED_ON(arm64);
2543
2544 inline void unsignedExtMulHighInt16x8(FloatRegister rhs,
2545 FloatRegister lhsDest)
2546 DEFINED_ON(x86_shared, arm64);
2547
2548 inline void unsignedExtMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
2549 FloatRegister dest) DEFINED_ON(arm64);
2550
2551 inline void extMulLowInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2552 DEFINED_ON(x86_shared, arm64);
2553
2554 inline void extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
2555 FloatRegister dest) DEFINED_ON(arm64);
2556
2557 inline void extMulHighInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2558 DEFINED_ON(x86_shared, arm64);
2559
2560 inline void extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
2561 FloatRegister dest) DEFINED_ON(arm64);
2562
2563 inline void unsignedExtMulLowInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2564 DEFINED_ON(x86_shared, arm64);
2565
2566 inline void unsignedExtMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
2567 FloatRegister dest) DEFINED_ON(arm64);
2568
2569 inline void unsignedExtMulHighInt32x4(FloatRegister rhs,
2570 FloatRegister lhsDest)
2571 DEFINED_ON(x86_shared, arm64);
2572
2573 inline void unsignedExtMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
2574 FloatRegister dest) DEFINED_ON(arm64);
2575
2576 inline void q15MulrSatInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2577 DEFINED_ON(x86_shared, arm64);
2578
2579 inline void q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2580 FloatRegister dest) DEFINED_ON(arm64);
2581
2582 // Integer Negate
2583
2584 inline void negInt8x16(FloatRegister src, FloatRegister dest)
2585 DEFINED_ON(x86_shared, arm64);
2586
2587 inline void negInt16x8(FloatRegister src, FloatRegister dest)
2588 DEFINED_ON(x86_shared, arm64);
2589
2590 inline void negInt32x4(FloatRegister src, FloatRegister dest)
2591 DEFINED_ON(x86_shared, arm64);
2592
2593 inline void negInt64x2(FloatRegister src, FloatRegister dest)
2594 DEFINED_ON(x86_shared, arm64);
2595
2596 // Saturating integer add
2597
2598 inline void addSatInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2599 DEFINED_ON(x86_shared, arm64);
2600
2601 inline void addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2602 FloatRegister dest) DEFINED_ON(arm64);
2603
2604 inline void addSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2605 FloatRegister dest) DEFINED_ON(x86_shared);
2606
2607 inline void unsignedAddSatInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2608 DEFINED_ON(x86_shared, arm64);
2609
2610 inline void unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2611 FloatRegister dest) DEFINED_ON(arm64);
2612
2613 inline void unsignedAddSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2614 FloatRegister dest) DEFINED_ON(x86_shared);
2615
2616 inline void addSatInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2617 DEFINED_ON(x86_shared, arm64);
2618
2619 inline void addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2620 FloatRegister dest) DEFINED_ON(arm64);
2621
2622 inline void addSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2623 FloatRegister dest) DEFINED_ON(x86_shared);
2624
2625 inline void unsignedAddSatInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2626 DEFINED_ON(x86_shared, arm64);
2627
2628 inline void unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2629 FloatRegister dest) DEFINED_ON(arm64);
2630
2631 inline void unsignedAddSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2632 FloatRegister dest) DEFINED_ON(x86_shared);
2633
2634 // Saturating integer subtract
2635
2636 inline void subSatInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2637 DEFINED_ON(x86_shared, arm64);
2638
2639 inline void subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2640 FloatRegister dest) DEFINED_ON(arm64);
2641
2642 inline void subSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2643 FloatRegister dest) DEFINED_ON(x86_shared);
2644
2645 inline void unsignedSubSatInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2646 DEFINED_ON(x86_shared, arm64);
2647
2648 inline void unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
2649 FloatRegister dest) DEFINED_ON(arm64);
2650
2651 inline void unsignedSubSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2652 FloatRegister dest) DEFINED_ON(x86_shared);
2653
2654 inline void subSatInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2655 DEFINED_ON(x86_shared, arm64);
2656
2657 inline void subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2658 FloatRegister dest) DEFINED_ON(arm64);
2659
2660 inline void subSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2661 FloatRegister dest) DEFINED_ON(x86_shared);
2662
2663 inline void unsignedSubSatInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2664 DEFINED_ON(x86_shared, arm64);
2665
2666 inline void unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
2667 FloatRegister dest) DEFINED_ON(arm64);
2668
2669 inline void unsignedSubSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2670 FloatRegister dest) DEFINED_ON(x86_shared);
2671
2672 // Lane-wise integer minimum
2673
2674 inline void minInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2675 DEFINED_ON(x86_shared, arm64);
2676
2677 inline void minInt8x16(FloatRegister lhs, FloatRegister rhs,
2678 FloatRegister dest) DEFINED_ON(arm64);
2679
2680 inline void minInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2681 FloatRegister dest) DEFINED_ON(x86_shared);
2682
2683 inline void unsignedMinInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2684 DEFINED_ON(x86_shared, arm64);
2685
2686 inline void unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
2687 FloatRegister dest) DEFINED_ON(arm64);
2688
2689 inline void unsignedMinInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2690 FloatRegister dest) DEFINED_ON(x86_shared);
2691
2692 inline void minInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2693 DEFINED_ON(x86_shared, arm64);
2694
2695 inline void minInt16x8(FloatRegister lhs, FloatRegister rhs,
2696 FloatRegister dest) DEFINED_ON(arm64);
2697
2698 inline void minInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2699 FloatRegister dest) DEFINED_ON(x86_shared);
2700
2701 inline void unsignedMinInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2702 DEFINED_ON(x86_shared, arm64);
2703
2704 inline void unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
2705 FloatRegister dest) DEFINED_ON(arm64);
2706
2707 inline void unsignedMinInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2708 FloatRegister dest) DEFINED_ON(x86_shared);
2709
2710 inline void minInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2711 DEFINED_ON(x86_shared, arm64);
2712
2713 inline void minInt32x4(FloatRegister lhs, FloatRegister rhs,
2714 FloatRegister dest) DEFINED_ON(arm64);
2715
2716 inline void minInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2717 FloatRegister dest) DEFINED_ON(x86_shared);
2718
2719 inline void unsignedMinInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2720 DEFINED_ON(x86_shared, arm64);
2721
2722 inline void unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
2723 FloatRegister dest) DEFINED_ON(arm64);
2724
2725 inline void unsignedMinInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2726 FloatRegister dest) DEFINED_ON(x86_shared);
2727
2728 // Lane-wise integer maximum
2729
2730 inline void maxInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2731 DEFINED_ON(x86_shared, arm64);
2732
2733 inline void maxInt8x16(FloatRegister lhs, FloatRegister rhs,
2734 FloatRegister dest) DEFINED_ON(arm64);
2735
2736 inline void maxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2737 FloatRegister dest) DEFINED_ON(x86_shared);
2738
2739 inline void unsignedMaxInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2740 DEFINED_ON(x86_shared, arm64);
2741
2742 inline void unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
2743 FloatRegister dest) DEFINED_ON(arm64);
2744
2745 inline void unsignedMaxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
2746 FloatRegister dest) DEFINED_ON(x86_shared);
2747
2748 inline void maxInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2749 DEFINED_ON(x86_shared, arm64);
2750
2751 inline void maxInt16x8(FloatRegister lhs, FloatRegister rhs,
2752 FloatRegister dest) DEFINED_ON(arm64);
2753
2754 inline void maxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2755 FloatRegister dest) DEFINED_ON(x86_shared);
2756
2757 inline void unsignedMaxInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2758 DEFINED_ON(x86_shared, arm64);
2759
2760 inline void unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
2761 FloatRegister dest) DEFINED_ON(arm64);
2762
2763 inline void unsignedMaxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
2764 FloatRegister dest) DEFINED_ON(x86_shared);
2765
2766 inline void maxInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2767 DEFINED_ON(x86_shared, arm64);
2768
2769 inline void maxInt32x4(FloatRegister lhs, FloatRegister rhs,
2770 FloatRegister dest) DEFINED_ON(arm64);
2771
2772 inline void maxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2773 FloatRegister dest) DEFINED_ON(x86_shared);
2774
2775 inline void unsignedMaxInt32x4(FloatRegister rhs, FloatRegister lhsDest)
2776 DEFINED_ON(x86_shared, arm64);
2777
2778 inline void unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
2779 FloatRegister dest) DEFINED_ON(arm64);
2780
2781 inline void unsignedMaxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
2782 FloatRegister dest) DEFINED_ON(x86_shared);
2783
2784 // Lane-wise integer rounding average
2785
2786 inline void unsignedAverageInt8x16(FloatRegister rhs, FloatRegister lhsDest)
2787 DEFINED_ON(x86_shared, arm64);
2788
2789 inline void unsignedAverageInt8x16(FloatRegister lhs, FloatRegister rhs,
2790 FloatRegister dest) DEFINED_ON(arm64);
2791
2792 inline void unsignedAverageInt16x8(FloatRegister rhs, FloatRegister lhsDest)
2793 DEFINED_ON(x86_shared, arm64);
2794
2795 inline void unsignedAverageInt16x8(FloatRegister lhs, FloatRegister rhs,
2796 FloatRegister dest) DEFINED_ON(arm64);
2797
2798 // Lane-wise integer absolute value
2799
2800 inline void absInt8x16(FloatRegister src, FloatRegister dest)
2801 DEFINED_ON(x86_shared, arm64);
2802
2803 inline void absInt16x8(FloatRegister src, FloatRegister dest)
2804 DEFINED_ON(x86_shared, arm64);
2805
2806 inline void absInt32x4(FloatRegister src, FloatRegister dest)
2807 DEFINED_ON(x86_shared, arm64);
2808
2809 inline void absInt64x2(FloatRegister src, FloatRegister dest)
2810 DEFINED_ON(x86_shared, arm64);
2811
2812 // Left shift by scalar. Immediates and variable shifts must have been
2813 // masked; shifts of zero will work but may or may not generate code.
2814
2815 inline void leftShiftInt8x16(Register rhs, FloatRegister lhsDest,
2816 FloatRegister temp) DEFINED_ON(x86_shared);
2817
2818 inline void leftShiftInt8x16(FloatRegister lhs, Register rhs,
2819 FloatRegister dest) DEFINED_ON(arm64);
2820
2821 inline void leftShiftInt8x16(Imm32 count, FloatRegister src,
2822 FloatRegister dest)
2823 DEFINED_ON(x86_shared, arm64);
2824
2825 inline void leftShiftInt16x8(Register rhs, FloatRegister lhsDest)
2826 DEFINED_ON(x86_shared);
2827
2828 inline void leftShiftInt16x8(FloatRegister lhs, Register rhs,
2829 FloatRegister dest) DEFINED_ON(arm64);
2830
2831 inline void leftShiftInt16x8(Imm32 count, FloatRegister src,
2832 FloatRegister dest)
2833 DEFINED_ON(x86_shared, arm64);
2834
2835 inline void leftShiftInt32x4(Register rhs, FloatRegister lhsDest)
2836 DEFINED_ON(x86_shared);
2837
2838 inline void leftShiftInt32x4(FloatRegister lhs, Register rhs,
2839 FloatRegister dest) DEFINED_ON(arm64);
2840
2841 inline void leftShiftInt32x4(Imm32 count, FloatRegister src,
2842 FloatRegister dest)
2843 DEFINED_ON(x86_shared, arm64);
2844
2845 inline void leftShiftInt64x2(Register rhs, FloatRegister lhsDest)
2846 DEFINED_ON(x86_shared);
2847
2848 inline void leftShiftInt64x2(FloatRegister lhs, Register rhs,
2849 FloatRegister dest) DEFINED_ON(arm64);
2850
2851 inline void leftShiftInt64x2(Imm32 count, FloatRegister src,
2852 FloatRegister dest)
2853 DEFINED_ON(x86_shared, arm64);
2854
2855 // Right shift by scalar. Immediates and variable shifts must have been
2856 // masked; shifts of zero will work but may or may not generate code.
2857
2858 inline void rightShiftInt8x16(Register rhs, FloatRegister lhsDest,
2859 FloatRegister temp) DEFINED_ON(x86_shared);
2860
2861 inline void rightShiftInt8x16(FloatRegister lhs, Register rhs,
2862 FloatRegister dest) DEFINED_ON(arm64);
2863
2864 inline void rightShiftInt8x16(Imm32 count, FloatRegister src,
2865 FloatRegister dest)
2866 DEFINED_ON(x86_shared, arm64);
2867
2868 inline void unsignedRightShiftInt8x16(Register rhs, FloatRegister lhsDest,
2869 FloatRegister temp)
2870 DEFINED_ON(x86_shared);
2871
2872 inline void unsignedRightShiftInt8x16(FloatRegister lhs, Register rhs,
2873 FloatRegister dest) DEFINED_ON(arm64);
2874
2875 inline void unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
2876 FloatRegister dest)
2877 DEFINED_ON(x86_shared, arm64);
2878
2879 inline void rightShiftInt16x8(Register rhs, FloatRegister lhsDest)
2880 DEFINED_ON(x86_shared);
2881
2882 inline void rightShiftInt16x8(FloatRegister lhs, Register rhs,
2883 FloatRegister dest) DEFINED_ON(arm64);
2884
2885 inline void rightShiftInt16x8(Imm32 count, FloatRegister src,
2886 FloatRegister dest)
2887 DEFINED_ON(x86_shared, arm64);
2888
2889 inline void unsignedRightShiftInt16x8(Register rhs, FloatRegister lhsDest)
2890 DEFINED_ON(x86_shared);
2891
2892 inline void unsignedRightShiftInt16x8(FloatRegister lhs, Register rhs,
2893 FloatRegister dest) DEFINED_ON(arm64);
2894
2895 inline void unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
2896 FloatRegister dest)
2897 DEFINED_ON(x86_shared, arm64);
2898
2899 inline void rightShiftInt32x4(Register rhs, FloatRegister lhsDest)
2900 DEFINED_ON(x86_shared);
2901
2902 inline void rightShiftInt32x4(FloatRegister lhs, Register rhs,
2903 FloatRegister dest) DEFINED_ON(arm64);
2904
2905 inline void rightShiftInt32x4(Imm32 count, FloatRegister src,
2906 FloatRegister dest)
2907 DEFINED_ON(x86_shared, arm64);
2908
2909 inline void unsignedRightShiftInt32x4(Register rhs, FloatRegister lhsDest)
2910 DEFINED_ON(x86_shared);
2911
2912 inline void unsignedRightShiftInt32x4(FloatRegister lhs, Register rhs,
2913 FloatRegister dest) DEFINED_ON(arm64);
2914
2915 inline void unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
2916 FloatRegister dest)
2917 DEFINED_ON(x86_shared, arm64);
2918
2919 inline void rightShiftInt64x2(Register rhs, FloatRegister lhsDest,
2920 FloatRegister temp) DEFINED_ON(x86_shared);
2921
2922 inline void rightShiftInt64x2(Imm32 count, FloatRegister src,
2923 FloatRegister dest)
2924 DEFINED_ON(x86_shared, arm64);
2925
2926 inline void rightShiftInt64x2(FloatRegister lhs, Register rhs,
2927 FloatRegister dest) DEFINED_ON(arm64);
2928
2929 inline void unsignedRightShiftInt64x2(Register rhs, FloatRegister lhsDest)
2930 DEFINED_ON(x86_shared);
2931
2932 inline void unsignedRightShiftInt64x2(FloatRegister lhs, Register rhs,
2933 FloatRegister dest) DEFINED_ON(arm64);
2934
2935 inline void unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
2936 FloatRegister dest)
2937 DEFINED_ON(x86_shared, arm64);
2938
2939 // Sign replication operation
2940
2941 inline void signReplicationInt8x16(FloatRegister src, FloatRegister dest)
2942 DEFINED_ON(x86_shared);
2943
2944 inline void signReplicationInt16x8(FloatRegister src, FloatRegister dest)
2945 DEFINED_ON(x86_shared);
2946
2947 inline void signReplicationInt32x4(FloatRegister src, FloatRegister dest)
2948 DEFINED_ON(x86_shared);
2949
2950 inline void signReplicationInt64x2(FloatRegister src, FloatRegister dest)
2951 DEFINED_ON(x86_shared);
2952
2953 // Bitwise and, or, xor, not
2954
2955 inline void bitwiseAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
2956 DEFINED_ON(x86_shared, arm64);
2957
2958 inline void bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
2959 FloatRegister dest)
2960 DEFINED_ON(x86_shared, arm64);
2961
2962 inline void bitwiseAndSimd128(FloatRegister lhs, const SimdConstant& rhs,
2963 FloatRegister dest) DEFINED_ON(x86_shared);
2964
2965 inline void bitwiseOrSimd128(FloatRegister rhs, FloatRegister lhsDest)
2966 DEFINED_ON(x86_shared, arm64);
2967
2968 inline void bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
2969 FloatRegister dest)
2970 DEFINED_ON(x86_shared, arm64);
2971
2972 inline void bitwiseOrSimd128(FloatRegister lhs, const SimdConstant& rhs,
2973 FloatRegister dest) DEFINED_ON(x86_shared);
2974
2975 inline void bitwiseXorSimd128(FloatRegister rhs, FloatRegister lhsDest)
2976 DEFINED_ON(x86_shared, arm64);
2977
2978 inline void bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
2979 FloatRegister dest)
2980 DEFINED_ON(x86_shared, arm64);
2981
2982 inline void bitwiseXorSimd128(FloatRegister lhs, const SimdConstant& rhs,
2983 FloatRegister dest) DEFINED_ON(x86_shared);
2984
2985 inline void bitwiseNotSimd128(FloatRegister src, FloatRegister dest)
2986 DEFINED_ON(x86_shared, arm64);
2987
2988 // Bitwise AND with compliment: dest = lhs & ~rhs, note only arm64 can do it.
2989 inline void bitwiseAndNotSimd128(FloatRegister lhs, FloatRegister rhs,
2990 FloatRegister lhsDest) DEFINED_ON(arm64);
2991
2992 // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
2993 // wants but what the x86 hardware offers. Hence the name.
2994
2995 inline void bitwiseNotAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
2996 DEFINED_ON(x86_shared, arm64);
2997
2998 inline void bitwiseNotAndSimd128(FloatRegister lhs, FloatRegister rhs,
2999 FloatRegister lhsDest)
3000 DEFINED_ON(x86_shared);
3001
3002 // Bitwise select
3003
3004 inline void bitwiseSelectSimd128(FloatRegister mask, FloatRegister onTrue,
3005 FloatRegister onFalse, FloatRegister dest,
3006 FloatRegister temp) DEFINED_ON(x86_shared);
3007
3008 inline void bitwiseSelectSimd128(FloatRegister onTrue, FloatRegister onFalse,
3009 FloatRegister maskDest) DEFINED_ON(arm64);
3010
3011 // Population count
3012
3013 inline void popcntInt8x16(FloatRegister src, FloatRegister dest,
3014 FloatRegister temp) DEFINED_ON(x86_shared);
3015
3016 inline void popcntInt8x16(FloatRegister src, FloatRegister dest)
3017 DEFINED_ON(arm64);
3018
3019 // Any lane true, ie, any bit set
3020
3021 inline void anyTrueSimd128(FloatRegister src, Register dest)
3022 DEFINED_ON(x86, x64, arm64);
3023
3024 // All lanes true
3025
3026 inline void allTrueInt8x16(FloatRegister src, Register dest)
3027 DEFINED_ON(x86_shared, arm64);
3028
3029 inline void allTrueInt16x8(FloatRegister src, Register dest)
3030 DEFINED_ON(x86_shared, arm64);
3031
3032 inline void allTrueInt32x4(FloatRegister src, Register dest)
3033 DEFINED_ON(x86_shared, arm64);
3034
3035 inline void allTrueInt64x2(FloatRegister src, Register dest)
3036 DEFINED_ON(x86_shared, arm64);
3037
3038 // Bitmask, ie extract and compress high bits of all lanes
3039
3040 inline void bitmaskInt8x16(FloatRegister src, Register dest)
3041 DEFINED_ON(x86_shared);
3042
3043 inline void bitmaskInt8x16(FloatRegister src, Register dest,
3044 FloatRegister temp) DEFINED_ON(arm64);
3045
3046 inline void bitmaskInt16x8(FloatRegister src, Register dest)
3047 DEFINED_ON(x86_shared);
3048
3049 inline void bitmaskInt16x8(FloatRegister src, Register dest,
3050 FloatRegister temp) DEFINED_ON(arm64);
3051
3052 inline void bitmaskInt32x4(FloatRegister src, Register dest)
3053 DEFINED_ON(x86_shared);
3054
3055 inline void bitmaskInt32x4(FloatRegister src, Register dest,
3056 FloatRegister temp) DEFINED_ON(arm64);
3057
3058 inline void bitmaskInt64x2(FloatRegister src, Register dest)
3059 DEFINED_ON(x86_shared);
3060
3061 inline void bitmaskInt64x2(FloatRegister src, Register dest,
3062 FloatRegister temp) DEFINED_ON(arm64);
3063
3064 // Comparisons (integer and floating-point)
3065
3066 inline void compareInt8x16(Assembler::Condition cond, FloatRegister rhs,
3067 FloatRegister lhsDest)
3068 DEFINED_ON(x86_shared, arm64);
3069
3070 // On x86_shared, limited to !=, ==, <=, >
3071 inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
3072 const SimdConstant& rhs, FloatRegister dest)
3073 DEFINED_ON(x86_shared);
3074
3075 // On arm64, use any integer comparison condition.
3076 inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
3077 FloatRegister rhs, FloatRegister dest)
3078 DEFINED_ON(arm64);
3079
3080 inline void compareInt16x8(Assembler::Condition cond, FloatRegister rhs,
3081 FloatRegister lhsDest)
3082 DEFINED_ON(x86_shared, arm64);
3083
3084 inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
3085 FloatRegister rhs, FloatRegister dest)
3086 DEFINED_ON(arm64);
3087
3088 // On x86_shared, limited to !=, ==, <=, >
3089 inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
3090 const SimdConstant& rhs, FloatRegister dest)
3091 DEFINED_ON(x86_shared);
3092
3093 // On x86_shared, limited to !=, ==, <=, >
3094 inline void compareInt32x4(Assembler::Condition cond, FloatRegister rhs,
3095 FloatRegister lhsDest)
3096 DEFINED_ON(x86_shared, arm64);
3097
3098 inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
3099 const SimdConstant& rhs, FloatRegister dest)
3100 DEFINED_ON(x86_shared);
3101
3102 // On arm64, use any integer comparison condition.
3103 inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
3104 FloatRegister rhs, FloatRegister dest)
3105 DEFINED_ON(x86_shared, arm64);
3106
3107 inline void compareForEqualityInt64x2(Assembler::Condition cond,
3108 FloatRegister rhs,
3109 FloatRegister lhsDest)
3110 DEFINED_ON(x86_shared);
3111
3112 inline void compareForOrderingInt64x2(
3113 Assembler::Condition cond, FloatRegister rhs, FloatRegister lhsDest,
3114 FloatRegister temp1, FloatRegister temp2) DEFINED_ON(x86_shared);
3115
3116 inline void compareInt64x2(Assembler::Condition cond, FloatRegister rhs,
3117 FloatRegister lhsDest) DEFINED_ON(arm64);
3118
3119 inline void compareInt64x2(Assembler::Condition cond, FloatRegister lhs,
3120 FloatRegister rhs, FloatRegister dest)
3121 DEFINED_ON(arm64);
3122
3123 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister rhs,
3124 FloatRegister lhsDest)
3125 DEFINED_ON(x86_shared, arm64);
3126
3127 // On x86_shared, limited to ==, !=, <, <=
3128 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
3129 const SimdConstant& rhs, FloatRegister dest)
3130 DEFINED_ON(x86_shared);
3131
3132 // On x86_shared, limited to ==, !=, <, <=
3133 // On arm64, use any float-point comparison condition.
3134 inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
3135 FloatRegister rhs, FloatRegister dest)
3136 DEFINED_ON(x86_shared, arm64);
3137
3138 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister rhs,
3139 FloatRegister lhsDest)
3140 DEFINED_ON(x86_shared, arm64);
3141
3142 // On x86_shared, limited to ==, !=, <, <=
3143 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
3144 const SimdConstant& rhs, FloatRegister dest)
3145 DEFINED_ON(x86_shared);
3146
3147 // On arm64, use any float-point comparison condition.
3148 inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
3149 FloatRegister rhs, FloatRegister dest)
3150 DEFINED_ON(arm64);
3151
3152 // Load
3153
3154 inline void loadUnalignedSimd128(const Operand& src, FloatRegister dest)
3155 DEFINED_ON(x86_shared);
3156
3157 inline void loadUnalignedSimd128(const Address& src, FloatRegister dest)
3158 DEFINED_ON(x86_shared, arm64);
3159
3160 inline void loadUnalignedSimd128(const BaseIndex& src, FloatRegister dest)
3161 DEFINED_ON(x86_shared, arm64);
3162
3163 // Store
3164
3165 inline void storeUnalignedSimd128(FloatRegister src, const Address& dest)
3166 DEFINED_ON(x86_shared, arm64);
3167
3168 inline void storeUnalignedSimd128(FloatRegister src, const BaseIndex& dest)
3169 DEFINED_ON(x86_shared, arm64);
3170
3171 // Floating point negation
3172
3173 inline void negFloat32x4(FloatRegister src, FloatRegister dest)
3174 DEFINED_ON(x86_shared, arm64);
3175
3176 inline void negFloat64x2(FloatRegister src, FloatRegister dest)
3177 DEFINED_ON(x86_shared, arm64);
3178
3179 // Floating point absolute value
3180
3181 inline void absFloat32x4(FloatRegister src, FloatRegister dest)
3182 DEFINED_ON(x86_shared, arm64);
3183
3184 inline void absFloat64x2(FloatRegister src, FloatRegister dest)
3185 DEFINED_ON(x86_shared, arm64);
3186
3187 // NaN-propagating minimum
3188
3189 inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
3190 FloatRegister dest, FloatRegister temp1,
3191 FloatRegister temp2) DEFINED_ON(x86_shared);
3192
3193 inline void minFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3194 DEFINED_ON(arm64);
3195
3196 inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
3197 FloatRegister dest) DEFINED_ON(arm64);
3198
3199 inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
3200 FloatRegister dest, FloatRegister temp1,
3201 FloatRegister temp2) DEFINED_ON(x86_shared);
3202
3203 inline void minFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3204 DEFINED_ON(arm64);
3205
3206 inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
3207 FloatRegister dest) DEFINED_ON(arm64);
3208
3209 // NaN-propagating maximum
3210
3211 inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
3212 FloatRegister dest, FloatRegister temp1,
3213 FloatRegister temp2) DEFINED_ON(x86_shared);
3214
3215 inline void maxFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3216 DEFINED_ON(arm64);
3217
3218 inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
3219 FloatRegister dest) DEFINED_ON(arm64);
3220
3221 inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
3222 FloatRegister dest, FloatRegister temp1,
3223 FloatRegister temp2) DEFINED_ON(x86_shared);
3224
3225 inline void maxFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3226 DEFINED_ON(arm64);
3227
3228 inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
3229 FloatRegister dest) DEFINED_ON(arm64);
3230
3231 // Floating add
3232
3233 inline void addFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3234 DEFINED_ON(x86_shared, arm64);
3235
3236 inline void addFloat32x4(FloatRegister lhs, FloatRegister rhs,
3237 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
3238
3239 inline void addFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
3240 FloatRegister dest) DEFINED_ON(x86_shared);
3241
3242 inline void addFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3243 DEFINED_ON(x86_shared, arm64);
3244
3245 inline void addFloat64x2(FloatRegister lhs, FloatRegister rhs,
3246 FloatRegister dest) DEFINED_ON(arm64);
3247
3248 inline void addFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
3249 FloatRegister dest) DEFINED_ON(x86_shared);
3250
3251 // Floating subtract
3252
3253 inline void subFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3254 DEFINED_ON(x86_shared, arm64);
3255
3256 inline void subFloat32x4(FloatRegister lhs, FloatRegister rhs,
3257 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
3258
3259 inline void subFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
3260 FloatRegister dest) DEFINED_ON(x86_shared);
3261
3262 inline void subFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3263 DEFINED_ON(x86_shared, arm64);
3264
3265 inline void subFloat64x2(FloatRegister lhs, FloatRegister rhs,
3266 FloatRegister dest) DEFINED_ON(arm64);
3267
3268 inline void subFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
3269 FloatRegister dest) DEFINED_ON(x86_shared);
3270
3271 // Floating division
3272
3273 inline void divFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3274 DEFINED_ON(x86_shared, arm64);
3275
3276 inline void divFloat32x4(FloatRegister lhs, FloatRegister rhs,
3277 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
3278
3279 inline void divFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
3280 FloatRegister dest) DEFINED_ON(x86_shared);
3281
3282 inline void divFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3283 DEFINED_ON(x86_shared, arm64);
3284
3285 inline void divFloat64x2(FloatRegister lhs, FloatRegister rhs,
3286 FloatRegister dest) DEFINED_ON(arm64);
3287
3288 inline void divFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
3289 FloatRegister dest) DEFINED_ON(x86_shared);
3290
3291 // Floating Multiply
3292
3293 inline void mulFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
3294 DEFINED_ON(x86_shared, arm64);
3295
3296 inline void mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
3297 FloatRegister dest) DEFINED_ON(x86_shared, arm64);
3298
3299 inline void mulFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
3300 FloatRegister dest) DEFINED_ON(x86_shared);
3301
3302 inline void mulFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
3303 DEFINED_ON(x86_shared, arm64);
3304
3305 inline void mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
3306 FloatRegister dest) DEFINED_ON(arm64);
3307
3308 inline void mulFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
3309 FloatRegister dest) DEFINED_ON(x86_shared);
3310
3311 // Pairwise add
3312
3313 inline void extAddPairwiseInt8x16(FloatRegister src, FloatRegister dest)
3314 DEFINED_ON(x86_shared, arm64);
3315
3316 inline void unsignedExtAddPairwiseInt8x16(FloatRegister src,
3317 FloatRegister dest)
3318 DEFINED_ON(x86_shared, arm64);
3319
3320 inline void extAddPairwiseInt16x8(FloatRegister src, FloatRegister dest)
3321 DEFINED_ON(x86_shared, arm64);
3322
3323 inline void unsignedExtAddPairwiseInt16x8(FloatRegister src,
3324 FloatRegister dest)
3325 DEFINED_ON(x86_shared, arm64);
3326
3327 // Floating square root
3328
3329 inline void sqrtFloat32x4(FloatRegister src, FloatRegister dest)
3330 DEFINED_ON(x86_shared, arm64);
3331
3332 inline void sqrtFloat64x2(FloatRegister src, FloatRegister dest)
3333 DEFINED_ON(x86_shared, arm64);
3334
3335 // Integer to floating point with rounding
3336
3337 inline void convertInt32x4ToFloat32x4(FloatRegister src, FloatRegister dest)
3338 DEFINED_ON(x86_shared, arm64);
3339
3340 inline void unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
3341 FloatRegister dest)
3342 DEFINED_ON(x86_shared, arm64);
3343
3344 inline void convertInt32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
3345 DEFINED_ON(x86_shared, arm64);
3346
3347 inline void unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
3348 FloatRegister dest)
3349 DEFINED_ON(x86_shared, arm64);
3350
3351 // Floating point to integer with saturation
3352
3353 inline void truncSatFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest)
3354 DEFINED_ON(x86_shared, arm64);
3355
3356 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
3357 FloatRegister dest,
3358 FloatRegister temp)
3359 DEFINED_ON(x86_shared);
3360
3361 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
3362 FloatRegister dest)
3363 DEFINED_ON(arm64);
3364
3365 inline void truncSatFloat64x2ToInt32x4(FloatRegister src, FloatRegister dest,
3366 FloatRegister temp)
3367 DEFINED_ON(x86_shared, arm64);
3368
3369 inline void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
3370 FloatRegister dest,
3371 FloatRegister temp)
3372 DEFINED_ON(x86_shared, arm64);
3373
3374 inline void truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
3375 FloatRegister dest)
3376 DEFINED_ON(x86_shared, arm64);
3377
3378 inline void unsignedTruncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
3379 FloatRegister dest)
3380 DEFINED_ON(x86_shared, arm64);
3381
3382 inline void truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
3383 FloatRegister dest)
3384 DEFINED_ON(x86_shared, arm64);
3385
3386 inline void unsignedTruncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
3387 FloatRegister dest)
3388 DEFINED_ON(x86_shared, arm64);
3389
3390 // Floating point narrowing
3391
3392 inline void convertFloat64x2ToFloat32x4(FloatRegister src, FloatRegister dest)
3393 DEFINED_ON(x86_shared, arm64);
3394
3395 // Floating point widening
3396
3397 inline void convertFloat32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
3398 DEFINED_ON(x86_shared, arm64);
3399
3400 // Integer to integer narrowing
3401
3402 inline void narrowInt16x8(FloatRegister rhs, FloatRegister lhsDest)
3403 DEFINED_ON(x86_shared, arm64);
3404
3405 inline void narrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
3406 FloatRegister dest) DEFINED_ON(x86_shared);
3407
3408 inline void narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
3409 FloatRegister dest) DEFINED_ON(arm64);
3410
3411 inline void unsignedNarrowInt16x8(FloatRegister rhs, FloatRegister lhsDest)
3412 DEFINED_ON(x86_shared, arm64);
3413
3414 inline void unsignedNarrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
3415 FloatRegister dest) DEFINED_ON(x86_shared);
3416
3417 inline void unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
3418 FloatRegister dest) DEFINED_ON(arm64);
3419
3420 inline void narrowInt32x4(FloatRegister rhs, FloatRegister lhsDest)
3421 DEFINED_ON(x86_shared, arm64);
3422
3423 inline void narrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
3424 FloatRegister dest) DEFINED_ON(x86_shared);
3425
3426 inline void narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
3427 FloatRegister dest) DEFINED_ON(arm64);
3428
3429 inline void unsignedNarrowInt32x4(FloatRegister rhs, FloatRegister lhsDest)
3430 DEFINED_ON(x86_shared, arm64);
3431
3432 inline void unsignedNarrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
3433 FloatRegister dest) DEFINED_ON(x86_shared);
3434
3435 inline void unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
3436 FloatRegister dest) DEFINED_ON(arm64);
3437
3438 // Integer to integer widening
3439
3440 inline void widenLowInt8x16(FloatRegister src, FloatRegister dest)
3441 DEFINED_ON(x86_shared, arm64);
3442
3443 inline void widenHighInt8x16(FloatRegister src, FloatRegister dest)
3444 DEFINED_ON(x86_shared, arm64);
3445
3446 inline void unsignedWidenLowInt8x16(FloatRegister src, FloatRegister dest)
3447 DEFINED_ON(x86_shared, arm64);
3448
3449 inline void unsignedWidenHighInt8x16(FloatRegister src, FloatRegister dest)
3450 DEFINED_ON(x86_shared, arm64);
3451
3452 inline void widenLowInt16x8(FloatRegister src, FloatRegister dest)
3453 DEFINED_ON(x86_shared, arm64);
3454
3455 inline void widenHighInt16x8(FloatRegister src, FloatRegister dest)
3456 DEFINED_ON(x86_shared, arm64);
3457
3458 inline void unsignedWidenLowInt16x8(FloatRegister src, FloatRegister dest)
3459 DEFINED_ON(x86_shared, arm64);
3460
3461 inline void unsignedWidenHighInt16x8(FloatRegister src, FloatRegister dest)
3462 DEFINED_ON(x86_shared, arm64);
3463
3464 inline void widenLowInt32x4(FloatRegister src, FloatRegister dest)
3465 DEFINED_ON(x86_shared, arm64);
3466
3467 inline void unsignedWidenLowInt32x4(FloatRegister src, FloatRegister dest)
3468 DEFINED_ON(x86_shared, arm64);
3469
3470 inline void widenHighInt32x4(FloatRegister src, FloatRegister dest)
3471 DEFINED_ON(x86_shared, arm64);
3472
3473 inline void unsignedWidenHighInt32x4(FloatRegister src, FloatRegister dest)
3474 DEFINED_ON(x86_shared, arm64);
3475
3476 // Compare-based minimum/maximum
3477 //
3478 // On x86, the signature is (rhsDest, lhs); on arm64 it is (rhs, lhsDest).
3479 //
3480 // The masm preprocessor can't deal with multiple declarations with identical
3481 // signatures even if they are on different platforms, hence the weird
3482 // argument names.
3483
3484 inline void pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
3485 FloatRegister lhsOrLhsDest)
3486 DEFINED_ON(x86_shared, arm64);
3487
3488 inline void pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
3489 FloatRegister dest) DEFINED_ON(arm64);
3490
3491 inline void pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
3492 FloatRegister lhsOrLhsDest)
3493 DEFINED_ON(x86_shared, arm64);
3494
3495 inline void pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
3496 FloatRegister dest) DEFINED_ON(arm64);
3497
3498 inline void pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
3499 FloatRegister lhsOrLhsDest)
3500 DEFINED_ON(x86_shared, arm64);
3501
3502 inline void pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
3503 FloatRegister dest) DEFINED_ON(arm64);
3504
3505 inline void pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
3506 FloatRegister lhsOrLhsDest)
3507 DEFINED_ON(x86_shared, arm64);
3508
3509 inline void pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
3510 FloatRegister dest) DEFINED_ON(arm64);
3511
3512 // Widening/pairwise integer dot product
3513
3514 inline void widenDotInt16x8(FloatRegister rhs, FloatRegister lhsDest)
3515 DEFINED_ON(x86_shared, arm64);
3516
3517 inline void widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
3518 FloatRegister dest) DEFINED_ON(arm64);
3519
3520 inline void widenDotInt16x8(FloatRegister lhs, const SimdConstant& rhs,
3521 FloatRegister dest) DEFINED_ON(x86_shared);
3522
3523 // Floating point rounding
3524
3525 inline void ceilFloat32x4(FloatRegister src, FloatRegister dest)
3526 DEFINED_ON(x86_shared, arm64);
3527
3528 inline void ceilFloat64x2(FloatRegister src, FloatRegister dest)
3529 DEFINED_ON(x86_shared, arm64);
3530
3531 inline void floorFloat32x4(FloatRegister src, FloatRegister dest)
3532 DEFINED_ON(x86_shared, arm64);
3533
3534 inline void floorFloat64x2(FloatRegister src, FloatRegister dest)
3535 DEFINED_ON(x86_shared, arm64);
3536
3537 inline void truncFloat32x4(FloatRegister src, FloatRegister dest)
3538 DEFINED_ON(x86_shared, arm64);
3539
3540 inline void truncFloat64x2(FloatRegister src, FloatRegister dest)
3541 DEFINED_ON(x86_shared, arm64);
3542
3543 inline void nearestFloat32x4(FloatRegister src, FloatRegister dest)
3544 DEFINED_ON(x86_shared, arm64);
3545
3546 inline void nearestFloat64x2(FloatRegister src, FloatRegister dest)
3547 DEFINED_ON(x86_shared, arm64);
3548
3549 // Floating multiply-accumulate: srcDest [+-]= src1 * src2
3550
3551 inline void fmaFloat32x4(FloatRegister src1, FloatRegister src2,
3552 FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
3553
3554 inline void fmsFloat32x4(FloatRegister src1, FloatRegister src2,
3555 FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
3556
3557 inline void fmaFloat64x2(FloatRegister src1, FloatRegister src2,
3558 FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
3559
3560 inline void fmsFloat64x2(FloatRegister src1, FloatRegister src2,
3561 FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
3562
3563 inline void minFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
3564 DEFINED_ON(x86_shared, arm64);
3565
3566 inline void minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
3567 FloatRegister dest) DEFINED_ON(arm64);
3568
3569 inline void maxFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
3570 DEFINED_ON(x86_shared, arm64);
3571
3572 inline void maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
3573 FloatRegister dest) DEFINED_ON(arm64);
3574
3575 inline void minFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
3576 DEFINED_ON(x86_shared, arm64);
3577
3578 inline void minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
3579 FloatRegister dest) DEFINED_ON(arm64);
3580
3581 inline void maxFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
3582 DEFINED_ON(x86_shared, arm64);
3583
3584 inline void maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
3585 FloatRegister dest) DEFINED_ON(arm64);
3586
3587 public:
3588 // ========================================================================
3589 // Truncate floating point.
3590
3591 // Undefined behaviour when truncation is outside Int64 range.
3592 // Needs a temp register if SSE3 is not present.
3593 inline void truncateFloat32ToInt64(Address src, Address dest, Register temp)
3594 DEFINED_ON(x86_shared);
3595 inline void truncateFloat32ToUInt64(Address src, Address dest, Register temp,
3596 FloatRegister floatTemp)
3597 DEFINED_ON(x86, x64);
3598 inline void truncateDoubleToInt64(Address src, Address dest, Register temp)
3599 DEFINED_ON(x86_shared);
3600 inline void truncateDoubleToUInt64(Address src, Address dest, Register temp,
3601 FloatRegister floatTemp)
3602 DEFINED_ON(x86, x64);
3603
3604 public:
3605 // ========================================================================
3606 // Convert floating point.
3607
3608 // temp required on x86 and x64; must be undefined on mips64 and loong64.
3609 void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp)
3610 DEFINED_ON(arm64, mips64, loong64, x64, x86);
3611
3612 void convertInt64ToFloat32(Register64 src, FloatRegister dest)
3613 DEFINED_ON(arm64, mips64, loong64, x64, x86);
3614
3615 bool convertUInt64ToDoubleNeedsTemp() PER_ARCH;
3616
3617 // temp required when convertUInt64ToDoubleNeedsTemp() returns true.
3618 void convertUInt64ToDouble(Register64 src, FloatRegister dest,
3619 Register temp) PER_ARCH;
3620
3621 void convertInt64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;
3622
3623 void convertIntPtrToDouble(Register src, FloatRegister dest) PER_ARCH;
3624
3625 public:
3626 // ========================================================================
3627 // wasm support
3628
3629 CodeOffset wasmTrapInstruction() PER_SHARED_ARCH;
3630
3631 void wasmTrap(wasm::Trap trap, wasm::BytecodeOffset bytecodeOffset);
3632 void wasmInterruptCheck(Register tls, wasm::BytecodeOffset bytecodeOffset);
3633 #ifdef ENABLE_WASM_EXCEPTIONS
3634 [[nodiscard]] bool wasmStartTry(size_t* tryNoteIndex);
3635 #endif
3636
3637 // Load all pinned regs via WasmTlsReg. If the trapOffset is something, give
3638 // the first load a trap descriptor with type IndirectCallToNull, so that a
3639 // null Tls will cause a trap.
3640 void loadWasmPinnedRegsFromTls(
3641 mozilla::Maybe<wasm::BytecodeOffset> trapOffset = mozilla::Nothing());
3642
3643 // Returns a pair: the offset of the undefined (trapping) instruction, and
3644 // the number of extra bytes of stack allocated prior to the trap
3645 // instruction proper.
3646 std::pair<CodeOffset, uint32_t> wasmReserveStackChecked(
3647 uint32_t amount, wasm::BytecodeOffset trapOffset);
3648
3649 // Emit a bounds check against the wasm heap limit, jumping to 'ok' if
3650 // 'cond' holds. If JitOptions.spectreMaskIndex is true, in speculative
3651 // executions 'index' is saturated in-place to 'boundsCheckLimit'.
3652 //
3653 // On 32-bit systems for both wasm and asm.js, and on 64-bit systems for
3654 // asm.js, heap lengths are limited to 2GB. On 64-bit systems for wasm,
3655 // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be
3656 // limited to something much larger.
3657
3658 void wasmBoundsCheck32(Condition cond, Register index,
3659 Register boundsCheckLimit, Label* ok)
3660 DEFINED_ON(arm, arm64, mips32, mips64, x86_shared, loong64);
3661
3662 void wasmBoundsCheck32(Condition cond, Register index,
3663 Address boundsCheckLimit, Label* ok)
3664 DEFINED_ON(arm, arm64, mips32, mips64, x86_shared, loong64);
3665
3666 void wasmBoundsCheck64(Condition cond, Register64 index,
3667 Register64 boundsCheckLimit, Label* ok)
3668 DEFINED_ON(arm64, mips64, x64, x86, arm, loong64);
3669
3670 void wasmBoundsCheck64(Condition cond, Register64 index,
3671 Address boundsCheckLimit, Label* ok)
3672 DEFINED_ON(arm64, mips64, x64, x86, arm, loong64);
3673
3674 // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds.
3675 void wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr,
3676 AnyRegister out) DEFINED_ON(x86, x64);
3677 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAddr,
3678 Register64 out) DEFINED_ON(x86, x64);
3679 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
3680 Operand dstAddr) DEFINED_ON(x86, x64);
3681 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
3682 Operand dstAddr) DEFINED_ON(x86);
3683
3684 // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr`
3685 // MUST equal `ptrScratch`, and that register will be updated based on
3686 // conditions listed below (where it is only mentioned as `ptr`).
3687
3688 // `ptr` will be updated if access.offset() != 0 or access.type() ==
3689 // Scalar::Int64.
3690 void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
3691 Register ptr, Register ptrScratch, AnyRegister output)
3692 DEFINED_ON(arm, loong64, mips_shared);
3693 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
3694 Register ptr, Register ptrScratch, Register64 output)
3695 DEFINED_ON(arm, mips32, mips64, loong64);
3696 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
3697 Register memoryBase, Register ptr, Register ptrScratch)
3698 DEFINED_ON(arm, loong64, mips_shared);
3699 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
3700 Register memoryBase, Register ptr, Register ptrScratch)
3701 DEFINED_ON(arm, mips32, mips64, loong64);
3702
3703 // These accept general memoryBase + ptr + offset (in `access`); the offset is
3704 // always smaller than the guard region. They will insert an additional add
3705 // if the offset is nonzero, and of course that add may require a temporary
3706 // register for the offset if the offset is large, and instructions to set it
3707 // up.
3708 void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
3709 Register ptr, AnyRegister output) DEFINED_ON(arm64);
3710 void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
3711 Register ptr, Register64 output) DEFINED_ON(arm64);
3712 void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
3713 Register memoryBase, Register ptr) DEFINED_ON(arm64);
3714 void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
3715 Register memoryBase, Register ptr) DEFINED_ON(arm64);
3716
3717 // `ptr` will always be updated.
3718 void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
3719 Register memoryBase, Register ptr, Register ptrScratch,
3720 Register output, Register tmp)
3721 DEFINED_ON(mips32, mips64);
3722
3723 // MIPS: `ptr` will always be updated.
3724 void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
3725 Register memoryBase, Register ptr,
3726 Register ptrScratch, FloatRegister output,
3727 Register tmp1) DEFINED_ON(mips32, mips64);
3728
3729 // `ptr` will always be updated.
3730 void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
3731 Register memoryBase, Register ptr,
3732 Register ptrScratch, Register64 output,
3733 Register tmp) DEFINED_ON(mips32, mips64);
3734
3735 // MIPS: `ptr` will always be updated.
3736 void wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value,
3737 Register memoryBase, Register ptr,
3738 Register ptrScratch, Register tmp)
3739 DEFINED_ON(mips32, mips64);
3740
3741 // `ptr` will always be updated.
3742 void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
3743 FloatRegister floatValue, Register memoryBase,
3744 Register ptr, Register ptrScratch, Register tmp)
3745 DEFINED_ON(mips32, mips64);
3746
3747 // `ptr` will always be updated.
3748 void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
3749 Register64 value, Register memoryBase,
3750 Register ptr, Register ptrScratch, Register tmp)
3751 DEFINED_ON(mips32, mips64);
3752
3753 // wasm specific methods, used in both the wasm baseline compiler and ion.
3754
3755 // The truncate-to-int32 methods do not bind the rejoin label; clients must
3756 // do so if oolWasmTruncateCheckF64ToI32() can jump to it.
3757 void wasmTruncateDoubleToUInt32(FloatRegister input, Register output,
3758 bool isSaturating, Label* oolEntry) PER_ARCH;
3759 void wasmTruncateDoubleToInt32(FloatRegister input, Register output,
3760 bool isSaturating,
3761 Label* oolEntry) PER_SHARED_ARCH;
3762 void oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output,
3763 TruncFlags flags, wasm::BytecodeOffset off,
3764 Label* rejoin)
3765 DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64);
3766
3767 void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output,
3768 bool isSaturating, Label* oolEntry) PER_ARCH;
3769 void wasmTruncateFloat32ToInt32(FloatRegister input, Register output,
3770 bool isSaturating,
3771 Label* oolEntry) PER_SHARED_ARCH;
3772 void oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output,
3773 TruncFlags flags, wasm::BytecodeOffset off,
3774 Label* rejoin)
3775 DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64);
3776
3777 // The truncate-to-int64 methods will always bind the `oolRejoin` label
3778 // after the last emitted instruction.
3779 void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output,
3780 bool isSaturating, Label* oolEntry,
3781 Label* oolRejoin, FloatRegister tempDouble)
3782 DEFINED_ON(arm64, x86, x64, mips64, loong64);
3783 void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output,
3784 bool isSaturating, Label* oolEntry,
3785 Label* oolRejoin, FloatRegister tempDouble)
3786 DEFINED_ON(arm64, x86, x64, mips64, loong64);
3787 void oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output,
3788 TruncFlags flags, wasm::BytecodeOffset off,
3789 Label* rejoin)
3790 DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64);
3791
3792 void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output,
3793 bool isSaturating, Label* oolEntry,
3794 Label* oolRejoin, FloatRegister tempDouble)
3795 DEFINED_ON(arm64, x86, x64, mips64, loong64);
3796 void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output,
3797 bool isSaturating, Label* oolEntry,
3798 Label* oolRejoin, FloatRegister tempDouble)
3799 DEFINED_ON(arm64, x86, x64, mips64, loong64);
3800 void oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output,
3801 TruncFlags flags, wasm::BytecodeOffset off,
3802 Label* rejoin)
3803 DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64);
3804
3805 void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest);
3806
3807 // This function takes care of loading the callee's TLS and pinned regs but
3808 // it is the caller's responsibility to save/restore TLS or pinned regs.
3809 CodeOffset wasmCallImport(const wasm::CallSiteDesc& desc,
3810 const wasm::CalleeDesc& callee);
3811
3812 // WasmTableCallIndexReg must contain the index of the indirect call. This is
3813 // for wasm calls only.
3814 //
3815 // Indirect calls use a dual-path mechanism where a run-time test determines
3816 // whether a context switch is needed (slow path) or not (fast path). This
3817 // gives rise to two call instructions, both of which need safe points. As
3818 // per normal, the call offsets are the code offsets at the end of the call
3819 // instructions (the return points).
3820 void wasmCallIndirect(const wasm::CallSiteDesc& desc,
3821 const wasm::CalleeDesc& callee, bool needsBoundsCheck,
3822 mozilla::Maybe<uint32_t> tableSize,
3823 CodeOffset* fastCallOffset, CodeOffset* slowCallOffset);
3824
3825 // WasmTableCallIndexReg must contain the index of the indirect call.
3826 // This is for asm.js calls only.
3827 CodeOffset asmCallIndirect(const wasm::CallSiteDesc& desc,
3828 const wasm::CalleeDesc& callee);
3829
3830 // This function takes care of loading the pointer to the current instance
3831 // as the implicit first argument. It preserves TLS and pinned registers.
3832 // (TLS & pinned regs are non-volatile registers in the system ABI).
3833 CodeOffset wasmCallBuiltinInstanceMethod(const wasm::CallSiteDesc& desc,
3834 const ABIArg& instanceArg,
3835 wasm::SymbolicAddress builtin,
3836 wasm::FailureMode failureMode);
3837
3838 // Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
3839 // May destroy indexTemp32. The value of indexTemp32 must be positive, and it
3840 // is implementation-defined what happens if bits are lost or the value
3841 // becomes negative through the shift. On 64-bit systems, the high 32 bits of
3842 // indexTemp32 must be zero, not garbage.
3843 void shiftIndex32AndAdd(Register indexTemp32, int shift,
3844 Register pointer) PER_SHARED_ARCH;
3845
3846 // The System ABI frequently states that the high bits of a 64-bit register
3847 // that holds a 32-bit return value are unpredictable, and C++ compilers will
3848 // indeed generate code that leaves garbage in the upper bits.
3849 //
3850 // Adjust the contents of the 64-bit register `r` to conform to our internal
3851 // convention, which requires predictable high bits. In practice, this means
3852 // that the 32-bit value will be zero-extended or sign-extended to 64 bits as
3853 // appropriate for the platform.
3854 void widenInt32(Register r) DEFINED_ON(arm64, x64, mips64, loong64);
3855
3856 // As enterFakeExitFrame(), but using register conventions appropriate for
3857 // wasm stubs.
3858 void enterFakeExitFrameForWasm(Register cxreg, Register scratch,
3859 ExitFrameType type) PER_SHARED_ARCH;
3860
3861 public:
3862 // ========================================================================
3863 // Barrier functions.
3864
3865 void emitPreBarrierFastPath(JSRuntime* rt, MIRType type, Register temp1,
3866 Register temp2, Register temp3, Label* noBarrier);
3867
3868 public:
3869 // ========================================================================
3870 // Clamping functions.
3871
3872 inline void clampIntToUint8(Register reg) PER_SHARED_ARCH;
3873
3874 public:
3875 // ========================================================================
3876 // Primitive atomic operations.
3877 //
3878 // If the access is from JS and the eventual destination of the result is a
3879 // js::Value, it's probably best to use the JS-specific versions of these,
3880 // see further below.
3881 //
3882 // Temp registers must be defined unless otherwise noted in the per-function
3883 // constraints.
3884
3885 // 8-bit, 16-bit, and 32-bit wide operations.
3886 //
3887 // The 8-bit and 16-bit operations zero-extend or sign-extend the result to
3888 // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of the
3889 // result will be zero on some platforms (eg, on x64) and will be the sign
3890 // extension of the lower bits on other platforms (eg, MIPS).
3891
3892 // CompareExchange with memory. Return the value that was in memory,
3893 // whether we wrote or not.
3894 //
3895 // x86-shared: `output` must be eax.
3896 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
3897 // and 16-bit wide operations.
3898
3899 void compareExchange(Scalar::Type type, const Synchronization& sync,
3900 const Address& mem, Register expected,
3901 Register replacement, Register output)
3902 DEFINED_ON(arm, arm64, x86_shared);
3903
3904 void compareExchange(Scalar::Type type, const Synchronization& sync,
3905 const BaseIndex& mem, Register expected,
3906 Register replacement, Register output)
3907 DEFINED_ON(arm, arm64, x86_shared);
3908
3909 void compareExchange(Scalar::Type type, const Synchronization& sync,
3910 const Address& mem, Register expected,
3911 Register replacement, Register valueTemp,
3912 Register offsetTemp, Register maskTemp, Register output)
3913 DEFINED_ON(mips_shared, loong64);
3914
3915 void compareExchange(Scalar::Type type, const Synchronization& sync,
3916 const BaseIndex& mem, Register expected,
3917 Register replacement, Register valueTemp,
3918 Register offsetTemp, Register maskTemp, Register output)
3919 DEFINED_ON(mips_shared, loong64);
3920
3921 // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx.
3922 // x64: `output` must be rax.
3923 // ARM: Registers must be distinct; `replacement` and `output` must be
3924 // (even,odd) pairs.
3925
3926 void compareExchange64(const Synchronization& sync, const Address& mem,
3927 Register64 expected, Register64 replacement,
3928 Register64 output)
3929 DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
3930
3931 void compareExchange64(const Synchronization& sync, const BaseIndex& mem,
3932 Register64 expected, Register64 replacement,
3933 Register64 output)
3934 DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
3935
3936 // Exchange with memory. Return the value initially in memory.
3937 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
3938 // and 16-bit wide operations.
3939
3940 void atomicExchange(Scalar::Type type, const Synchronization& sync,
3941 const Address& mem, Register value, Register output)
3942 DEFINED_ON(arm, arm64, x86_shared);
3943
3944 void atomicExchange(Scalar::Type type, const Synchronization& sync,
3945 const BaseIndex& mem, Register value, Register output)
3946 DEFINED_ON(arm, arm64, x86_shared);
3947
3948 void atomicExchange(Scalar::Type type, const Synchronization& sync,
3949 const Address& mem, Register value, Register valueTemp,
3950 Register offsetTemp, Register maskTemp, Register output)
3951 DEFINED_ON(mips_shared, loong64);
3952
3953 void atomicExchange(Scalar::Type type, const Synchronization& sync,
3954 const BaseIndex& mem, Register value, Register valueTemp,
3955 Register offsetTemp, Register maskTemp, Register output)
3956 DEFINED_ON(mips_shared, loong64);
3957
3958 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
3959 // ARM: `value` and `output` must be distinct and (even,odd) pairs.
3960 // ARM64: `value` and `output` must be distinct.
3961
3962 void atomicExchange64(const Synchronization& sync, const Address& mem,
3963 Register64 value, Register64 output)
3964 DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
3965
3966 void atomicExchange64(const Synchronization& sync, const BaseIndex& mem,
3967 Register64 value, Register64 output)
3968 DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
3969
3970 // Read-modify-write with memory. Return the value in memory before the
3971 // operation.
3972 //
3973 // x86-shared:
3974 // For 8-bit operations, `value` and `output` must have a byte subregister.
3975 // For Add and Sub, `temp` must be invalid.
3976 // For And, Or, and Xor, `output` must be eax and `temp` must have a byte
3977 // subregister.
3978 //
3979 // ARM: Registers `value` and `output` must differ.
3980 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
3981 // and 16-bit wide operations; `value` and `output` must differ.
3982
3983 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
3984 AtomicOp op, Register value, const Address& mem,
3985 Register temp, Register output)
3986 DEFINED_ON(arm, arm64, x86_shared);
3987
3988 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
3989 AtomicOp op, Imm32 value, const Address& mem,
3990 Register temp, Register output) DEFINED_ON(x86_shared);
3991
3992 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
3993 AtomicOp op, Register value, const BaseIndex& mem,
3994 Register temp, Register output)
3995 DEFINED_ON(arm, arm64, x86_shared);
3996
3997 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
3998 AtomicOp op, Imm32 value, const BaseIndex& mem,
3999 Register temp, Register output) DEFINED_ON(x86_shared);
4000
4001 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
4002 AtomicOp op, Register value, const Address& mem,
4003 Register valueTemp, Register offsetTemp, Register maskTemp,
4004 Register output) DEFINED_ON(mips_shared, loong64);
4005
4006 void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
4007 AtomicOp op, Register value, const BaseIndex& mem,
4008 Register valueTemp, Register offsetTemp, Register maskTemp,
4009 Register output) DEFINED_ON(mips_shared, loong64);
4010
4011 // x86:
4012 // `temp` must be ecx:ebx; `output` must be edx:eax.
4013 // x64:
4014 // For Add and Sub, `temp` is ignored.
4015 // For And, Or, and Xor, `output` must be rax.
4016 // ARM:
4017 // `temp` and `output` must be (even,odd) pairs and distinct from `value`.
4018 // ARM64:
4019 // Registers `value`, `temp`, and `output` must all differ.
4020
4021 void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
4022 Register64 value, const Address& mem, Register64 temp,
4023 Register64 output)
4024 DEFINED_ON(arm, arm64, x64, mips64, loong64);
4025
4026 void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
4027 const Address& value, const Address& mem,
4028 Register64 temp, Register64 output) DEFINED_ON(x86);
4029
4030 void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
4031 Register64 value, const BaseIndex& mem, Register64 temp,
4032 Register64 output)
4033 DEFINED_ON(arm, arm64, x64, mips64, loong64);
4034
4035 void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
4036 const Address& value, const BaseIndex& mem,
4037 Register64 temp, Register64 output) DEFINED_ON(x86);
4038
4039 // x64:
4040 // `value` can be any register.
4041 // ARM:
4042 // `temp` must be an (even,odd) pair and distinct from `value`.
4043 // ARM64:
4044 // Registers `value` and `temp` must differ.
4045
4046 void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
4047 Register64 value, const Address& mem) DEFINED_ON(x64);
4048
4049 void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
4050 Register64 value, const Address& mem, Register64 temp)
4051 DEFINED_ON(arm, arm64, mips64, loong64);
4052
4053 void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
4054 Register64 value, const BaseIndex& mem) DEFINED_ON(x64);
4055
4056 void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
4057 Register64 value, const BaseIndex& mem, Register64 temp)
4058 DEFINED_ON(arm, arm64, mips64, loong64);
4059
4060 // 64-bit atomic load. On 64-bit systems, use regular load with
4061 // Synchronization::Load, not this method.
4062 //
4063 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4064 // ARM: `output` must be (even,odd) pair.
4065
4066 void atomicLoad64(const Synchronization& sync, const Address& mem,
4067 Register64 temp, Register64 output) DEFINED_ON(x86);
4068
4069 void atomicLoad64(const Synchronization& sync, const BaseIndex& mem,
4070 Register64 temp, Register64 output) DEFINED_ON(x86);
4071
4072 void atomicLoad64(const Synchronization& sync, const Address& mem,
4073 Register64 output) DEFINED_ON(arm);
4074
4075 void atomicLoad64(const Synchronization& sync, const BaseIndex& mem,
4076 Register64 output) DEFINED_ON(arm);
4077
4078 // 64-bit atomic store. On 64-bit systems, use regular store with
4079 // Synchronization::Store, not this method.
4080 //
4081 // x86: `value` must be ecx:ebx; `temp` must be edx:eax.
4082 // ARM: `value` and `temp` must be (even,odd) pairs.
4083
4084 void atomicStore64(const Synchronization& sync, const Address& mem,
4085 Register64 value, Register64 temp) DEFINED_ON(x86, arm);
4086
4087 void atomicStore64(const Synchronization& sync, const BaseIndex& mem,
4088 Register64 value, Register64 temp) DEFINED_ON(x86, arm);
4089
4090 // ========================================================================
4091 // Wasm atomic operations.
4092 //
4093 // Constraints, when omitted, are exactly as for the primitive operations
4094 // above.
4095
4096 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4097 const Address& mem, Register expected,
4098 Register replacement, Register output)
4099 DEFINED_ON(arm, arm64, x86_shared);
4100
4101 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4102 const BaseIndex& mem, Register expected,
4103 Register replacement, Register output)
4104 DEFINED_ON(arm, arm64, x86_shared);
4105
4106 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4107 const Address& mem, Register expected,
4108 Register replacement, Register valueTemp,
4109 Register offsetTemp, Register maskTemp,
4110 Register output) DEFINED_ON(mips_shared, loong64);
4111
4112 void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
4113 const BaseIndex& mem, Register expected,
4114 Register replacement, Register valueTemp,
4115 Register offsetTemp, Register maskTemp,
4116 Register output) DEFINED_ON(mips_shared, loong64);
4117
4118 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
4119 const Address& mem, Register value, Register output)
4120 DEFINED_ON(arm, arm64, x86_shared);
4121
4122 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
4123 const BaseIndex& mem, Register value, Register output)
4124 DEFINED_ON(arm, arm64, x86_shared);
4125
4126 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
4127 const Address& mem, Register value,
4128 Register valueTemp, Register offsetTemp,
4129 Register maskTemp, Register output)
4130 DEFINED_ON(mips_shared, loong64);
4131
4132 void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
4133 const BaseIndex& mem, Register value,
4134 Register valueTemp, Register offsetTemp,
4135 Register maskTemp, Register output)
4136 DEFINED_ON(mips_shared, loong64);
4137
4138 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4139 Register value, const Address& mem, Register temp,
4140 Register output) DEFINED_ON(arm, arm64, x86_shared);
4141
4142 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4143 Imm32 value, const Address& mem, Register temp,
4144 Register output) DEFINED_ON(x86_shared);
4145
4146 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4147 Register value, const BaseIndex& mem, Register temp,
4148 Register output) DEFINED_ON(arm, arm64, x86_shared);
4149
4150 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4151 Imm32 value, const BaseIndex& mem, Register temp,
4152 Register output) DEFINED_ON(x86_shared);
4153
4154 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4155 Register value, const Address& mem, Register valueTemp,
4156 Register offsetTemp, Register maskTemp,
4157 Register output) DEFINED_ON(mips_shared, loong64);
4158
4159 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4160 Register value, const BaseIndex& mem,
4161 Register valueTemp, Register offsetTemp,
4162 Register maskTemp, Register output)
4163 DEFINED_ON(mips_shared, loong64);
4164
4165 // Read-modify-write with memory. Return no value.
4166 //
4167 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4168 // and 16-bit wide operations.
4169
4170 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4171 Register value, const Address& mem, Register temp)
4172 DEFINED_ON(arm, arm64, x86_shared);
4173
4174 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4175 Imm32 value, const Address& mem, Register temp)
4176 DEFINED_ON(x86_shared);
4177
4178 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4179 Register value, const BaseIndex& mem, Register temp)
4180 DEFINED_ON(arm, arm64, x86_shared);
4181
4182 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4183 Imm32 value, const BaseIndex& mem, Register temp)
4184 DEFINED_ON(x86_shared);
4185
4186 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4187 Register value, const Address& mem,
4188 Register valueTemp, Register offsetTemp,
4189 Register maskTemp) DEFINED_ON(mips_shared, loong64);
4190
4191 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
4192 Register value, const BaseIndex& mem,
4193 Register valueTemp, Register offsetTemp,
4194 Register maskTemp) DEFINED_ON(mips_shared, loong64);
4195
4196 // 64-bit wide operations.
4197
4198 // 64-bit atomic load. On 64-bit systems, use regular wasm load with
4199 // Synchronization::Load, not this method.
4200 //
4201 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4202 // ARM: `temp` should be invalid; `output` must be (even,odd) pair.
4203 // MIPS32: `temp` should be invalid.
4204
4205 void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
4206 const Address& mem, Register64 temp, Register64 output)
4207 DEFINED_ON(arm, mips32, x86);
4208
4209 void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
4210 const BaseIndex& mem, Register64 temp,
4211 Register64 output) DEFINED_ON(arm, mips32, x86);
4212
4213 // x86: `expected` must be the same as `output`, and must be edx:eax.
4214 // x86: `replacement` must be ecx:ebx.
4215 // x64: `output` must be rax.
4216 // ARM: Registers must be distinct; `replacement` and `output` must be
4217 // (even,odd) pairs.
4218 // ARM64: The base register in `mem` must not overlap `output`.
4219 // MIPS: Registers must be distinct.
4220
4221 void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
4222 const Address& mem, Register64 expected,
4223 Register64 replacement,
4224 Register64 output) PER_ARCH;
4225
4226 void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
4227 const BaseIndex& mem, Register64 expected,
4228 Register64 replacement,
4229 Register64 output) PER_ARCH;
4230
4231 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
4232 // ARM: Registers must be distinct; `value` and `output` must be (even,odd)
4233 // pairs.
4234 // MIPS: Registers must be distinct.
4235
4236 void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
4237 const Address& mem, Register64 value,
4238 Register64 output) PER_ARCH;
4239
4240 void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
4241 const BaseIndex& mem, Register64 value,
4242 Register64 output) PER_ARCH;
4243
4244 // x86: `output` must be edx:eax, `temp` must be ecx:ebx.
4245 // x64: For And, Or, and Xor `output` must be rax.
4246 // ARM: Registers must be distinct; `temp` and `output` must be (even,odd)
4247 // pairs.
4248 // MIPS: Registers must be distinct.
4249 // MIPS32: `temp` should be invalid.
4250
4251 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4252 Register64 value, const Address& mem,
4253 Register64 temp, Register64 output)
4254 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64);
4255
4256 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4257 Register64 value, const BaseIndex& mem,
4258 Register64 temp, Register64 output)
4259 DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64);
4260
4261 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4262 const Address& value, const Address& mem,
4263 Register64 temp, Register64 output) DEFINED_ON(x86);
4264
4265 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4266 const Address& value, const BaseIndex& mem,
4267 Register64 temp, Register64 output) DEFINED_ON(x86);
4268
4269 // Here `value` can be any register.
4270
4271 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4272 Register64 value, const BaseIndex& mem)
4273 DEFINED_ON(x64);
4274
4275 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
4276 Register64 value, const BaseIndex& mem,
4277 Register64 temp) DEFINED_ON(arm64);
4278
4279 // ========================================================================
4280 // JS atomic operations.
4281 //
4282 // Here the arrayType must be a type that is valid for JS. As of 2017 that
4283 // is an 8-bit, 16-bit, or 32-bit integer type.
4284 //
4285 // If arrayType is Scalar::Uint32 then:
4286 //
4287 // - `output` must be a float register
4288 // - if the operation takes one temp register then `temp` must be defined
4289 // - if the operation takes two temp registers then `temp2` must be defined.
4290 //
4291 // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg.
4292 // (`temp1` must always be valid.)
4293 //
4294 // For additional register constraints, see the primitive 32-bit operations
4295 // and/or wasm operations above.
4296
4297 void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4298 const Address& mem, Register expected,
4299 Register replacement, Register temp,
4300 AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
4301
4302 void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4303 const BaseIndex& mem, Register expected,
4304 Register replacement, Register temp,
4305 AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
4306
4307 void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4308 const Address& mem, Register expected,
4309 Register replacement, Register valueTemp,
4310 Register offsetTemp, Register maskTemp, Register temp,
4311 AnyRegister output) DEFINED_ON(mips_shared, loong64);
4312
4313 void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4314 const BaseIndex& mem, Register expected,
4315 Register replacement, Register valueTemp,
4316 Register offsetTemp, Register maskTemp, Register temp,
4317 AnyRegister output) DEFINED_ON(mips_shared, loong64);
4318
4319 void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4320 const Address& mem, Register value, Register temp,
4321 AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
4322
4323 void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4324 const BaseIndex& mem, Register value, Register temp,
4325 AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
4326
4327 void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4328 const Address& mem, Register value, Register valueTemp,
4329 Register offsetTemp, Register maskTemp, Register temp,
4330 AnyRegister output) DEFINED_ON(mips_shared, loong64);
4331
4332 void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
4333 const BaseIndex& mem, Register value,
4334 Register valueTemp, Register offsetTemp,
4335 Register maskTemp, Register temp, AnyRegister output)
4336 DEFINED_ON(mips_shared, loong64);
4337
4338 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4339 AtomicOp op, Register value, const Address& mem,
4340 Register temp1, Register temp2, AnyRegister output)
4341 DEFINED_ON(arm, arm64, x86_shared);
4342
4343 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4344 AtomicOp op, Register value, const BaseIndex& mem,
4345 Register temp1, Register temp2, AnyRegister output)
4346 DEFINED_ON(arm, arm64, x86_shared);
4347
4348 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4349 AtomicOp op, Imm32 value, const Address& mem,
4350 Register temp1, Register temp2, AnyRegister output)
4351 DEFINED_ON(x86_shared);
4352
4353 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4354 AtomicOp op, Imm32 value, const BaseIndex& mem,
4355 Register temp1, Register temp2, AnyRegister output)
4356 DEFINED_ON(x86_shared);
4357
4358 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4359 AtomicOp op, Register value, const Address& mem,
4360 Register valueTemp, Register offsetTemp,
4361 Register maskTemp, Register temp, AnyRegister output)
4362 DEFINED_ON(mips_shared, loong64);
4363
4364 void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
4365 AtomicOp op, Register value, const BaseIndex& mem,
4366 Register valueTemp, Register offsetTemp,
4367 Register maskTemp, Register temp, AnyRegister output)
4368 DEFINED_ON(mips_shared, loong64);
4369
4370 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4371 AtomicOp op, Register value, const Address& mem,
4372 Register temp) DEFINED_ON(arm, arm64, x86_shared);
4373
4374 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4375 AtomicOp op, Register value, const BaseIndex& mem,
4376 Register temp) DEFINED_ON(arm, arm64, x86_shared);
4377
4378 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4379 AtomicOp op, Imm32 value, const Address& mem,
4380 Register temp) DEFINED_ON(x86_shared);
4381
4382 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4383 AtomicOp op, Imm32 value, const BaseIndex& mem,
4384 Register temp) DEFINED_ON(x86_shared);
4385
4386 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4387 AtomicOp op, Register value, const Address& mem,
4388 Register valueTemp, Register offsetTemp,
4389 Register maskTemp) DEFINED_ON(mips_shared, loong64);
4390
4391 void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
4392 AtomicOp op, Register value, const BaseIndex& mem,
4393 Register valueTemp, Register offsetTemp,
4394 Register maskTemp) DEFINED_ON(mips_shared, loong64);
4395
4396 void atomicIsLockFreeJS(Register value, Register output);
4397
4398 // ========================================================================
4399 // Spectre Mitigations.
4400 //
4401 // Spectre attacks are side-channel attacks based on cache pollution or
4402 // slow-execution of some instructions. We have multiple spectre mitigations
4403 // possible:
4404 //
4405 // - Stop speculative executions, with memory barriers. Memory barriers
4406 // force all branches depending on loads to be resolved, and thus
4407 // resolve all miss-speculated paths.
4408 //
4409 // - Use conditional move instructions. Some CPUs have a branch predictor,
4410 // and not a flag predictor. In such cases, using a conditional move
4411 // instruction to zero some pointer/index is enough to add a
4412 // data-dependency which prevents any futher executions until the load is
4413 // resolved.
4414
4415 void spectreMaskIndex32(Register index, Register length, Register output);
4416 void spectreMaskIndex32(Register index, const Address& length,
4417 Register output);
4418 void spectreMaskIndexPtr(Register index, Register length, Register output);
4419 void spectreMaskIndexPtr(Register index, const Address& length,
4420 Register output);
4421
4422 // The length must be a power of two. Performs a bounds check and Spectre
4423 // index masking.
4424 void boundsCheck32PowerOfTwo(Register index, uint32_t length, Label* failure);
4425
4426 void speculationBarrier() PER_SHARED_ARCH;
4427
4428 //}}} check_macroassembler_decl_style
4429 public:
4430 // Unsafe here means the caller is responsible for Spectre mitigations if
4431 // needed. Prefer branchTestObjClass or one of the other masm helpers!
4432 inline void loadObjClassUnsafe(Register obj, Register dest);
4433
4434 template <typename EmitPreBarrier>
4435 inline void storeObjShape(Register shape, Register obj,
4436 EmitPreBarrier emitPreBarrier);
4437 template <typename EmitPreBarrier>
4438 inline void storeObjShape(Shape* shape, Register obj,
4439 EmitPreBarrier emitPreBarrier);
4440
4441 inline void loadObjProto(Register obj, Register dest);
4442
4443 inline void loadStringLength(Register str, Register dest);
4444
4445 void loadStringChars(Register str, Register dest, CharEncoding encoding);
4446
4447 void loadNonInlineStringChars(Register str, Register dest,
4448 CharEncoding encoding);
4449 void loadNonInlineStringCharsForStore(Register str, Register dest);
4450 void storeNonInlineStringChars(Register chars, Register str);
4451
4452 void loadInlineStringChars(Register str, Register dest,
4453 CharEncoding encoding);
4454 void loadInlineStringCharsForStore(Register str, Register dest);
4455
4456 void loadStringChar(Register str, Register index, Register output,
4457 Register scratch, Label* fail);
4458
4459 void loadRopeLeftChild(Register str, Register dest);
4460 void storeRopeChildren(Register left, Register right, Register str);
4461
4462 void loadDependentStringBase(Register str, Register dest);
4463 void storeDependentStringBase(Register base, Register str);
4464
4465 void loadStringIndexValue(Register str, Register dest, Label* fail);
4466
4467 /**
4468 * Store the character in |src| to |dest|.
4469 */
4470 template <typename T>
storeChar(const T & src,Address dest,CharEncoding encoding)4471 void storeChar(const T& src, Address dest, CharEncoding encoding) {
4472 if (encoding == CharEncoding::Latin1) {
4473 store8(src, dest);
4474 } else {
4475 store16(src, dest);
4476 }
4477 }
4478
4479 /**
4480 * Load the character at |src| into |dest|.
4481 */
4482 template <typename T>
loadChar(const T & src,Register dest,CharEncoding encoding)4483 void loadChar(const T& src, Register dest, CharEncoding encoding) {
4484 if (encoding == CharEncoding::Latin1) {
4485 load8ZeroExtend(src, dest);
4486 } else {
4487 load16ZeroExtend(src, dest);
4488 }
4489 }
4490
4491 /**
4492 * Load the character at |chars[index + offset]| into |dest|. The optional
4493 * offset argument is not scaled to the character encoding.
4494 */
4495 void loadChar(Register chars, Register index, Register dest,
4496 CharEncoding encoding, int32_t offset = 0);
4497
4498 /**
4499 * Add |index| to |chars| so that |chars| now points at |chars[index]|.
4500 */
4501 void addToCharPtr(Register chars, Register index, CharEncoding encoding);
4502
4503 /**
4504 * Load the BigInt digits from |bigInt| into |digits|.
4505 */
4506 void loadBigIntDigits(Register bigInt, Register digits);
4507
4508 /**
4509 * Load the first [u]int64 value from |bigInt| into |dest|.
4510 */
4511 void loadBigInt64(Register bigInt, Register64 dest);
4512
4513 /**
4514 * Load the first digit from |bigInt| into |dest|. Handles the case when the
4515 * BigInt digits length is zero.
4516 *
4517 * Note: A BigInt digit is a pointer-sized value.
4518 */
4519 void loadFirstBigIntDigitOrZero(Register bigInt, Register dest);
4520
4521 /**
4522 * Load the number stored in |bigInt| into |dest|. Handles the case when the
4523 * BigInt digits length is zero. Jumps to |fail| when the number can't be
4524 * saved into a single pointer-sized register.
4525 */
4526 void loadBigInt(Register bigInt, Register dest, Label* fail);
4527
4528 /**
4529 * Load the number stored in |bigInt| into |dest|. Doesn't handle the case
4530 * when the BigInt digits length is zero. Jumps to |fail| when the number
4531 * can't be saved into a single pointer-sized register.
4532 */
4533 void loadBigIntNonZero(Register bigInt, Register dest, Label* fail);
4534
4535 /**
4536 * Load the absolute number stored in |bigInt| into |dest|. Handles the case
4537 * when the BigInt digits length is zero. Jumps to |fail| when the number
4538 * can't be saved into a single pointer-sized register.
4539 */
4540 void loadBigIntAbsolute(Register bigInt, Register dest, Label* fail);
4541
4542 /**
4543 * In-place modifies the BigInt digit to a signed pointer-sized value. Jumps
4544 * to |fail| when the digit exceeds the representable range.
4545 */
4546 void bigIntDigitToSignedPtr(Register bigInt, Register digit, Label* fail);
4547
4548 /**
4549 * Initialize a BigInt from |val|. Clobbers |val|!
4550 */
4551 void initializeBigInt64(Scalar::Type type, Register bigInt, Register64 val);
4552
4553 /**
4554 * Initialize a BigInt from the signed, pointer-sized register |val|.
4555 * Clobbers |val|!
4556 */
4557 void initializeBigInt(Register bigInt, Register val);
4558
4559 /**
4560 * Initialize a BigInt from the pointer-sized register |val|.
4561 */
4562 void initializeBigIntAbsolute(Register bigInt, Register val);
4563
4564 /**
4565 * Copy a BigInt. Jumps to |fail| on allocation failure or when the BigInt
4566 * digits need to be heap allocated.
4567 */
4568 void copyBigIntWithInlineDigits(Register src, Register dest, Register temp,
4569 gc::InitialHeap initialHeap, Label* fail);
4570
4571 /**
4572 * Compare a BigInt and an Int32 value. Falls through to the false case.
4573 */
4574 void compareBigIntAndInt32(JSOp op, Register bigInt, Register int32,
4575 Register scratch1, Register scratch2,
4576 Label* ifTrue, Label* ifFalse);
4577
4578 /**
4579 * Compare two BigInts for equality. Falls through if both BigInts are equal
4580 * to each other.
4581 *
4582 * - When we jump to |notSameLength|, |temp1| holds the length of the right
4583 * operand.
4584 * - When we jump to |notSameDigit|, |temp2| points to the current digit of
4585 * the left operand and |temp4| holds the current digit of the right
4586 * operand.
4587 */
4588 void equalBigInts(Register left, Register right, Register temp1,
4589 Register temp2, Register temp3, Register temp4,
4590 Label* notSameSign, Label* notSameLength,
4591 Label* notSameDigit);
4592
4593 void loadJSContext(Register dest);
4594
4595 void switchToRealm(Register realm);
4596 void switchToRealm(const void* realm, Register scratch);
4597 void switchToObjectRealm(Register obj, Register scratch);
4598 void switchToBaselineFrameRealm(Register scratch);
4599 void switchToWasmTlsRealm(Register scratch1, Register scratch2);
4600 void debugAssertContextRealm(const void* realm, Register scratch);
4601
4602 void loadJitActivation(Register dest);
4603
4604 void guardSpecificAtom(Register str, JSAtom* atom, Register scratch,
4605 const LiveRegisterSet& volatileRegs, Label* fail);
4606
4607 void guardStringToInt32(Register str, Register output, Register scratch,
4608 LiveRegisterSet volatileRegs, Label* fail);
4609
4610 template <typename T>
loadTypedOrValue(const T & src,TypedOrValueRegister dest)4611 void loadTypedOrValue(const T& src, TypedOrValueRegister dest) {
4612 if (dest.hasValue()) {
4613 loadValue(src, dest.valueReg());
4614 } else {
4615 loadUnboxedValue(src, dest.type(), dest.typedReg());
4616 }
4617 }
4618
4619 template <typename T>
storeTypedOrValue(TypedOrValueRegister src,const T & dest)4620 void storeTypedOrValue(TypedOrValueRegister src, const T& dest) {
4621 if (src.hasValue()) {
4622 storeValue(src.valueReg(), dest);
4623 } else if (IsFloatingPointType(src.type())) {
4624 FloatRegister reg = src.typedReg().fpu();
4625 if (src.type() == MIRType::Float32) {
4626 ScratchDoubleScope fpscratch(*this);
4627 convertFloat32ToDouble(reg, fpscratch);
4628 boxDouble(fpscratch, dest);
4629 } else {
4630 boxDouble(reg, dest);
4631 }
4632 } else {
4633 storeValue(ValueTypeFromMIRType(src.type()), src.typedReg().gpr(), dest);
4634 }
4635 }
4636
4637 template <typename T>
storeConstantOrRegister(const ConstantOrRegister & src,const T & dest)4638 void storeConstantOrRegister(const ConstantOrRegister& src, const T& dest) {
4639 if (src.constant()) {
4640 storeValue(src.value(), dest);
4641 } else {
4642 storeTypedOrValue(src.reg(), dest);
4643 }
4644 }
4645
storeCallPointerResult(Register reg)4646 void storeCallPointerResult(Register reg) {
4647 if (reg != ReturnReg) {
4648 mov(ReturnReg, reg);
4649 }
4650 }
4651
4652 inline void storeCallBoolResult(Register reg);
4653 inline void storeCallInt32Result(Register reg);
4654
storeCallFloatResult(FloatRegister reg)4655 void storeCallFloatResult(FloatRegister reg) {
4656 if (reg != ReturnDoubleReg) {
4657 moveDouble(ReturnDoubleReg, reg);
4658 }
4659 }
4660
4661 inline void storeCallResultValue(AnyRegister dest, JSValueType type);
4662
storeCallResultValue(ValueOperand dest)4663 void storeCallResultValue(ValueOperand dest) {
4664 #if defined(JS_NUNBOX32)
4665 // reshuffle the return registers used for a call result to store into
4666 // dest, using ReturnReg as a scratch register if necessary. This must
4667 // only be called after returning from a call, at a point when the
4668 // return register is not live. XXX would be better to allow wrappers
4669 // to store the return value to different places.
4670 if (dest.typeReg() == JSReturnReg_Data) {
4671 if (dest.payloadReg() == JSReturnReg_Type) {
4672 // swap the two registers.
4673 mov(JSReturnReg_Type, ReturnReg);
4674 mov(JSReturnReg_Data, JSReturnReg_Type);
4675 mov(ReturnReg, JSReturnReg_Data);
4676 } else {
4677 mov(JSReturnReg_Data, dest.payloadReg());
4678 mov(JSReturnReg_Type, dest.typeReg());
4679 }
4680 } else {
4681 mov(JSReturnReg_Type, dest.typeReg());
4682 mov(JSReturnReg_Data, dest.payloadReg());
4683 }
4684 #elif defined(JS_PUNBOX64)
4685 if (dest.valueReg() != JSReturnReg) {
4686 mov(JSReturnReg, dest.valueReg());
4687 }
4688 #else
4689 # error "Bad architecture"
4690 #endif
4691 }
4692
4693 inline void storeCallResultValue(TypedOrValueRegister dest);
4694
4695 private:
4696 TrampolinePtr preBarrierTrampoline(MIRType type);
4697
4698 template <typename T>
unguardedCallPreBarrier(const T & address,MIRType type)4699 void unguardedCallPreBarrier(const T& address, MIRType type) {
4700 Label done;
4701 if (type == MIRType::Value) {
4702 branchTestGCThing(Assembler::NotEqual, address, &done);
4703 } else if (type == MIRType::Object || type == MIRType::String) {
4704 branchPtr(Assembler::Equal, address, ImmWord(0), &done);
4705 }
4706
4707 Push(PreBarrierReg);
4708 computeEffectiveAddress(address, PreBarrierReg);
4709
4710 TrampolinePtr preBarrier = preBarrierTrampoline(type);
4711
4712 call(preBarrier);
4713 Pop(PreBarrierReg);
4714 // On arm64, SP may be < PSP now (that's OK).
4715 // eg testcase: tests/auto-regress/bug702915.js
4716 bind(&done);
4717 }
4718
4719 public:
4720 template <typename T>
guardedCallPreBarrier(const T & address,MIRType type)4721 void guardedCallPreBarrier(const T& address, MIRType type) {
4722 Label done;
4723 branchTestNeedsIncrementalBarrier(Assembler::Zero, &done);
4724 unguardedCallPreBarrier(address, type);
4725 bind(&done);
4726 }
4727
4728 // Like guardedCallPreBarrier, but unlike guardedCallPreBarrier this can be
4729 // called from runtime-wide trampolines because it loads cx->zone (instead of
4730 // baking in the current Zone) if JitContext::realm is nullptr.
4731 template <typename T>
guardedCallPreBarrierAnyZone(const T & address,MIRType type,Register scratch)4732 void guardedCallPreBarrierAnyZone(const T& address, MIRType type,
4733 Register scratch) {
4734 Label done;
4735 branchTestNeedsIncrementalBarrierAnyZone(Assembler::Zero, &done, scratch);
4736 unguardedCallPreBarrier(address, type);
4737 bind(&done);
4738 }
4739
4740 enum class Uint32Mode { FailOnDouble, ForceDouble };
4741
4742 void boxUint32(Register source, ValueOperand dest, Uint32Mode uint32Mode,
4743 Label* fail);
4744
4745 template <typename T>
4746 void loadFromTypedArray(Scalar::Type arrayType, const T& src,
4747 AnyRegister dest, Register temp, Label* fail);
4748
4749 template <typename T>
4750 void loadFromTypedArray(Scalar::Type arrayType, const T& src,
4751 const ValueOperand& dest, Uint32Mode uint32Mode,
4752 Register temp, Label* fail);
4753
4754 template <typename T>
4755 void loadFromTypedBigIntArray(Scalar::Type arrayType, const T& src,
4756 Register bigInt, Register64 temp);
4757
4758 template <typename S, typename T>
storeToTypedIntArray(Scalar::Type arrayType,const S & value,const T & dest)4759 void storeToTypedIntArray(Scalar::Type arrayType, const S& value,
4760 const T& dest) {
4761 switch (arrayType) {
4762 case Scalar::Int8:
4763 case Scalar::Uint8:
4764 case Scalar::Uint8Clamped:
4765 store8(value, dest);
4766 break;
4767 case Scalar::Int16:
4768 case Scalar::Uint16:
4769 store16(value, dest);
4770 break;
4771 case Scalar::Int32:
4772 case Scalar::Uint32:
4773 store32(value, dest);
4774 break;
4775 default:
4776 MOZ_CRASH("Invalid typed array type");
4777 }
4778 }
4779
4780 void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value,
4781 const BaseIndex& dest);
4782 void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value,
4783 const Address& dest);
4784
4785 void storeToTypedBigIntArray(Scalar::Type arrayType, Register64 value,
4786 const BaseIndex& dest);
4787 void storeToTypedBigIntArray(Scalar::Type arrayType, Register64 value,
4788 const Address& dest);
4789
4790 void memoryBarrierBefore(const Synchronization& sync);
4791 void memoryBarrierAfter(const Synchronization& sync);
4792
4793 void debugAssertIsObject(const ValueOperand& val);
4794 void debugAssertObjHasFixedSlots(Register obj, Register scratch);
4795
4796 void debugAssertObjectHasClass(Register obj, Register scratch,
4797 const JSClass* clasp);
4798
4799 void branchArrayIsNotPacked(Register array, Register temp1, Register temp2,
4800 Label* label);
4801
4802 void setIsPackedArray(Register obj, Register output, Register temp);
4803
4804 void packedArrayPop(Register array, ValueOperand output, Register temp1,
4805 Register temp2, Label* fail);
4806 void packedArrayShift(Register array, ValueOperand output, Register temp1,
4807 Register temp2, LiveRegisterSet volatileRegs,
4808 Label* fail);
4809
4810 void loadArgumentsObjectElement(Register obj, Register index,
4811 ValueOperand output, Register temp,
4812 Label* fail);
4813 void loadArgumentsObjectElementHole(Register obj, Register index,
4814 ValueOperand output, Register temp,
4815 Label* fail);
4816 void loadArgumentsObjectElementExists(Register obj, Register index,
4817 Register output, Register temp,
4818 Label* fail);
4819
4820 void loadArgumentsObjectLength(Register obj, Register output, Label* fail);
4821
4822 void branchTestArgumentsObjectFlags(Register obj, Register temp,
4823 uint32_t flags, Condition cond,
4824 Label* label);
4825
4826 void typedArrayElementSize(Register obj, Register output);
4827 void branchIfClassIsNotTypedArray(Register clasp, Label* notTypedArray);
4828
4829 void branchIfHasDetachedArrayBuffer(Register obj, Register temp,
4830 Label* label);
4831
4832 void branchIfNativeIteratorNotReusable(Register ni, Label* notReusable);
4833
4834 void iteratorMore(Register obj, ValueOperand output, Register temp);
4835 void iteratorClose(Register obj, Register temp1, Register temp2,
4836 Register temp3);
4837
4838 void toHashableNonGCThing(ValueOperand value, ValueOperand result,
4839 FloatRegister tempFloat);
4840
4841 void toHashableValue(ValueOperand value, ValueOperand result,
4842 FloatRegister tempFloat, Label* atomizeString,
4843 Label* tagString);
4844
4845 private:
4846 void scrambleHashCode(Register result);
4847
4848 public:
4849 void prepareHashNonGCThing(ValueOperand value, Register result,
4850 Register temp);
4851 void prepareHashString(Register str, Register result, Register temp);
4852 void prepareHashSymbol(Register sym, Register result);
4853 void prepareHashBigInt(Register bigInt, Register result, Register temp1,
4854 Register temp2, Register temp3);
4855 void prepareHashObject(Register setObj, ValueOperand value, Register result,
4856 Register temp1, Register temp2, Register temp3,
4857 Register temp4);
4858 void prepareHashValue(Register setObj, ValueOperand value, Register result,
4859 Register temp1, Register temp2, Register temp3,
4860 Register temp4);
4861
4862 private:
4863 enum class IsBigInt { No, Yes, Maybe };
4864
4865 /**
4866 * Search for a value in a OrderedHashTable.
4867 *
4868 * When we jump to |found|, |entryTemp| holds the found hashtable entry.
4869 */
4870 template <typename OrderedHashTable>
4871 void orderedHashTableLookup(Register setOrMapObj, ValueOperand value,
4872 Register hash, Register entryTemp, Register temp1,
4873 Register temp3, Register temp4, Register temp5,
4874 Label* found, IsBigInt isBigInt);
4875
4876 void setObjectHas(Register setObj, ValueOperand value, Register hash,
4877 Register result, Register temp1, Register temp2,
4878 Register temp3, Register temp4, IsBigInt isBigInt);
4879
4880 void mapObjectHas(Register mapObj, ValueOperand value, Register hash,
4881 Register result, Register temp1, Register temp2,
4882 Register temp3, Register temp4, IsBigInt isBigInt);
4883
4884 void mapObjectGet(Register mapObj, ValueOperand value, Register hash,
4885 ValueOperand result, Register temp1, Register temp2,
4886 Register temp3, Register temp4, Register temp5,
4887 IsBigInt isBigInt);
4888
4889 public:
setObjectHasNonBigInt(Register setObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2)4890 void setObjectHasNonBigInt(Register setObj, ValueOperand value, Register hash,
4891 Register result, Register temp1, Register temp2) {
4892 return setObjectHas(setObj, value, hash, result, temp1, temp2, InvalidReg,
4893 InvalidReg, IsBigInt::No);
4894 }
setObjectHasBigInt(Register setObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2,Register temp3,Register temp4)4895 void setObjectHasBigInt(Register setObj, ValueOperand value, Register hash,
4896 Register result, Register temp1, Register temp2,
4897 Register temp3, Register temp4) {
4898 return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
4899 IsBigInt::Yes);
4900 }
setObjectHasValue(Register setObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2,Register temp3,Register temp4)4901 void setObjectHasValue(Register setObj, ValueOperand value, Register hash,
4902 Register result, Register temp1, Register temp2,
4903 Register temp3, Register temp4) {
4904 return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
4905 IsBigInt::Maybe);
4906 }
4907
mapObjectHasNonBigInt(Register mapObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2)4908 void mapObjectHasNonBigInt(Register mapObj, ValueOperand value, Register hash,
4909 Register result, Register temp1, Register temp2) {
4910 return mapObjectHas(mapObj, value, hash, result, temp1, temp2, InvalidReg,
4911 InvalidReg, IsBigInt::No);
4912 }
mapObjectHasBigInt(Register mapObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2,Register temp3,Register temp4)4913 void mapObjectHasBigInt(Register mapObj, ValueOperand value, Register hash,
4914 Register result, Register temp1, Register temp2,
4915 Register temp3, Register temp4) {
4916 return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
4917 IsBigInt::Yes);
4918 }
mapObjectHasValue(Register mapObj,ValueOperand value,Register hash,Register result,Register temp1,Register temp2,Register temp3,Register temp4)4919 void mapObjectHasValue(Register mapObj, ValueOperand value, Register hash,
4920 Register result, Register temp1, Register temp2,
4921 Register temp3, Register temp4) {
4922 return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
4923 IsBigInt::Maybe);
4924 }
4925
mapObjectGetNonBigInt(Register mapObj,ValueOperand value,Register hash,ValueOperand result,Register temp1,Register temp2,Register temp3)4926 void mapObjectGetNonBigInt(Register mapObj, ValueOperand value, Register hash,
4927 ValueOperand result, Register temp1,
4928 Register temp2, Register temp3) {
4929 return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3,
4930 InvalidReg, InvalidReg, IsBigInt::No);
4931 }
mapObjectGetBigInt(Register mapObj,ValueOperand value,Register hash,ValueOperand result,Register temp1,Register temp2,Register temp3,Register temp4,Register temp5)4932 void mapObjectGetBigInt(Register mapObj, ValueOperand value, Register hash,
4933 ValueOperand result, Register temp1, Register temp2,
4934 Register temp3, Register temp4, Register temp5) {
4935 return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
4936 temp5, IsBigInt::Yes);
4937 }
mapObjectGetValue(Register mapObj,ValueOperand value,Register hash,ValueOperand result,Register temp1,Register temp2,Register temp3,Register temp4,Register temp5)4938 void mapObjectGetValue(Register mapObj, ValueOperand value, Register hash,
4939 ValueOperand result, Register temp1, Register temp2,
4940 Register temp3, Register temp4, Register temp5) {
4941 return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
4942 temp5, IsBigInt::Maybe);
4943 }
4944
4945 // Inline version of js_TypedArray_uint8_clamp_double.
4946 // This function clobbers the input register.
4947 void clampDoubleToUint8(FloatRegister input, Register output) PER_ARCH;
4948
4949 using MacroAssemblerSpecific::ensureDouble;
4950
4951 template <typename S>
ensureDouble(const S & source,FloatRegister dest,Label * failure)4952 void ensureDouble(const S& source, FloatRegister dest, Label* failure) {
4953 Label isDouble, done;
4954 branchTestDouble(Assembler::Equal, source, &isDouble);
4955 branchTestInt32(Assembler::NotEqual, source, failure);
4956
4957 convertInt32ToDouble(source, dest);
4958 jump(&done);
4959
4960 bind(&isDouble);
4961 unboxDouble(source, dest);
4962
4963 bind(&done);
4964 }
4965
4966 // Inline allocation.
4967 private:
4968 void checkAllocatorState(Label* fail);
4969 bool shouldNurseryAllocate(gc::AllocKind allocKind,
4970 gc::InitialHeap initialHeap);
4971 void nurseryAllocateObject(
4972 Register result, Register temp, gc::AllocKind allocKind,
4973 size_t nDynamicSlots, Label* fail,
4974 const AllocSiteInput& allocSite = AllocSiteInput());
4975 void bumpPointerAllocate(Register result, Register temp, Label* fail,
4976 CompileZone* zone, void* posAddr,
4977 const void* curEddAddr, JS::TraceKind traceKind,
4978 uint32_t size,
4979 const AllocSiteInput& allocSite = AllocSiteInput());
4980 void updateAllocSite(Register temp, Register result, CompileZone* zone,
4981 Register site);
4982
4983 void freeListAllocate(Register result, Register temp, gc::AllocKind allocKind,
4984 Label* fail);
4985 void allocateObject(Register result, Register temp, gc::AllocKind allocKind,
4986 uint32_t nDynamicSlots, gc::InitialHeap initialHeap,
4987 Label* fail,
4988 const AllocSiteInput& allocSite = AllocSiteInput());
4989 void nurseryAllocateString(Register result, Register temp,
4990 gc::AllocKind allocKind, Label* fail);
4991 void allocateString(Register result, Register temp, gc::AllocKind allocKind,
4992 gc::InitialHeap initialHeap, Label* fail);
4993 void nurseryAllocateBigInt(Register result, Register temp, Label* fail);
4994 void copySlotsFromTemplate(Register obj,
4995 const TemplateNativeObject& templateObj,
4996 uint32_t start, uint32_t end);
4997 void fillSlotsWithConstantValue(Address addr, Register temp, uint32_t start,
4998 uint32_t end, const Value& v);
4999 void fillSlotsWithUndefined(Address addr, Register temp, uint32_t start,
5000 uint32_t end);
5001 void fillSlotsWithUninitialized(Address addr, Register temp, uint32_t start,
5002 uint32_t end);
5003
5004 void initGCSlots(Register obj, Register temp,
5005 const TemplateNativeObject& templateObj, bool initContents);
5006
5007 public:
5008 void callFreeStub(Register slots);
5009 void createGCObject(Register result, Register temp,
5010 const TemplateObject& templateObj,
5011 gc::InitialHeap initialHeap, Label* fail,
5012 bool initContents = true);
5013
5014 void createPlainGCObject(Register result, Register shape, Register temp,
5015 Register temp2, uint32_t numFixedSlots,
5016 uint32_t numDynamicSlots, gc::AllocKind allocKind,
5017 gc::InitialHeap initialHeap, Label* fail,
5018 const AllocSiteInput& allocSite = AllocSiteInput());
5019
5020 void createArrayWithFixedElements(
5021 Register result, Register shape, Register temp, uint32_t arrayLength,
5022 uint32_t arrayCapacity, gc::AllocKind allocKind,
5023 gc::InitialHeap initialHeap, Label* fail,
5024 const AllocSiteInput& allocSite = AllocSiteInput());
5025
5026 void initGCThing(Register obj, Register temp,
5027 const TemplateObject& templateObj, bool initContents = true);
5028
5029 enum class TypedArrayLength { Fixed, Dynamic };
5030
5031 void initTypedArraySlots(Register obj, Register temp, Register lengthReg,
5032 LiveRegisterSet liveRegs, Label* fail,
5033 TypedArrayObject* templateObj,
5034 TypedArrayLength lengthKind);
5035
5036 void newGCString(Register result, Register temp, gc::InitialHeap initialHeap,
5037 Label* fail);
5038 void newGCFatInlineString(Register result, Register temp,
5039 gc::InitialHeap initialHeap, Label* fail);
5040
5041 void newGCBigInt(Register result, Register temp, gc::InitialHeap initialHeap,
5042 Label* fail);
5043
5044 // Compares two strings for equality based on the JSOP.
5045 // This checks for identical pointers, atoms and length and fails for
5046 // everything else.
5047 void compareStrings(JSOp op, Register left, Register right, Register result,
5048 Label* fail);
5049
5050 // Result of the typeof operation. Falls back to slow-path for proxies.
5051 void typeOfObject(Register objReg, Register scratch, Label* slow,
5052 Label* isObject, Label* isCallable, Label* isUndefined);
5053
5054 // Implementation of IsCallable. Doesn't handle proxies.
isCallable(Register obj,Register output,Label * isProxy)5055 void isCallable(Register obj, Register output, Label* isProxy) {
5056 isCallableOrConstructor(true, obj, output, isProxy);
5057 }
isConstructor(Register obj,Register output,Label * isProxy)5058 void isConstructor(Register obj, Register output, Label* isProxy) {
5059 isCallableOrConstructor(false, obj, output, isProxy);
5060 }
5061
5062 void setIsCrossRealmArrayConstructor(Register obj, Register output);
5063
5064 void setIsDefinitelyTypedArrayConstructor(Register obj, Register output);
5065
5066 void loadDOMExpandoValueGuardGeneration(
5067 Register obj, ValueOperand output,
5068 JS::ExpandoAndGeneration* expandoAndGeneration, uint64_t generation,
5069 Label* fail);
5070
5071 void guardNonNegativeIntPtrToInt32(Register reg, Label* fail);
5072
5073 void loadArrayBufferByteLengthIntPtr(Register obj, Register output);
5074 void loadArrayBufferViewByteOffsetIntPtr(Register obj, Register output);
5075 void loadArrayBufferViewLengthIntPtr(Register obj, Register output);
5076
5077 private:
5078 void isCallableOrConstructor(bool isCallable, Register obj, Register output,
5079 Label* isProxy);
5080
5081 public:
5082 // Generates code used to complete a bailout.
5083 void generateBailoutTail(Register scratch, Register bailoutInfo);
5084
5085 void assertRectifierFrameParentType(Register frameType);
5086
5087 public:
5088 #ifndef JS_CODEGEN_ARM64
5089 // StackPointer manipulation functions.
5090 // On ARM64, the StackPointer is implemented as two synchronized registers.
5091 // Code shared across platforms must use these functions to be valid.
5092 template <typename T>
5093 inline void addToStackPtr(T t);
5094 template <typename T>
5095 inline void addStackPtrTo(T t);
5096
5097 void subFromStackPtr(Imm32 imm32)
5098 DEFINED_ON(mips32, mips64, loong64, arm, x86, x64);
5099 void subFromStackPtr(Register reg);
5100
5101 template <typename T>
subStackPtrFrom(T t)5102 void subStackPtrFrom(T t) {
5103 subPtr(getStackPointer(), t);
5104 }
5105
5106 template <typename T>
andToStackPtr(T t)5107 void andToStackPtr(T t) {
5108 andPtr(t, getStackPointer());
5109 }
5110
5111 template <typename T>
moveToStackPtr(T t)5112 void moveToStackPtr(T t) {
5113 movePtr(t, getStackPointer());
5114 }
5115 template <typename T>
moveStackPtrTo(T t)5116 void moveStackPtrTo(T t) {
5117 movePtr(getStackPointer(), t);
5118 }
5119
5120 template <typename T>
loadStackPtr(T t)5121 void loadStackPtr(T t) {
5122 loadPtr(t, getStackPointer());
5123 }
5124 template <typename T>
storeStackPtr(T t)5125 void storeStackPtr(T t) {
5126 storePtr(getStackPointer(), t);
5127 }
5128
5129 // StackPointer testing functions.
5130 // On ARM64, sp can function as the zero register depending on context.
5131 // Code shared across platforms must use these functions to be valid.
5132 template <typename T>
5133 inline void branchTestStackPtr(Condition cond, T t, Label* label);
5134 template <typename T>
5135 inline void branchStackPtr(Condition cond, T rhs, Label* label);
5136 template <typename T>
5137 inline void branchStackPtrRhs(Condition cond, T lhs, Label* label);
5138
5139 // Move the stack pointer based on the requested amount.
5140 inline void reserveStack(uint32_t amount);
5141 #else // !JS_CODEGEN_ARM64
5142 void reserveStack(uint32_t amount);
5143 #endif
5144
5145 public:
enableProfilingInstrumentation()5146 void enableProfilingInstrumentation() {
5147 emitProfilingInstrumentation_ = true;
5148 }
5149
5150 private:
5151 // This class is used to surround call sites throughout the assembler. This
5152 // is used by callWithABI, and callJit functions, except if suffixed by
5153 // NoProfiler.
5154 class MOZ_RAII AutoProfilerCallInstrumentation {
5155 public:
5156 explicit AutoProfilerCallInstrumentation(MacroAssembler& masm);
5157 ~AutoProfilerCallInstrumentation() = default;
5158 };
5159 friend class AutoProfilerCallInstrumentation;
5160
appendProfilerCallSite(CodeOffset label)5161 void appendProfilerCallSite(CodeOffset label) {
5162 propagateOOM(profilerCallSites_.append(label));
5163 }
5164
5165 // Fix up the code pointers to be written for locations where profilerCallSite
5166 // emitted moves of RIP to a register.
5167 void linkProfilerCallSites(JitCode* code);
5168
5169 // This field is used to manage profiling instrumentation output. If
5170 // provided and enabled, then instrumentation will be emitted around call
5171 // sites.
5172 bool emitProfilingInstrumentation_;
5173
5174 // Record locations of the call sites.
5175 Vector<CodeOffset, 0, SystemAllocPolicy> profilerCallSites_;
5176
5177 public:
5178 void loadJitCodeRaw(Register func, Register dest);
5179 void loadBaselineJitCodeRaw(Register func, Register dest,
5180 Label* failure = nullptr);
5181 void storeICScriptInJSContext(Register icScript);
5182
5183 void loadBaselineFramePtr(Register framePtr, Register dest);
5184
pushBaselineFramePtr(Register framePtr,Register scratch)5185 void pushBaselineFramePtr(Register framePtr, Register scratch) {
5186 loadBaselineFramePtr(framePtr, scratch);
5187 push(scratch);
5188 }
5189
PushBaselineFramePtr(Register framePtr,Register scratch)5190 void PushBaselineFramePtr(Register framePtr, Register scratch) {
5191 loadBaselineFramePtr(framePtr, scratch);
5192 Push(scratch);
5193 }
5194
5195 using MacroAssemblerSpecific::movePtr;
5196
movePtr(TrampolinePtr ptr,Register dest)5197 void movePtr(TrampolinePtr ptr, Register dest) {
5198 movePtr(ImmPtr(ptr.value), dest);
5199 }
5200
5201 private:
5202 void handleFailure();
5203
5204 public:
exceptionLabel()5205 Label* exceptionLabel() {
5206 // Exceptions are currently handled the same way as sequential failures.
5207 return &failureLabel_;
5208 }
5209
failureLabel()5210 Label* failureLabel() { return &failureLabel_; }
5211
5212 void finish();
5213 void link(JitCode* code);
5214
5215 void assumeUnreachable(const char* output);
5216
5217 void printf(const char* output);
5218 void printf(const char* output, Register value);
5219
5220 #ifdef JS_TRACE_LOGGING
5221 void loadTraceLogger(Register logger);
5222 void tracelogStartId(Register logger, uint32_t textId, bool force = false);
5223 void tracelogStartId(Register logger, Register textId);
5224 void tracelogStartEvent(Register logger, Register event);
5225 void tracelogStopId(Register logger, uint32_t textId, bool force = false);
5226 void tracelogStopId(Register logger, Register textId);
5227 #endif
5228
5229 #define DISPATCH_FLOATING_POINT_OP(method, type, arg1d, arg1f, arg2) \
5230 MOZ_ASSERT(IsFloatingPointType(type)); \
5231 if (type == MIRType::Double) \
5232 method##Double(arg1d, arg2); \
5233 else \
5234 method##Float32(arg1f, arg2);
5235
loadConstantFloatingPoint(double d,float f,FloatRegister dest,MIRType destType)5236 void loadConstantFloatingPoint(double d, float f, FloatRegister dest,
5237 MIRType destType) {
5238 DISPATCH_FLOATING_POINT_OP(loadConstant, destType, d, f, dest);
5239 }
boolValueToFloatingPoint(ValueOperand value,FloatRegister dest,MIRType destType)5240 void boolValueToFloatingPoint(ValueOperand value, FloatRegister dest,
5241 MIRType destType) {
5242 DISPATCH_FLOATING_POINT_OP(boolValueTo, destType, value, value, dest);
5243 }
int32ValueToFloatingPoint(ValueOperand value,FloatRegister dest,MIRType destType)5244 void int32ValueToFloatingPoint(ValueOperand value, FloatRegister dest,
5245 MIRType destType) {
5246 DISPATCH_FLOATING_POINT_OP(int32ValueTo, destType, value, value, dest);
5247 }
convertInt32ToFloatingPoint(Register src,FloatRegister dest,MIRType destType)5248 void convertInt32ToFloatingPoint(Register src, FloatRegister dest,
5249 MIRType destType) {
5250 DISPATCH_FLOATING_POINT_OP(convertInt32To, destType, src, src, dest);
5251 }
5252
5253 #undef DISPATCH_FLOATING_POINT_OP
5254
5255 void convertValueToFloatingPoint(ValueOperand value, FloatRegister output,
5256 Label* fail, MIRType outputType);
5257
5258 void outOfLineTruncateSlow(FloatRegister src, Register dest,
5259 bool widenFloatToDouble, bool compilingWasm,
5260 wasm::BytecodeOffset callOffset);
5261
5262 void convertInt32ValueToDouble(ValueOperand val);
5263
convertValueToDouble(ValueOperand value,FloatRegister output,Label * fail)5264 void convertValueToDouble(ValueOperand value, FloatRegister output,
5265 Label* fail) {
5266 convertValueToFloatingPoint(value, output, fail, MIRType::Double);
5267 }
5268
convertValueToFloat(ValueOperand value,FloatRegister output,Label * fail)5269 void convertValueToFloat(ValueOperand value, FloatRegister output,
5270 Label* fail) {
5271 convertValueToFloatingPoint(value, output, fail, MIRType::Float32);
5272 }
5273
5274 //
5275 // Functions for converting values to int.
5276 //
5277 void convertDoubleToInt(FloatRegister src, Register output,
5278 FloatRegister temp, Label* truncateFail, Label* fail,
5279 IntConversionBehavior behavior);
5280
5281 // Strings may be handled by providing labels to jump to when the behavior
5282 // is truncation or clamping. The subroutine, usually an OOL call, is
5283 // passed the unboxed string in |stringReg| and should convert it to a
5284 // double store into |temp|.
5285 void convertValueToInt(
5286 ValueOperand value, Label* handleStringEntry, Label* handleStringRejoin,
5287 Label* truncateDoubleSlow, Register stringReg, FloatRegister temp,
5288 Register output, Label* fail, IntConversionBehavior behavior,
5289 IntConversionInputKind conversion = IntConversionInputKind::Any);
5290
5291 // This carries over the MToNumberInt32 operation on the ValueOperand
5292 // input; see comment at the top of this class.
5293 void convertValueToInt32(
5294 ValueOperand value, FloatRegister temp, Register output, Label* fail,
5295 bool negativeZeroCheck,
5296 IntConversionInputKind conversion = IntConversionInputKind::Any) {
5297 convertValueToInt(
5298 value, nullptr, nullptr, nullptr, InvalidReg, temp, output, fail,
5299 negativeZeroCheck ? IntConversionBehavior::NegativeZeroCheck
5300 : IntConversionBehavior::Normal,
5301 conversion);
5302 }
5303
5304 // This carries over the MTruncateToInt32 operation on the ValueOperand
5305 // input; see the comment at the top of this class.
truncateValueToInt32(ValueOperand value,Label * handleStringEntry,Label * handleStringRejoin,Label * truncateDoubleSlow,Register stringReg,FloatRegister temp,Register output,Label * fail)5306 void truncateValueToInt32(ValueOperand value, Label* handleStringEntry,
5307 Label* handleStringRejoin,
5308 Label* truncateDoubleSlow, Register stringReg,
5309 FloatRegister temp, Register output, Label* fail) {
5310 convertValueToInt(value, handleStringEntry, handleStringRejoin,
5311 truncateDoubleSlow, stringReg, temp, output, fail,
5312 IntConversionBehavior::Truncate);
5313 }
5314
truncateValueToInt32(ValueOperand value,FloatRegister temp,Register output,Label * fail)5315 void truncateValueToInt32(ValueOperand value, FloatRegister temp,
5316 Register output, Label* fail) {
5317 truncateValueToInt32(value, nullptr, nullptr, nullptr, InvalidReg, temp,
5318 output, fail);
5319 }
5320
5321 // Truncates, i.e. removes any fractional parts, but doesn't wrap around to
5322 // the int32 range.
truncateNoWrapValueToInt32(ValueOperand value,FloatRegister temp,Register output,Label * truncateDoubleSlow,Label * fail)5323 void truncateNoWrapValueToInt32(ValueOperand value, FloatRegister temp,
5324 Register output, Label* truncateDoubleSlow,
5325 Label* fail) {
5326 convertValueToInt(value, nullptr, nullptr, truncateDoubleSlow, InvalidReg,
5327 temp, output, fail,
5328 IntConversionBehavior::TruncateNoWrap);
5329 }
5330
5331 // Convenience functions for clamping values to uint8.
clampValueToUint8(ValueOperand value,Label * handleStringEntry,Label * handleStringRejoin,Register stringReg,FloatRegister temp,Register output,Label * fail)5332 void clampValueToUint8(ValueOperand value, Label* handleStringEntry,
5333 Label* handleStringRejoin, Register stringReg,
5334 FloatRegister temp, Register output, Label* fail) {
5335 convertValueToInt(value, handleStringEntry, handleStringRejoin, nullptr,
5336 stringReg, temp, output, fail,
5337 IntConversionBehavior::ClampToUint8);
5338 }
5339
5340 [[nodiscard]] bool icBuildOOLFakeExitFrame(void* fakeReturnAddr,
5341 AutoSaveLiveRegisters& save);
5342
5343 // Align the stack pointer based on the number of arguments which are pushed
5344 // on the stack, such that the JitFrameLayout would be correctly aligned on
5345 // the JitStackAlignment.
5346 void alignJitStackBasedOnNArgs(Register nargs, bool countIncludesThis);
5347 void alignJitStackBasedOnNArgs(uint32_t argc);
5348
5349 inline void assertStackAlignment(uint32_t alignment, int32_t offset = 0);
5350
5351 void touchFrameValues(Register numStackValues, Register scratch1,
5352 Register scratch2);
5353
5354 #ifdef JS_64BIT
5355 // See comment block "64-bit GPRs carrying 32-bit values" above. This asserts
5356 // that the high bits of the register are appropriate for the architecture and
5357 // the value in the low bits.
5358 void debugAssertCanonicalInt32(Register r);
5359 #endif
5360 };
5361
5362 // StackMacroAssembler checks no GC will happen while it's on the stack.
5363 class MOZ_RAII StackMacroAssembler : public MacroAssembler {
5364 JS::AutoCheckCannotGC nogc;
5365
5366 public:
StackMacroAssembler()5367 StackMacroAssembler() : MacroAssembler() {}
StackMacroAssembler(JSContext * cx)5368 explicit StackMacroAssembler(JSContext* cx) : MacroAssembler(cx) {}
5369 };
5370
5371 // WasmMacroAssembler does not contain GC pointers, so it doesn't need the no-GC
5372 // checking StackMacroAssembler has.
5373 class MOZ_RAII WasmMacroAssembler : public MacroAssembler {
5374 public:
5375 explicit WasmMacroAssembler(TempAllocator& alloc, bool limitedSize = true);
5376 explicit WasmMacroAssembler(TempAllocator& alloc,
5377 const wasm::ModuleEnvironment& env,
5378 bool limitedSize = true);
~WasmMacroAssembler()5379 ~WasmMacroAssembler() { assertNoGCThings(); }
5380 };
5381
5382 // Heap-allocated MacroAssembler used for Ion off-thread code generation.
5383 // GC cancels off-thread compilations.
5384 class IonHeapMacroAssembler : public MacroAssembler {
5385 public:
IonHeapMacroAssembler()5386 IonHeapMacroAssembler() : MacroAssembler() {
5387 MOZ_ASSERT(CurrentThreadIsIonCompiling());
5388 }
5389 };
5390
5391 //{{{ check_macroassembler_style
framePushed()5392 inline uint32_t MacroAssembler::framePushed() const { return framePushed_; }
5393
setFramePushed(uint32_t framePushed)5394 inline void MacroAssembler::setFramePushed(uint32_t framePushed) {
5395 framePushed_ = framePushed;
5396 }
5397
adjustFrame(int32_t value)5398 inline void MacroAssembler::adjustFrame(int32_t value) {
5399 MOZ_ASSERT_IF(value < 0, framePushed_ >= uint32_t(-value));
5400 setFramePushed(framePushed_ + value);
5401 }
5402
implicitPop(uint32_t bytes)5403 inline void MacroAssembler::implicitPop(uint32_t bytes) {
5404 MOZ_ASSERT(bytes % sizeof(intptr_t) == 0);
5405 MOZ_ASSERT(bytes <= INT32_MAX);
5406 adjustFrame(-int32_t(bytes));
5407 }
5408 //}}} check_macroassembler_style
5409
JSOpToDoubleCondition(JSOp op)5410 static inline Assembler::DoubleCondition JSOpToDoubleCondition(JSOp op) {
5411 switch (op) {
5412 case JSOp::Eq:
5413 case JSOp::StrictEq:
5414 return Assembler::DoubleEqual;
5415 case JSOp::Ne:
5416 case JSOp::StrictNe:
5417 return Assembler::DoubleNotEqualOrUnordered;
5418 case JSOp::Lt:
5419 return Assembler::DoubleLessThan;
5420 case JSOp::Le:
5421 return Assembler::DoubleLessThanOrEqual;
5422 case JSOp::Gt:
5423 return Assembler::DoubleGreaterThan;
5424 case JSOp::Ge:
5425 return Assembler::DoubleGreaterThanOrEqual;
5426 default:
5427 MOZ_CRASH("Unexpected comparison operation");
5428 }
5429 }
5430
5431 // Note: the op may have been inverted during lowering (to put constants in a
5432 // position where they can be immediates), so it is important to use the
5433 // lir->jsop() instead of the mir->jsop() when it is present.
JSOpToCondition(JSOp op,bool isSigned)5434 static inline Assembler::Condition JSOpToCondition(JSOp op, bool isSigned) {
5435 if (isSigned) {
5436 switch (op) {
5437 case JSOp::Eq:
5438 case JSOp::StrictEq:
5439 return Assembler::Equal;
5440 case JSOp::Ne:
5441 case JSOp::StrictNe:
5442 return Assembler::NotEqual;
5443 case JSOp::Lt:
5444 return Assembler::LessThan;
5445 case JSOp::Le:
5446 return Assembler::LessThanOrEqual;
5447 case JSOp::Gt:
5448 return Assembler::GreaterThan;
5449 case JSOp::Ge:
5450 return Assembler::GreaterThanOrEqual;
5451 default:
5452 MOZ_CRASH("Unrecognized comparison operation");
5453 }
5454 } else {
5455 switch (op) {
5456 case JSOp::Eq:
5457 case JSOp::StrictEq:
5458 return Assembler::Equal;
5459 case JSOp::Ne:
5460 case JSOp::StrictNe:
5461 return Assembler::NotEqual;
5462 case JSOp::Lt:
5463 return Assembler::Below;
5464 case JSOp::Le:
5465 return Assembler::BelowOrEqual;
5466 case JSOp::Gt:
5467 return Assembler::Above;
5468 case JSOp::Ge:
5469 return Assembler::AboveOrEqual;
5470 default:
5471 MOZ_CRASH("Unrecognized comparison operation");
5472 }
5473 }
5474 }
5475
StackDecrementForCall(uint32_t alignment,size_t bytesAlreadyPushed,size_t bytesToPush)5476 static inline size_t StackDecrementForCall(uint32_t alignment,
5477 size_t bytesAlreadyPushed,
5478 size_t bytesToPush) {
5479 return bytesToPush +
5480 ComputeByteAlignment(bytesAlreadyPushed + bytesToPush, alignment);
5481 }
5482
5483 // Helper for generatePreBarrier.
5484 inline DynFn JitPreWriteBarrier(MIRType type);
5485 } // namespace jit
5486
5487 } // namespace js
5488
5489 #endif /* jit_MacroAssembler_h */
5490