1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- 2 * vim: set ts=8 sts=4 et sw=4 tw=99: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_x86_shared_Assembler_x86_shared_h 8 #define jit_x86_shared_Assembler_x86_shared_h 9 10 #include <cstddef> 11 12 #include "jit/shared/Assembler-shared.h" 13 14 #if defined(JS_CODEGEN_X86) 15 # include "jit/x86/BaseAssembler-x86.h" 16 #elif defined(JS_CODEGEN_X64) 17 # include "jit/x64/BaseAssembler-x64.h" 18 #else 19 # error "Unknown architecture!" 20 #endif 21 22 namespace js { 23 namespace jit { 24 25 struct ScratchFloat32Scope : public AutoFloatRegisterScope 26 { 27 explicit ScratchFloat32Scope(MacroAssembler& masm) 28 : AutoFloatRegisterScope(masm, ScratchFloat32Reg) 29 { } 30 }; 31 32 struct ScratchDoubleScope : public AutoFloatRegisterScope 33 { 34 explicit ScratchDoubleScope(MacroAssembler& masm) 35 : AutoFloatRegisterScope(masm, ScratchDoubleReg) 36 { } 37 }; 38 39 struct ScratchSimd128Scope : public AutoFloatRegisterScope 40 { 41 explicit ScratchSimd128Scope(MacroAssembler& masm) 42 : AutoFloatRegisterScope(masm, ScratchSimd128Reg) 43 { } 44 }; 45 46 class Operand 47 { 48 public: 49 enum Kind { 50 REG, 51 MEM_REG_DISP, 52 FPREG, 53 MEM_SCALE, 54 MEM_ADDRESS32 55 }; 56 57 private: 58 Kind kind_ : 4; 59 // Used as a Register::Encoding and a FloatRegister::Encoding. 60 uint32_t base_ : 5; 61 Scale scale_ : 3; 62 // We don't use all 8 bits, of course, but GCC complains if the size of 63 // this field is smaller than the size of Register::Encoding. 64 Register::Encoding index_ : 8; 65 int32_t disp_; 66 67 public: 68 explicit Operand(Register reg) 69 : kind_(REG), 70 base_(reg.encoding()), 71 scale_(TimesOne), 72 index_(Registers::Invalid), 73 disp_(0) 74 { } 75 explicit Operand(FloatRegister reg) 76 : kind_(FPREG), 77 base_(reg.encoding()), 78 scale_(TimesOne), 79 index_(Registers::Invalid), 80 disp_(0) 81 { } 82 explicit Operand(const Address& address) 83 : kind_(MEM_REG_DISP), 84 base_(address.base.encoding()), 85 scale_(TimesOne), 86 index_(Registers::Invalid), 87 disp_(address.offset) 88 { } 89 explicit Operand(const BaseIndex& address) 90 : kind_(MEM_SCALE), 91 base_(address.base.encoding()), 92 scale_(address.scale), 93 index_(address.index.encoding()), 94 disp_(address.offset) 95 { } 96 Operand(Register base, Register index, Scale scale, int32_t disp = 0) 97 : kind_(MEM_SCALE), 98 base_(base.encoding()), 99 scale_(scale), 100 index_(index.encoding()), 101 disp_(disp) 102 { } 103 Operand(Register reg, int32_t disp) 104 : kind_(MEM_REG_DISP), 105 base_(reg.encoding()), 106 scale_(TimesOne), 107 index_(Registers::Invalid), 108 disp_(disp) 109 { } 110 explicit Operand(AbsoluteAddress address) 111 : kind_(MEM_ADDRESS32), 112 base_(Registers::Invalid), 113 scale_(TimesOne), 114 index_(Registers::Invalid), 115 disp_(X86Encoding::AddressImmediate(address.addr)) 116 { } 117 explicit Operand(PatchedAbsoluteAddress address) 118 : kind_(MEM_ADDRESS32), 119 base_(Registers::Invalid), 120 scale_(TimesOne), 121 index_(Registers::Invalid), 122 disp_(X86Encoding::AddressImmediate(address.addr)) 123 { } 124 125 Address toAddress() const { 126 MOZ_ASSERT(kind() == MEM_REG_DISP); 127 return Address(Register::FromCode(base()), disp()); 128 } 129 130 BaseIndex toBaseIndex() const { 131 MOZ_ASSERT(kind() == MEM_SCALE); 132 return BaseIndex(Register::FromCode(base()), Register::FromCode(index()), scale(), disp()); 133 } 134 135 Kind kind() const { 136 return kind_; 137 } 138 Register::Encoding reg() const { 139 MOZ_ASSERT(kind() == REG); 140 return Register::Encoding(base_); 141 } 142 Register::Encoding base() const { 143 MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); 144 return Register::Encoding(base_); 145 } 146 Register::Encoding index() const { 147 MOZ_ASSERT(kind() == MEM_SCALE); 148 return index_; 149 } 150 Scale scale() const { 151 MOZ_ASSERT(kind() == MEM_SCALE); 152 return scale_; 153 } 154 FloatRegister::Encoding fpu() const { 155 MOZ_ASSERT(kind() == FPREG); 156 return FloatRegister::Encoding(base_); 157 } 158 int32_t disp() const { 159 MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE); 160 return disp_; 161 } 162 void* address() const { 163 MOZ_ASSERT(kind() == MEM_ADDRESS32); 164 return reinterpret_cast<void*>(disp_); 165 } 166 167 bool containsReg(Register r) const { 168 switch (kind()) { 169 case REG: return r.encoding() == reg(); 170 case MEM_REG_DISP: return r.encoding() == base(); 171 case MEM_SCALE: return r.encoding() == base() || r.encoding() == index(); 172 default: return false; 173 } 174 } 175 }; 176 177 inline Imm32 178 Imm64::firstHalf() const 179 { 180 return low(); 181 } 182 183 inline Imm32 184 Imm64::secondHalf() const 185 { 186 return hi(); 187 } 188 189 class CPUInfo 190 { 191 public: 192 // As the SSE's were introduced in order, the presence of a later SSE implies 193 // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2 support. 194 enum SSEVersion { 195 UnknownSSE = 0, 196 NoSSE = 1, 197 SSE = 2, 198 SSE2 = 3, 199 SSE3 = 4, 200 SSSE3 = 5, 201 SSE4_1 = 6, 202 SSE4_2 = 7 203 }; 204 205 static SSEVersion GetSSEVersion() { 206 if (maxSSEVersion == UnknownSSE) 207 SetSSEVersion(); 208 209 MOZ_ASSERT(maxSSEVersion != UnknownSSE); 210 MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE, maxSSEVersion <= maxEnabledSSEVersion); 211 return maxSSEVersion; 212 } 213 214 static bool IsAVXPresent() { 215 if (MOZ_UNLIKELY(maxSSEVersion == UnknownSSE)) 216 SetSSEVersion(); 217 218 MOZ_ASSERT_IF(!avxEnabled, !avxPresent); 219 return avxPresent; 220 } 221 222 private: 223 static SSEVersion maxSSEVersion; 224 static SSEVersion maxEnabledSSEVersion; 225 static bool avxPresent; 226 static bool avxEnabled; 227 static bool popcntPresent; 228 static bool needAmdBugWorkaround; 229 230 static void SetSSEVersion(); 231 232 public: 233 static bool IsSSE2Present() { 234 #ifdef JS_CODEGEN_X64 235 return true; 236 #else 237 return GetSSEVersion() >= SSE2; 238 #endif 239 } 240 static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; } 241 static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; } 242 static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; } 243 static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; } 244 static bool IsPOPCNTPresent() { return popcntPresent; } 245 static bool NeedAmdBugWorkaround() { return needAmdBugWorkaround; } 246 247 static void SetSSE3Disabled() { maxEnabledSSEVersion = SSE2; avxEnabled = false; } 248 static void SetSSE4Disabled() { maxEnabledSSEVersion = SSSE3; avxEnabled = false; } 249 static void SetAVXEnabled() { avxEnabled = true; } 250 }; 251 252 class AssemblerX86Shared : public AssemblerShared 253 { 254 protected: 255 struct RelativePatch { 256 int32_t offset; 257 void* target; 258 Relocation::Kind kind; 259 260 RelativePatch(int32_t offset, void* target, Relocation::Kind kind) 261 : offset(offset), 262 target(target), 263 kind(kind) 264 { } 265 }; 266 267 Vector<RelativePatch, 8, SystemAllocPolicy> jumps_; 268 CompactBufferWriter jumpRelocations_; 269 CompactBufferWriter dataRelocations_; 270 CompactBufferWriter preBarriers_; 271 272 void writeDataRelocation(ImmGCPtr ptr) { 273 if (ptr.value) { 274 if (gc::IsInsideNursery(ptr.value)) 275 embedsNurseryPointers_ = true; 276 dataRelocations_.writeUnsigned(masm.currentOffset()); 277 } 278 } 279 void writePrebarrierOffset(CodeOffset label) { 280 preBarriers_.writeUnsigned(label.offset()); 281 } 282 283 protected: 284 X86Encoding::BaseAssemblerSpecific masm; 285 286 typedef X86Encoding::JmpSrc JmpSrc; 287 typedef X86Encoding::JmpDst JmpDst; 288 289 public: 290 AssemblerX86Shared() 291 { 292 if (!HasAVX()) 293 masm.disableVEX(); 294 } 295 296 enum Condition { 297 Equal = X86Encoding::ConditionE, 298 NotEqual = X86Encoding::ConditionNE, 299 Above = X86Encoding::ConditionA, 300 AboveOrEqual = X86Encoding::ConditionAE, 301 Below = X86Encoding::ConditionB, 302 BelowOrEqual = X86Encoding::ConditionBE, 303 GreaterThan = X86Encoding::ConditionG, 304 GreaterThanOrEqual = X86Encoding::ConditionGE, 305 LessThan = X86Encoding::ConditionL, 306 LessThanOrEqual = X86Encoding::ConditionLE, 307 Overflow = X86Encoding::ConditionO, 308 CarrySet = X86Encoding::ConditionC, 309 CarryClear = X86Encoding::ConditionNC, 310 Signed = X86Encoding::ConditionS, 311 NotSigned = X86Encoding::ConditionNS, 312 Zero = X86Encoding::ConditionE, 313 NonZero = X86Encoding::ConditionNE, 314 Parity = X86Encoding::ConditionP, 315 NoParity = X86Encoding::ConditionNP 316 }; 317 318 // If this bit is set, the vucomisd operands have to be inverted. 319 static const int DoubleConditionBitInvert = 0x10; 320 321 // Bit set when a DoubleCondition does not map to a single x86 condition. 322 // The macro assembler has to special-case these conditions. 323 static const int DoubleConditionBitSpecial = 0x20; 324 static const int DoubleConditionBits = DoubleConditionBitInvert | DoubleConditionBitSpecial; 325 326 enum DoubleCondition { 327 // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN. 328 DoubleOrdered = NoParity, 329 DoubleEqual = Equal | DoubleConditionBitSpecial, 330 DoubleNotEqual = NotEqual, 331 DoubleGreaterThan = Above, 332 DoubleGreaterThanOrEqual = AboveOrEqual, 333 DoubleLessThan = Above | DoubleConditionBitInvert, 334 DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert, 335 // If either operand is NaN, these conditions always evaluate to true. 336 DoubleUnordered = Parity, 337 DoubleEqualOrUnordered = Equal, 338 DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial, 339 DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert, 340 DoubleGreaterThanOrEqualOrUnordered = BelowOrEqual | DoubleConditionBitInvert, 341 DoubleLessThanOrUnordered = Below, 342 DoubleLessThanOrEqualOrUnordered = BelowOrEqual 343 }; 344 345 enum NaNCond { 346 NaN_HandledByCond, 347 NaN_IsTrue, 348 NaN_IsFalse 349 }; 350 351 // If the primary condition returned by ConditionFromDoubleCondition doesn't 352 // handle NaNs properly, return NaN_IsFalse if the comparison should be 353 // overridden to return false on NaN, NaN_IsTrue if it should be overridden 354 // to return true on NaN, or NaN_HandledByCond if no secondary check is 355 // needed. 356 static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) { 357 switch (cond) { 358 case DoubleOrdered: 359 case DoubleNotEqual: 360 case DoubleGreaterThan: 361 case DoubleGreaterThanOrEqual: 362 case DoubleLessThan: 363 case DoubleLessThanOrEqual: 364 case DoubleUnordered: 365 case DoubleEqualOrUnordered: 366 case DoubleGreaterThanOrUnordered: 367 case DoubleGreaterThanOrEqualOrUnordered: 368 case DoubleLessThanOrUnordered: 369 case DoubleLessThanOrEqualOrUnordered: 370 return NaN_HandledByCond; 371 case DoubleEqual: 372 return NaN_IsFalse; 373 case DoubleNotEqualOrUnordered: 374 return NaN_IsTrue; 375 } 376 377 MOZ_CRASH("Unknown double condition"); 378 } 379 380 static void StaticAsserts() { 381 // DoubleConditionBits should not interfere with x86 condition codes. 382 JS_STATIC_ASSERT(!((Equal | NotEqual | Above | AboveOrEqual | Below | 383 BelowOrEqual | Parity | NoParity) & DoubleConditionBits)); 384 } 385 386 static Condition InvertCondition(Condition cond); 387 static Condition UnsignedCondition(Condition cond); 388 static Condition ConditionWithoutEqual(Condition cond); 389 390 // Return the primary condition to test. Some primary conditions may not 391 // handle NaNs properly and may therefore require a secondary condition. 392 // Use NaNCondFromDoubleCondition to determine what else is needed. 393 static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) { 394 return static_cast<Condition>(cond & ~DoubleConditionBits); 395 } 396 397 static void TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader); 398 399 // MacroAssemblers hold onto gcthings, so they are traced by the GC. 400 void trace(JSTracer* trc); 401 402 bool oom() const { 403 return AssemblerShared::oom() || 404 masm.oom() || 405 jumpRelocations_.oom() || 406 dataRelocations_.oom() || 407 preBarriers_.oom(); 408 } 409 410 void setPrinter(Sprinter* sp) { 411 masm.setPrinter(sp); 412 } 413 414 static const Register getStackPointer() { 415 return StackPointer; 416 } 417 418 void executableCopy(void* buffer); 419 bool asmMergeWith(const AssemblerX86Shared& other) { 420 MOZ_ASSERT(other.jumps_.length() == 0); 421 if (!AssemblerShared::asmMergeWith(masm.size(), other)) 422 return false; 423 return masm.appendBuffer(other.masm); 424 } 425 void processCodeLabels(uint8_t* rawCode); 426 void copyJumpRelocationTable(uint8_t* dest); 427 void copyDataRelocationTable(uint8_t* dest); 428 void copyPreBarrierTable(uint8_t* dest); 429 430 // Size of the instruction stream, in bytes. 431 size_t size() const { 432 return masm.size(); 433 } 434 // Size of the jump relocation table, in bytes. 435 size_t jumpRelocationTableBytes() const { 436 return jumpRelocations_.length(); 437 } 438 size_t dataRelocationTableBytes() const { 439 return dataRelocations_.length(); 440 } 441 size_t preBarrierTableBytes() const { 442 return preBarriers_.length(); 443 } 444 // Size of the data table, in bytes. 445 size_t bytesNeeded() const { 446 return size() + 447 jumpRelocationTableBytes() + 448 dataRelocationTableBytes() + 449 preBarrierTableBytes(); 450 } 451 452 public: 453 void haltingAlign(int alignment) { 454 masm.haltingAlign(alignment); 455 } 456 void nopAlign(int alignment) { 457 masm.nopAlign(alignment); 458 } 459 void writeCodePointer(CodeOffset* label) { 460 // A CodeOffset only has one use, bake in the "end of list" value. 461 masm.jumpTablePointer(LabelBase::INVALID_OFFSET); 462 label->bind(masm.size()); 463 } 464 void cmovz(const Operand& src, Register dest) { 465 switch (src.kind()) { 466 case Operand::REG: 467 masm.cmovz_rr(src.reg(), dest.encoding()); 468 break; 469 case Operand::MEM_REG_DISP: 470 masm.cmovz_mr(src.disp(), src.base(), dest.encoding()); 471 break; 472 case Operand::MEM_SCALE: 473 masm.cmovz_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 474 break; 475 default: 476 MOZ_CRASH("unexpected operand kind"); 477 } 478 } 479 void movl(Imm32 imm32, Register dest) { 480 masm.movl_i32r(imm32.value, dest.encoding()); 481 } 482 void movl(Register src, Register dest) { 483 masm.movl_rr(src.encoding(), dest.encoding()); 484 } 485 void movl(const Operand& src, Register dest) { 486 switch (src.kind()) { 487 case Operand::REG: 488 masm.movl_rr(src.reg(), dest.encoding()); 489 break; 490 case Operand::MEM_REG_DISP: 491 masm.movl_mr(src.disp(), src.base(), dest.encoding()); 492 break; 493 case Operand::MEM_SCALE: 494 masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 495 break; 496 case Operand::MEM_ADDRESS32: 497 masm.movl_mr(src.address(), dest.encoding()); 498 break; 499 default: 500 MOZ_CRASH("unexpected operand kind"); 501 } 502 } 503 void movl(Register src, const Operand& dest) { 504 switch (dest.kind()) { 505 case Operand::REG: 506 masm.movl_rr(src.encoding(), dest.reg()); 507 break; 508 case Operand::MEM_REG_DISP: 509 masm.movl_rm(src.encoding(), dest.disp(), dest.base()); 510 break; 511 case Operand::MEM_SCALE: 512 masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 513 break; 514 case Operand::MEM_ADDRESS32: 515 masm.movl_rm(src.encoding(), dest.address()); 516 break; 517 default: 518 MOZ_CRASH("unexpected operand kind"); 519 } 520 } 521 void movl(Imm32 imm32, const Operand& dest) { 522 switch (dest.kind()) { 523 case Operand::REG: 524 masm.movl_i32r(imm32.value, dest.reg()); 525 break; 526 case Operand::MEM_REG_DISP: 527 masm.movl_i32m(imm32.value, dest.disp(), dest.base()); 528 break; 529 case Operand::MEM_SCALE: 530 masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(), dest.scale()); 531 break; 532 case Operand::MEM_ADDRESS32: 533 masm.movl_i32m(imm32.value, dest.address()); 534 break; 535 default: 536 MOZ_CRASH("unexpected operand kind"); 537 } 538 } 539 540 void xchgl(Register src, Register dest) { 541 masm.xchgl_rr(src.encoding(), dest.encoding()); 542 } 543 544 // Eventually vmovapd should be overloaded to support loads and 545 // stores too. 546 void vmovapd(FloatRegister src, FloatRegister dest) { 547 MOZ_ASSERT(HasSSE2()); 548 masm.vmovapd_rr(src.encoding(), dest.encoding()); 549 } 550 551 void vmovaps(FloatRegister src, FloatRegister dest) { 552 MOZ_ASSERT(HasSSE2()); 553 masm.vmovaps_rr(src.encoding(), dest.encoding()); 554 } 555 void vmovaps(const Operand& src, FloatRegister dest) { 556 MOZ_ASSERT(HasSSE2()); 557 switch (src.kind()) { 558 case Operand::MEM_REG_DISP: 559 masm.vmovaps_mr(src.disp(), src.base(), dest.encoding()); 560 break; 561 case Operand::MEM_SCALE: 562 masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 563 break; 564 case Operand::FPREG: 565 masm.vmovaps_rr(src.fpu(), dest.encoding()); 566 break; 567 default: 568 MOZ_CRASH("unexpected operand kind"); 569 } 570 } 571 void vmovaps(FloatRegister src, const Operand& dest) { 572 MOZ_ASSERT(HasSSE2()); 573 switch (dest.kind()) { 574 case Operand::MEM_REG_DISP: 575 masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base()); 576 break; 577 case Operand::MEM_SCALE: 578 masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 579 break; 580 default: 581 MOZ_CRASH("unexpected operand kind"); 582 } 583 } 584 void vmovups(const Operand& src, FloatRegister dest) { 585 MOZ_ASSERT(HasSSE2()); 586 switch (src.kind()) { 587 case Operand::MEM_REG_DISP: 588 masm.vmovups_mr(src.disp(), src.base(), dest.encoding()); 589 break; 590 case Operand::MEM_SCALE: 591 masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 592 break; 593 default: 594 MOZ_CRASH("unexpected operand kind"); 595 } 596 } 597 void vmovups(FloatRegister src, const Operand& dest) { 598 MOZ_ASSERT(HasSSE2()); 599 switch (dest.kind()) { 600 case Operand::MEM_REG_DISP: 601 masm.vmovups_rm(src.encoding(), dest.disp(), dest.base()); 602 break; 603 case Operand::MEM_SCALE: 604 masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 605 break; 606 default: 607 MOZ_CRASH("unexpected operand kind"); 608 } 609 } 610 611 // vmovsd is only provided in load/store form since the 612 // register-to-register form has different semantics (it doesn't clobber 613 // the whole output register) and isn't needed currently. 614 void vmovsd(const Address& src, FloatRegister dest) { 615 masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding()); 616 } 617 void vmovsd(const BaseIndex& src, FloatRegister dest) { 618 masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); 619 } 620 void vmovsd(FloatRegister src, const Address& dest) { 621 masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding()); 622 } 623 void vmovsd(FloatRegister src, const BaseIndex& dest) { 624 masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); 625 } 626 // Although vmovss is not only provided in load/store form (for the same 627 // reasons as vmovsd above), the register to register form should be only 628 // used in contexts where we care about not clearing the higher lanes of 629 // the FloatRegister. 630 void vmovss(const Address& src, FloatRegister dest) { 631 masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding()); 632 } 633 void vmovss(const BaseIndex& src, FloatRegister dest) { 634 masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(), src.scale, dest.encoding()); 635 } 636 void vmovss(FloatRegister src, const Address& dest) { 637 masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding()); 638 } 639 void vmovss(FloatRegister src, const BaseIndex& dest) { 640 masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(), dest.index.encoding(), dest.scale); 641 } 642 void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 643 masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 644 } 645 void vmovdqu(const Operand& src, FloatRegister dest) { 646 MOZ_ASSERT(HasSSE2()); 647 switch (src.kind()) { 648 case Operand::MEM_REG_DISP: 649 masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding()); 650 break; 651 case Operand::MEM_SCALE: 652 masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 653 break; 654 default: 655 MOZ_CRASH("unexpected operand kind"); 656 } 657 } 658 void vmovdqu(FloatRegister src, const Operand& dest) { 659 MOZ_ASSERT(HasSSE2()); 660 switch (dest.kind()) { 661 case Operand::MEM_REG_DISP: 662 masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base()); 663 break; 664 case Operand::MEM_SCALE: 665 masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 666 break; 667 default: 668 MOZ_CRASH("unexpected operand kind"); 669 } 670 } 671 void vmovdqa(const Operand& src, FloatRegister dest) { 672 MOZ_ASSERT(HasSSE2()); 673 switch (src.kind()) { 674 case Operand::FPREG: 675 masm.vmovdqa_rr(src.fpu(), dest.encoding()); 676 break; 677 case Operand::MEM_REG_DISP: 678 masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding()); 679 break; 680 case Operand::MEM_SCALE: 681 masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 682 break; 683 default: 684 MOZ_CRASH("unexpected operand kind"); 685 } 686 } 687 void vmovdqa(FloatRegister src, const Operand& dest) { 688 MOZ_ASSERT(HasSSE2()); 689 switch (dest.kind()) { 690 case Operand::MEM_REG_DISP: 691 masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base()); 692 break; 693 case Operand::MEM_SCALE: 694 masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 695 break; 696 default: 697 MOZ_CRASH("unexpected operand kind"); 698 } 699 } 700 void vmovdqa(FloatRegister src, FloatRegister dest) { 701 MOZ_ASSERT(HasSSE2()); 702 masm.vmovdqa_rr(src.encoding(), dest.encoding()); 703 } 704 void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 705 MOZ_ASSERT(HasSSE2()); 706 masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 707 } 708 void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 709 MOZ_ASSERT(HasSSE2()); 710 masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 711 } 712 void movzbl(const Operand& src, Register dest) { 713 switch (src.kind()) { 714 case Operand::MEM_REG_DISP: 715 masm.movzbl_mr(src.disp(), src.base(), dest.encoding()); 716 break; 717 case Operand::MEM_SCALE: 718 masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 719 break; 720 default: 721 MOZ_CRASH("unexpected operand kind"); 722 } 723 } 724 void movsbl(Register src, Register dest) { 725 masm.movsbl_rr(src.encoding(), dest.encoding()); 726 } 727 void movsbl(const Operand& src, Register dest) { 728 switch (src.kind()) { 729 case Operand::MEM_REG_DISP: 730 masm.movsbl_mr(src.disp(), src.base(), dest.encoding()); 731 break; 732 case Operand::MEM_SCALE: 733 masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 734 break; 735 default: 736 MOZ_CRASH("unexpected operand kind"); 737 } 738 } 739 void movb(const Operand& src, Register dest) { 740 switch (src.kind()) { 741 case Operand::MEM_REG_DISP: 742 masm.movb_mr(src.disp(), src.base(), dest.encoding()); 743 break; 744 case Operand::MEM_SCALE: 745 masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 746 break; 747 default: 748 MOZ_CRASH("unexpected operand kind"); 749 } 750 } 751 void movb(Imm32 src, Register dest) { 752 masm.movb_ir(src.value & 255, dest.encoding()); 753 } 754 void movb(Register src, const Operand& dest) { 755 switch (dest.kind()) { 756 case Operand::MEM_REG_DISP: 757 masm.movb_rm(src.encoding(), dest.disp(), dest.base()); 758 break; 759 case Operand::MEM_SCALE: 760 masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 761 break; 762 default: 763 MOZ_CRASH("unexpected operand kind"); 764 } 765 } 766 void movb(Imm32 src, const Operand& dest) { 767 switch (dest.kind()) { 768 case Operand::MEM_REG_DISP: 769 masm.movb_im(src.value, dest.disp(), dest.base()); 770 break; 771 case Operand::MEM_SCALE: 772 masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); 773 break; 774 default: 775 MOZ_CRASH("unexpected operand kind"); 776 } 777 } 778 void movzwl(const Operand& src, Register dest) { 779 switch (src.kind()) { 780 case Operand::REG: 781 masm.movzwl_rr(src.reg(), dest.encoding()); 782 break; 783 case Operand::MEM_REG_DISP: 784 masm.movzwl_mr(src.disp(), src.base(), dest.encoding()); 785 break; 786 case Operand::MEM_SCALE: 787 masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 788 break; 789 default: 790 MOZ_CRASH("unexpected operand kind"); 791 } 792 } 793 void movzwl(Register src, Register dest) { 794 masm.movzwl_rr(src.encoding(), dest.encoding()); 795 } 796 void movw(const Operand& src, Register dest) { 797 masm.prefix_16_for_32(); 798 movl(src, dest); 799 } 800 void movw(Imm32 src, Register dest) { 801 masm.prefix_16_for_32(); 802 movl(src, dest); 803 } 804 void movw(Register src, const Operand& dest) { 805 switch (dest.kind()) { 806 case Operand::MEM_REG_DISP: 807 masm.movw_rm(src.encoding(), dest.disp(), dest.base()); 808 break; 809 case Operand::MEM_SCALE: 810 masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 811 break; 812 default: 813 MOZ_CRASH("unexpected operand kind"); 814 } 815 } 816 void movw(Imm32 src, const Operand& dest) { 817 switch (dest.kind()) { 818 case Operand::MEM_REG_DISP: 819 masm.movw_im(src.value, dest.disp(), dest.base()); 820 break; 821 case Operand::MEM_SCALE: 822 masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(), dest.scale()); 823 break; 824 default: 825 MOZ_CRASH("unexpected operand kind"); 826 } 827 } 828 void movswl(Register src, Register dest) { 829 masm.movswl_rr(src.encoding(), dest.encoding()); 830 } 831 void movswl(const Operand& src, Register dest) { 832 switch (src.kind()) { 833 case Operand::MEM_REG_DISP: 834 masm.movswl_mr(src.disp(), src.base(), dest.encoding()); 835 break; 836 case Operand::MEM_SCALE: 837 masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 838 break; 839 default: 840 MOZ_CRASH("unexpected operand kind"); 841 } 842 } 843 void leal(const Operand& src, Register dest) { 844 switch (src.kind()) { 845 case Operand::MEM_REG_DISP: 846 masm.leal_mr(src.disp(), src.base(), dest.encoding()); 847 break; 848 case Operand::MEM_SCALE: 849 masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 850 break; 851 default: 852 MOZ_CRASH("unexpected operand kind"); 853 } 854 } 855 856 protected: 857 void jSrc(Condition cond, Label* label) { 858 if (label->bound()) { 859 // The jump can be immediately encoded to the correct destination. 860 masm.jCC_i(static_cast<X86Encoding::Condition>(cond), JmpDst(label->offset())); 861 } else { 862 // Thread the jump list through the unpatched jump targets. 863 JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond)); 864 JmpSrc prev = JmpSrc(label->use(j.offset())); 865 masm.setNextJump(j, prev); 866 } 867 } 868 void jmpSrc(Label* label) { 869 if (label->bound()) { 870 // The jump can be immediately encoded to the correct destination. 871 masm.jmp_i(JmpDst(label->offset())); 872 } else { 873 // Thread the jump list through the unpatched jump targets. 874 JmpSrc j = masm.jmp(); 875 JmpSrc prev = JmpSrc(label->use(j.offset())); 876 masm.setNextJump(j, prev); 877 } 878 } 879 880 // Comparison of EAX against the address given by a Label. 881 JmpSrc cmpSrc(Label* label) { 882 JmpSrc j = masm.cmp_eax(); 883 if (label->bound()) { 884 // The jump can be immediately patched to the correct destination. 885 masm.linkJump(j, JmpDst(label->offset())); 886 } else { 887 // Thread the jump list through the unpatched jump targets. 888 JmpSrc prev = JmpSrc(label->use(j.offset())); 889 masm.setNextJump(j, prev); 890 } 891 return j; 892 } 893 894 JmpSrc jSrc(Condition cond, RepatchLabel* label) { 895 JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond)); 896 if (label->bound()) { 897 // The jump can be immediately patched to the correct destination. 898 masm.linkJump(j, JmpDst(label->offset())); 899 } else { 900 label->use(j.offset()); 901 } 902 return j; 903 } 904 JmpSrc jmpSrc(RepatchLabel* label) { 905 JmpSrc j = masm.jmp(); 906 if (label->bound()) { 907 // The jump can be immediately patched to the correct destination. 908 masm.linkJump(j, JmpDst(label->offset())); 909 } else { 910 // Thread the jump list through the unpatched jump targets. 911 label->use(j.offset()); 912 } 913 return j; 914 } 915 916 public: 917 void nop() { masm.nop(); } 918 void nop(size_t n) { masm.insert_nop(n); } 919 void j(Condition cond, Label* label) { jSrc(cond, label); } 920 void jmp(Label* label) { jmpSrc(label); } 921 void j(Condition cond, RepatchLabel* label) { jSrc(cond, label); } 922 void jmp(RepatchLabel* label) { jmpSrc(label); } 923 924 void j(Condition cond, wasm::TrapDesc target) { 925 Label l; 926 j(cond, &l); 927 bindLater(&l, target); 928 } 929 void jmp(wasm::TrapDesc target) { 930 Label l; 931 jmp(&l); 932 bindLater(&l, target); 933 } 934 935 void jmp(const Operand& op) { 936 switch (op.kind()) { 937 case Operand::MEM_REG_DISP: 938 masm.jmp_m(op.disp(), op.base()); 939 break; 940 case Operand::MEM_SCALE: 941 masm.jmp_m(op.disp(), op.base(), op.index(), op.scale()); 942 break; 943 case Operand::REG: 944 masm.jmp_r(op.reg()); 945 break; 946 default: 947 MOZ_CRASH("unexpected operand kind"); 948 } 949 } 950 void cmpEAX(Label* label) { cmpSrc(label); } 951 void bind(Label* label) { 952 JmpDst dst(masm.label()); 953 if (label->used()) { 954 bool more; 955 JmpSrc jmp(label->offset()); 956 do { 957 JmpSrc next; 958 more = masm.nextJump(jmp, &next); 959 masm.linkJump(jmp, dst); 960 jmp = next; 961 } while (more); 962 } 963 label->bind(dst.offset()); 964 } 965 void bindLater(Label* label, wasm::TrapDesc target) { 966 if (label->used()) { 967 JmpSrc jmp(label->offset()); 968 do { 969 append(wasm::TrapSite(target, jmp.offset())); 970 } while (masm.nextJump(jmp, &jmp)); 971 } 972 label->reset(); 973 } 974 void bind(RepatchLabel* label) { 975 JmpDst dst(masm.label()); 976 if (label->used()) { 977 JmpSrc jmp(label->offset()); 978 masm.linkJump(jmp, dst); 979 } 980 label->bind(dst.offset()); 981 } 982 void use(CodeOffset* label) { 983 label->bind(currentOffset()); 984 } 985 uint32_t currentOffset() { 986 return masm.label().offset(); 987 } 988 989 // Re-routes pending jumps to a new label. 990 void retarget(Label* label, Label* target) { 991 if (!label->used()) 992 return; 993 bool more; 994 JmpSrc jmp(label->offset()); 995 do { 996 JmpSrc next; 997 more = masm.nextJump(jmp, &next); 998 if (target->bound()) { 999 // The jump can be immediately patched to the correct destination. 1000 masm.linkJump(jmp, JmpDst(target->offset())); 1001 } else { 1002 // Thread the jump list through the unpatched jump targets. 1003 JmpSrc prev(target->use(jmp.offset())); 1004 masm.setNextJump(jmp, prev); 1005 } 1006 jmp = JmpSrc(next.offset()); 1007 } while (more); 1008 label->reset(); 1009 } 1010 1011 static void Bind(uint8_t* raw, CodeOffset* label, const void* address) { 1012 if (label->bound()) { 1013 intptr_t offset = label->offset(); 1014 X86Encoding::SetPointer(raw + offset, address); 1015 } 1016 } 1017 1018 // See Bind and X86Encoding::setPointer. 1019 size_t labelToPatchOffset(CodeOffset label) { 1020 return label.offset() - sizeof(void*); 1021 } 1022 1023 void ret() { 1024 masm.ret(); 1025 } 1026 void retn(Imm32 n) { 1027 // Remove the size of the return address which is included in the frame. 1028 masm.ret_i(n.value - sizeof(void*)); 1029 } 1030 CodeOffset call(Label* label) { 1031 if (label->bound()) { 1032 masm.linkJump(masm.call(), JmpDst(label->offset())); 1033 } else { 1034 JmpSrc j = masm.call(); 1035 JmpSrc prev = JmpSrc(label->use(j.offset())); 1036 masm.setNextJump(j, prev); 1037 } 1038 return CodeOffset(masm.currentOffset()); 1039 } 1040 CodeOffset call(Register reg) { 1041 masm.call_r(reg.encoding()); 1042 return CodeOffset(masm.currentOffset()); 1043 } 1044 void call(const Operand& op) { 1045 switch (op.kind()) { 1046 case Operand::REG: 1047 masm.call_r(op.reg()); 1048 break; 1049 case Operand::MEM_REG_DISP: 1050 masm.call_m(op.disp(), op.base()); 1051 break; 1052 default: 1053 MOZ_CRASH("unexpected operand kind"); 1054 } 1055 } 1056 1057 CodeOffset callWithPatch() { 1058 return CodeOffset(masm.call().offset()); 1059 } 1060 1061 struct AutoPrepareForPatching : X86Encoding::AutoUnprotectAssemblerBufferRegion { 1062 explicit AutoPrepareForPatching(AssemblerX86Shared& masm) 1063 : X86Encoding::AutoUnprotectAssemblerBufferRegion(masm.masm, 0, masm.size()) 1064 {} 1065 }; 1066 1067 void patchCall(uint32_t callerOffset, uint32_t calleeOffset) { 1068 // The caller uses AutoUnprotectBuffer. 1069 unsigned char* code = masm.data(); 1070 X86Encoding::SetRel32(code + callerOffset, code + calleeOffset); 1071 } 1072 CodeOffset farJumpWithPatch() { 1073 return CodeOffset(masm.jmp().offset()); 1074 } 1075 void patchFarJump(CodeOffset farJump, uint32_t targetOffset) { 1076 // The caller uses AutoUnprotectBuffer. 1077 unsigned char* code = masm.data(); 1078 X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset); 1079 } 1080 static void repatchFarJump(uint8_t* code, uint32_t farJumpOffset, uint32_t targetOffset) { 1081 X86Encoding::SetRel32(code + farJumpOffset, code + targetOffset); 1082 } 1083 1084 CodeOffset twoByteNop() { 1085 return CodeOffset(masm.twoByteNop().offset()); 1086 } 1087 static void patchTwoByteNopToJump(uint8_t* jump, uint8_t* target) { 1088 X86Encoding::BaseAssembler::patchTwoByteNopToJump(jump, target); 1089 } 1090 static void patchJumpToTwoByteNop(uint8_t* jump) { 1091 X86Encoding::BaseAssembler::patchJumpToTwoByteNop(jump); 1092 } 1093 1094 void breakpoint() { 1095 masm.int3(); 1096 } 1097 1098 static bool HasSSE2() { return CPUInfo::IsSSE2Present(); } 1099 static bool HasSSE3() { return CPUInfo::IsSSE3Present(); } 1100 static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); } 1101 static bool HasSSE41() { return CPUInfo::IsSSE41Present(); } 1102 static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); } 1103 static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); } 1104 static bool SupportsUnalignedAccesses() { return true; } 1105 static bool SupportsSimd() { return CPUInfo::IsSSE2Present(); } 1106 static bool HasAVX() { return CPUInfo::IsAVXPresent(); } 1107 1108 void cmpl(Register rhs, Register lhs) { 1109 masm.cmpl_rr(rhs.encoding(), lhs.encoding()); 1110 } 1111 void cmpl(const Operand& rhs, Register lhs) { 1112 switch (rhs.kind()) { 1113 case Operand::REG: 1114 masm.cmpl_rr(rhs.reg(), lhs.encoding()); 1115 break; 1116 case Operand::MEM_REG_DISP: 1117 masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding()); 1118 break; 1119 case Operand::MEM_ADDRESS32: 1120 masm.cmpl_mr(rhs.address(), lhs.encoding()); 1121 break; 1122 default: 1123 MOZ_CRASH("unexpected operand kind"); 1124 } 1125 } 1126 void cmpl(Register rhs, const Operand& lhs) { 1127 switch (lhs.kind()) { 1128 case Operand::REG: 1129 masm.cmpl_rr(rhs.encoding(), lhs.reg()); 1130 break; 1131 case Operand::MEM_REG_DISP: 1132 masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base()); 1133 break; 1134 case Operand::MEM_ADDRESS32: 1135 masm.cmpl_rm(rhs.encoding(), lhs.address()); 1136 break; 1137 default: 1138 MOZ_CRASH("unexpected operand kind"); 1139 } 1140 } 1141 void cmpl(Imm32 rhs, Register lhs) { 1142 masm.cmpl_ir(rhs.value, lhs.encoding()); 1143 } 1144 void cmpl(Imm32 rhs, const Operand& lhs) { 1145 switch (lhs.kind()) { 1146 case Operand::REG: 1147 masm.cmpl_ir(rhs.value, lhs.reg()); 1148 break; 1149 case Operand::MEM_REG_DISP: 1150 masm.cmpl_im(rhs.value, lhs.disp(), lhs.base()); 1151 break; 1152 case Operand::MEM_SCALE: 1153 masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(), lhs.scale()); 1154 break; 1155 case Operand::MEM_ADDRESS32: 1156 masm.cmpl_im(rhs.value, lhs.address()); 1157 break; 1158 default: 1159 MOZ_CRASH("unexpected operand kind"); 1160 } 1161 } 1162 CodeOffset cmplWithPatch(Imm32 rhs, Register lhs) { 1163 masm.cmpl_i32r(rhs.value, lhs.encoding()); 1164 return CodeOffset(masm.currentOffset()); 1165 } 1166 void cmpw(Register rhs, Register lhs) { 1167 masm.cmpw_rr(rhs.encoding(), lhs.encoding()); 1168 } 1169 void setCC(Condition cond, Register r) { 1170 masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding()); 1171 } 1172 void testb(Register rhs, Register lhs) { 1173 MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs)); 1174 MOZ_ASSERT(AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs)); 1175 masm.testb_rr(rhs.encoding(), lhs.encoding()); 1176 } 1177 void testw(Register rhs, Register lhs) { 1178 masm.testw_rr(lhs.encoding(), rhs.encoding()); 1179 } 1180 void testl(Register rhs, Register lhs) { 1181 masm.testl_rr(lhs.encoding(), rhs.encoding()); 1182 } 1183 void testl(Imm32 rhs, Register lhs) { 1184 masm.testl_ir(rhs.value, lhs.encoding()); 1185 } 1186 void testl(Imm32 rhs, const Operand& lhs) { 1187 switch (lhs.kind()) { 1188 case Operand::REG: 1189 masm.testl_ir(rhs.value, lhs.reg()); 1190 break; 1191 case Operand::MEM_REG_DISP: 1192 masm.testl_i32m(rhs.value, lhs.disp(), lhs.base()); 1193 break; 1194 case Operand::MEM_ADDRESS32: 1195 masm.testl_i32m(rhs.value, lhs.address()); 1196 break; 1197 default: 1198 MOZ_CRASH("unexpected operand kind"); 1199 break; 1200 } 1201 } 1202 1203 void addl(Imm32 imm, Register dest) { 1204 masm.addl_ir(imm.value, dest.encoding()); 1205 } 1206 CodeOffset addlWithPatch(Imm32 imm, Register dest) { 1207 masm.addl_i32r(imm.value, dest.encoding()); 1208 return CodeOffset(masm.currentOffset()); 1209 } 1210 void addl(Imm32 imm, const Operand& op) { 1211 switch (op.kind()) { 1212 case Operand::REG: 1213 masm.addl_ir(imm.value, op.reg()); 1214 break; 1215 case Operand::MEM_REG_DISP: 1216 masm.addl_im(imm.value, op.disp(), op.base()); 1217 break; 1218 case Operand::MEM_ADDRESS32: 1219 masm.addl_im(imm.value, op.address()); 1220 break; 1221 case Operand::MEM_SCALE: 1222 masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1223 break; 1224 default: 1225 MOZ_CRASH("unexpected operand kind"); 1226 } 1227 } 1228 void addw(Imm32 imm, const Operand& op) { 1229 switch (op.kind()) { 1230 case Operand::REG: 1231 masm.addw_ir(imm.value, op.reg()); 1232 break; 1233 case Operand::MEM_REG_DISP: 1234 masm.addw_im(imm.value, op.disp(), op.base()); 1235 break; 1236 case Operand::MEM_ADDRESS32: 1237 masm.addw_im(imm.value, op.address()); 1238 break; 1239 case Operand::MEM_SCALE: 1240 masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1241 break; 1242 default: 1243 MOZ_CRASH("unexpected operand kind"); 1244 } 1245 } 1246 void subl(Imm32 imm, Register dest) { 1247 masm.subl_ir(imm.value, dest.encoding()); 1248 } 1249 void subl(Imm32 imm, const Operand& op) { 1250 switch (op.kind()) { 1251 case Operand::REG: 1252 masm.subl_ir(imm.value, op.reg()); 1253 break; 1254 case Operand::MEM_REG_DISP: 1255 masm.subl_im(imm.value, op.disp(), op.base()); 1256 break; 1257 case Operand::MEM_SCALE: 1258 masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1259 break; 1260 default: 1261 MOZ_CRASH("unexpected operand kind"); 1262 } 1263 } 1264 void subw(Imm32 imm, const Operand& op) { 1265 switch (op.kind()) { 1266 case Operand::REG: 1267 masm.subw_ir(imm.value, op.reg()); 1268 break; 1269 case Operand::MEM_REG_DISP: 1270 masm.subw_im(imm.value, op.disp(), op.base()); 1271 break; 1272 case Operand::MEM_SCALE: 1273 masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1274 break; 1275 default: 1276 MOZ_CRASH("unexpected operand kind"); 1277 } 1278 } 1279 void addl(Register src, Register dest) { 1280 masm.addl_rr(src.encoding(), dest.encoding()); 1281 } 1282 void addl(Register src, const Operand& dest) { 1283 switch (dest.kind()) { 1284 case Operand::REG: 1285 masm.addl_rr(src.encoding(), dest.reg()); 1286 break; 1287 case Operand::MEM_REG_DISP: 1288 masm.addl_rm(src.encoding(), dest.disp(), dest.base()); 1289 break; 1290 case Operand::MEM_SCALE: 1291 masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1292 break; 1293 default: 1294 MOZ_CRASH("unexpected operand kind"); 1295 } 1296 } 1297 void addw(Register src, const Operand& dest) { 1298 switch (dest.kind()) { 1299 case Operand::REG: 1300 masm.addw_rr(src.encoding(), dest.reg()); 1301 break; 1302 case Operand::MEM_REG_DISP: 1303 masm.addw_rm(src.encoding(), dest.disp(), dest.base()); 1304 break; 1305 case Operand::MEM_SCALE: 1306 masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1307 break; 1308 default: 1309 MOZ_CRASH("unexpected operand kind"); 1310 } 1311 } 1312 void subl(Register src, Register dest) { 1313 masm.subl_rr(src.encoding(), dest.encoding()); 1314 } 1315 void subl(const Operand& src, Register dest) { 1316 switch (src.kind()) { 1317 case Operand::REG: 1318 masm.subl_rr(src.reg(), dest.encoding()); 1319 break; 1320 case Operand::MEM_REG_DISP: 1321 masm.subl_mr(src.disp(), src.base(), dest.encoding()); 1322 break; 1323 default: 1324 MOZ_CRASH("unexpected operand kind"); 1325 } 1326 } 1327 void subl(Register src, const Operand& dest) { 1328 switch (dest.kind()) { 1329 case Operand::REG: 1330 masm.subl_rr(src.encoding(), dest.reg()); 1331 break; 1332 case Operand::MEM_REG_DISP: 1333 masm.subl_rm(src.encoding(), dest.disp(), dest.base()); 1334 break; 1335 case Operand::MEM_SCALE: 1336 masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1337 break; 1338 default: 1339 MOZ_CRASH("unexpected operand kind"); 1340 } 1341 } 1342 void subw(Register src, const Operand& dest) { 1343 switch (dest.kind()) { 1344 case Operand::REG: 1345 masm.subw_rr(src.encoding(), dest.reg()); 1346 break; 1347 case Operand::MEM_REG_DISP: 1348 masm.subw_rm(src.encoding(), dest.disp(), dest.base()); 1349 break; 1350 case Operand::MEM_SCALE: 1351 masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1352 break; 1353 default: 1354 MOZ_CRASH("unexpected operand kind"); 1355 } 1356 } 1357 void orl(Register reg, Register dest) { 1358 masm.orl_rr(reg.encoding(), dest.encoding()); 1359 } 1360 void orl(Register src, const Operand& dest) { 1361 switch (dest.kind()) { 1362 case Operand::REG: 1363 masm.orl_rr(src.encoding(), dest.reg()); 1364 break; 1365 case Operand::MEM_REG_DISP: 1366 masm.orl_rm(src.encoding(), dest.disp(), dest.base()); 1367 break; 1368 case Operand::MEM_SCALE: 1369 masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1370 break; 1371 default: 1372 MOZ_CRASH("unexpected operand kind"); 1373 } 1374 } 1375 void orw(Register src, const Operand& dest) { 1376 switch (dest.kind()) { 1377 case Operand::REG: 1378 masm.orw_rr(src.encoding(), dest.reg()); 1379 break; 1380 case Operand::MEM_REG_DISP: 1381 masm.orw_rm(src.encoding(), dest.disp(), dest.base()); 1382 break; 1383 case Operand::MEM_SCALE: 1384 masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1385 break; 1386 default: 1387 MOZ_CRASH("unexpected operand kind"); 1388 } 1389 } 1390 void orl(Imm32 imm, Register reg) { 1391 masm.orl_ir(imm.value, reg.encoding()); 1392 } 1393 void orl(Imm32 imm, const Operand& op) { 1394 switch (op.kind()) { 1395 case Operand::REG: 1396 masm.orl_ir(imm.value, op.reg()); 1397 break; 1398 case Operand::MEM_REG_DISP: 1399 masm.orl_im(imm.value, op.disp(), op.base()); 1400 break; 1401 case Operand::MEM_SCALE: 1402 masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1403 break; 1404 default: 1405 MOZ_CRASH("unexpected operand kind"); 1406 } 1407 } 1408 void orw(Imm32 imm, const Operand& op) { 1409 switch (op.kind()) { 1410 case Operand::REG: 1411 masm.orw_ir(imm.value, op.reg()); 1412 break; 1413 case Operand::MEM_REG_DISP: 1414 masm.orw_im(imm.value, op.disp(), op.base()); 1415 break; 1416 case Operand::MEM_SCALE: 1417 masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1418 break; 1419 default: 1420 MOZ_CRASH("unexpected operand kind"); 1421 } 1422 } 1423 void xorl(Register src, Register dest) { 1424 masm.xorl_rr(src.encoding(), dest.encoding()); 1425 } 1426 void xorl(Register src, const Operand& dest) { 1427 switch (dest.kind()) { 1428 case Operand::REG: 1429 masm.xorl_rr(src.encoding(), dest.reg()); 1430 break; 1431 case Operand::MEM_REG_DISP: 1432 masm.xorl_rm(src.encoding(), dest.disp(), dest.base()); 1433 break; 1434 case Operand::MEM_SCALE: 1435 masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1436 break; 1437 default: 1438 MOZ_CRASH("unexpected operand kind"); 1439 } 1440 } 1441 void xorw(Register src, const Operand& dest) { 1442 switch (dest.kind()) { 1443 case Operand::REG: 1444 masm.xorw_rr(src.encoding(), dest.reg()); 1445 break; 1446 case Operand::MEM_REG_DISP: 1447 masm.xorw_rm(src.encoding(), dest.disp(), dest.base()); 1448 break; 1449 case Operand::MEM_SCALE: 1450 masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1451 break; 1452 default: 1453 MOZ_CRASH("unexpected operand kind"); 1454 } 1455 } 1456 void xorl(Imm32 imm, Register reg) { 1457 masm.xorl_ir(imm.value, reg.encoding()); 1458 } 1459 void xorl(Imm32 imm, const Operand& op) { 1460 switch (op.kind()) { 1461 case Operand::REG: 1462 masm.xorl_ir(imm.value, op.reg()); 1463 break; 1464 case Operand::MEM_REG_DISP: 1465 masm.xorl_im(imm.value, op.disp(), op.base()); 1466 break; 1467 case Operand::MEM_SCALE: 1468 masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1469 break; 1470 default: 1471 MOZ_CRASH("unexpected operand kind"); 1472 } 1473 } 1474 void xorw(Imm32 imm, const Operand& op) { 1475 switch (op.kind()) { 1476 case Operand::REG: 1477 masm.xorw_ir(imm.value, op.reg()); 1478 break; 1479 case Operand::MEM_REG_DISP: 1480 masm.xorw_im(imm.value, op.disp(), op.base()); 1481 break; 1482 case Operand::MEM_SCALE: 1483 masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1484 break; 1485 default: 1486 MOZ_CRASH("unexpected operand kind"); 1487 } 1488 } 1489 void andl(Register src, Register dest) { 1490 masm.andl_rr(src.encoding(), dest.encoding()); 1491 } 1492 void andl(Register src, const Operand& dest) { 1493 switch (dest.kind()) { 1494 case Operand::REG: 1495 masm.andl_rr(src.encoding(), dest.reg()); 1496 break; 1497 case Operand::MEM_REG_DISP: 1498 masm.andl_rm(src.encoding(), dest.disp(), dest.base()); 1499 break; 1500 case Operand::MEM_SCALE: 1501 masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1502 break; 1503 default: 1504 MOZ_CRASH("unexpected operand kind"); 1505 } 1506 } 1507 void andw(Register src, const Operand& dest) { 1508 switch (dest.kind()) { 1509 case Operand::REG: 1510 masm.andw_rr(src.encoding(), dest.reg()); 1511 break; 1512 case Operand::MEM_REG_DISP: 1513 masm.andw_rm(src.encoding(), dest.disp(), dest.base()); 1514 break; 1515 case Operand::MEM_SCALE: 1516 masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 1517 break; 1518 default: 1519 MOZ_CRASH("unexpected operand kind"); 1520 } 1521 } 1522 void andl(Imm32 imm, Register dest) { 1523 masm.andl_ir(imm.value, dest.encoding()); 1524 } 1525 void andl(Imm32 imm, const Operand& op) { 1526 switch (op.kind()) { 1527 case Operand::REG: 1528 masm.andl_ir(imm.value, op.reg()); 1529 break; 1530 case Operand::MEM_REG_DISP: 1531 masm.andl_im(imm.value, op.disp(), op.base()); 1532 break; 1533 case Operand::MEM_SCALE: 1534 masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1535 break; 1536 default: 1537 MOZ_CRASH("unexpected operand kind"); 1538 } 1539 } 1540 void andw(Imm32 imm, const Operand& op) { 1541 switch (op.kind()) { 1542 case Operand::REG: 1543 masm.andw_ir(imm.value, op.reg()); 1544 break; 1545 case Operand::MEM_REG_DISP: 1546 masm.andw_im(imm.value, op.disp(), op.base()); 1547 break; 1548 case Operand::MEM_SCALE: 1549 masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1550 break; 1551 default: 1552 MOZ_CRASH("unexpected operand kind"); 1553 } 1554 } 1555 void addl(const Operand& src, Register dest) { 1556 switch (src.kind()) { 1557 case Operand::REG: 1558 masm.addl_rr(src.reg(), dest.encoding()); 1559 break; 1560 case Operand::MEM_REG_DISP: 1561 masm.addl_mr(src.disp(), src.base(), dest.encoding()); 1562 break; 1563 default: 1564 MOZ_CRASH("unexpected operand kind"); 1565 } 1566 } 1567 void orl(const Operand& src, Register dest) { 1568 switch (src.kind()) { 1569 case Operand::REG: 1570 masm.orl_rr(src.reg(), dest.encoding()); 1571 break; 1572 case Operand::MEM_REG_DISP: 1573 masm.orl_mr(src.disp(), src.base(), dest.encoding()); 1574 break; 1575 default: 1576 MOZ_CRASH("unexpected operand kind"); 1577 } 1578 } 1579 void xorl(const Operand& src, Register dest) { 1580 switch (src.kind()) { 1581 case Operand::REG: 1582 masm.xorl_rr(src.reg(), dest.encoding()); 1583 break; 1584 case Operand::MEM_REG_DISP: 1585 masm.xorl_mr(src.disp(), src.base(), dest.encoding()); 1586 break; 1587 default: 1588 MOZ_CRASH("unexpected operand kind"); 1589 } 1590 } 1591 void andl(const Operand& src, Register dest) { 1592 switch (src.kind()) { 1593 case Operand::REG: 1594 masm.andl_rr(src.reg(), dest.encoding()); 1595 break; 1596 case Operand::MEM_REG_DISP: 1597 masm.andl_mr(src.disp(), src.base(), dest.encoding()); 1598 break; 1599 default: 1600 MOZ_CRASH("unexpected operand kind"); 1601 } 1602 } 1603 void bsrl(const Register& src, const Register& dest) { 1604 masm.bsrl_rr(src.encoding(), dest.encoding()); 1605 } 1606 void bsfl(const Register& src, const Register& dest) { 1607 masm.bsfl_rr(src.encoding(), dest.encoding()); 1608 } 1609 void popcntl(const Register& src, const Register& dest) { 1610 masm.popcntl_rr(src.encoding(), dest.encoding()); 1611 } 1612 void imull(Register multiplier) { 1613 masm.imull_r(multiplier.encoding()); 1614 } 1615 void umull(Register multiplier) { 1616 masm.mull_r(multiplier.encoding()); 1617 } 1618 void imull(Imm32 imm, Register dest) { 1619 masm.imull_ir(imm.value, dest.encoding(), dest.encoding()); 1620 } 1621 void imull(Register src, Register dest) { 1622 masm.imull_rr(src.encoding(), dest.encoding()); 1623 } 1624 void imull(Imm32 imm, Register src, Register dest) { 1625 masm.imull_ir(imm.value, src.encoding(), dest.encoding()); 1626 } 1627 void imull(const Operand& src, Register dest) { 1628 switch (src.kind()) { 1629 case Operand::REG: 1630 masm.imull_rr(src.reg(), dest.encoding()); 1631 break; 1632 case Operand::MEM_REG_DISP: 1633 masm.imull_mr(src.disp(), src.base(), dest.encoding()); 1634 break; 1635 default: 1636 MOZ_CRASH("unexpected operand kind"); 1637 } 1638 } 1639 void negl(const Operand& src) { 1640 switch (src.kind()) { 1641 case Operand::REG: 1642 masm.negl_r(src.reg()); 1643 break; 1644 case Operand::MEM_REG_DISP: 1645 masm.negl_m(src.disp(), src.base()); 1646 break; 1647 default: 1648 MOZ_CRASH("unexpected operand kind"); 1649 } 1650 } 1651 void negl(Register reg) { 1652 masm.negl_r(reg.encoding()); 1653 } 1654 void notl(const Operand& src) { 1655 switch (src.kind()) { 1656 case Operand::REG: 1657 masm.notl_r(src.reg()); 1658 break; 1659 case Operand::MEM_REG_DISP: 1660 masm.notl_m(src.disp(), src.base()); 1661 break; 1662 default: 1663 MOZ_CRASH("unexpected operand kind"); 1664 } 1665 } 1666 void notl(Register reg) { 1667 masm.notl_r(reg.encoding()); 1668 } 1669 void shrl(const Imm32 imm, Register dest) { 1670 masm.shrl_ir(imm.value, dest.encoding()); 1671 } 1672 void shll(const Imm32 imm, Register dest) { 1673 masm.shll_ir(imm.value, dest.encoding()); 1674 } 1675 void sarl(const Imm32 imm, Register dest) { 1676 masm.sarl_ir(imm.value, dest.encoding()); 1677 } 1678 void shrl_cl(Register dest) { 1679 masm.shrl_CLr(dest.encoding()); 1680 } 1681 void shll_cl(Register dest) { 1682 masm.shll_CLr(dest.encoding()); 1683 } 1684 void sarl_cl(Register dest) { 1685 masm.sarl_CLr(dest.encoding()); 1686 } 1687 void shrdl_cl(Register src, Register dest) { 1688 masm.shrdl_CLr(src.encoding(), dest.encoding()); 1689 } 1690 void shldl_cl(Register src, Register dest) { 1691 masm.shldl_CLr(src.encoding(), dest.encoding()); 1692 } 1693 1694 void roll(const Imm32 imm, Register dest) { 1695 masm.roll_ir(imm.value, dest.encoding()); 1696 } 1697 void roll_cl(Register dest) { 1698 masm.roll_CLr(dest.encoding()); 1699 } 1700 void rorl(const Imm32 imm, Register dest) { 1701 masm.rorl_ir(imm.value, dest.encoding()); 1702 } 1703 void rorl_cl(Register dest) { 1704 masm.rorl_CLr(dest.encoding()); 1705 } 1706 1707 void incl(const Operand& op) { 1708 switch (op.kind()) { 1709 case Operand::MEM_REG_DISP: 1710 masm.incl_m32(op.disp(), op.base()); 1711 break; 1712 default: 1713 MOZ_CRASH("unexpected operand kind"); 1714 } 1715 } 1716 void lock_incl(const Operand& op) { 1717 masm.prefix_lock(); 1718 incl(op); 1719 } 1720 1721 void decl(const Operand& op) { 1722 switch (op.kind()) { 1723 case Operand::MEM_REG_DISP: 1724 masm.decl_m32(op.disp(), op.base()); 1725 break; 1726 default: 1727 MOZ_CRASH("unexpected operand kind"); 1728 } 1729 } 1730 void lock_decl(const Operand& op) { 1731 masm.prefix_lock(); 1732 decl(op); 1733 } 1734 1735 void addb(Imm32 imm, const Operand& op) { 1736 switch (op.kind()) { 1737 case Operand::MEM_REG_DISP: 1738 masm.addb_im(imm.value, op.disp(), op.base()); 1739 break; 1740 case Operand::MEM_SCALE: 1741 masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1742 break; 1743 default: 1744 MOZ_CRASH("unexpected operand kind"); 1745 break; 1746 } 1747 } 1748 void addb(Register src, const Operand& op) { 1749 switch (op.kind()) { 1750 case Operand::MEM_REG_DISP: 1751 masm.addb_rm(src.encoding(), op.disp(), op.base()); 1752 break; 1753 case Operand::MEM_SCALE: 1754 masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); 1755 break; 1756 default: 1757 MOZ_CRASH("unexpected operand kind"); 1758 break; 1759 } 1760 } 1761 1762 void subb(Imm32 imm, const Operand& op) { 1763 switch (op.kind()) { 1764 case Operand::MEM_REG_DISP: 1765 masm.subb_im(imm.value, op.disp(), op.base()); 1766 break; 1767 case Operand::MEM_SCALE: 1768 masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1769 break; 1770 default: 1771 MOZ_CRASH("unexpected operand kind"); 1772 break; 1773 } 1774 } 1775 void subb(Register src, const Operand& op) { 1776 switch (op.kind()) { 1777 case Operand::MEM_REG_DISP: 1778 masm.subb_rm(src.encoding(), op.disp(), op.base()); 1779 break; 1780 case Operand::MEM_SCALE: 1781 masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); 1782 break; 1783 default: 1784 MOZ_CRASH("unexpected operand kind"); 1785 break; 1786 } 1787 } 1788 1789 void andb(Imm32 imm, const Operand& op) { 1790 switch (op.kind()) { 1791 case Operand::MEM_REG_DISP: 1792 masm.andb_im(imm.value, op.disp(), op.base()); 1793 break; 1794 case Operand::MEM_SCALE: 1795 masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1796 break; 1797 default: 1798 MOZ_CRASH("unexpected operand kind"); 1799 break; 1800 } 1801 } 1802 void andb(Register src, const Operand& op) { 1803 switch (op.kind()) { 1804 case Operand::MEM_REG_DISP: 1805 masm.andb_rm(src.encoding(), op.disp(), op.base()); 1806 break; 1807 case Operand::MEM_SCALE: 1808 masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); 1809 break; 1810 default: 1811 MOZ_CRASH("unexpected operand kind"); 1812 break; 1813 } 1814 } 1815 1816 void orb(Imm32 imm, const Operand& op) { 1817 switch (op.kind()) { 1818 case Operand::MEM_REG_DISP: 1819 masm.orb_im(imm.value, op.disp(), op.base()); 1820 break; 1821 case Operand::MEM_SCALE: 1822 masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1823 break; 1824 default: 1825 MOZ_CRASH("unexpected operand kind"); 1826 break; 1827 } 1828 } 1829 void orb(Register src, const Operand& op) { 1830 switch (op.kind()) { 1831 case Operand::MEM_REG_DISP: 1832 masm.orb_rm(src.encoding(), op.disp(), op.base()); 1833 break; 1834 case Operand::MEM_SCALE: 1835 masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); 1836 break; 1837 default: 1838 MOZ_CRASH("unexpected operand kind"); 1839 break; 1840 } 1841 } 1842 1843 void xorb(Imm32 imm, const Operand& op) { 1844 switch (op.kind()) { 1845 case Operand::MEM_REG_DISP: 1846 masm.xorb_im(imm.value, op.disp(), op.base()); 1847 break; 1848 case Operand::MEM_SCALE: 1849 masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale()); 1850 break; 1851 default: 1852 MOZ_CRASH("unexpected operand kind"); 1853 break; 1854 } 1855 } 1856 void xorb(Register src, const Operand& op) { 1857 switch (op.kind()) { 1858 case Operand::MEM_REG_DISP: 1859 masm.xorb_rm(src.encoding(), op.disp(), op.base()); 1860 break; 1861 case Operand::MEM_SCALE: 1862 masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(), op.scale()); 1863 break; 1864 default: 1865 MOZ_CRASH("unexpected operand kind"); 1866 break; 1867 } 1868 } 1869 1870 template<typename T> 1871 void lock_addb(T src, const Operand& op) { 1872 masm.prefix_lock(); 1873 addb(src, op); 1874 } 1875 template<typename T> 1876 void lock_subb(T src, const Operand& op) { 1877 masm.prefix_lock(); 1878 subb(src, op); 1879 } 1880 template<typename T> 1881 void lock_andb(T src, const Operand& op) { 1882 masm.prefix_lock(); 1883 andb(src, op); 1884 } 1885 template<typename T> 1886 void lock_orb(T src, const Operand& op) { 1887 masm.prefix_lock(); 1888 orb(src, op); 1889 } 1890 template<typename T> 1891 void lock_xorb(T src, const Operand& op) { 1892 masm.prefix_lock(); 1893 xorb(src, op); 1894 } 1895 1896 template<typename T> 1897 void lock_addw(T src, const Operand& op) { 1898 masm.prefix_lock(); 1899 addw(src, op); 1900 } 1901 template<typename T> 1902 void lock_subw(T src, const Operand& op) { 1903 masm.prefix_lock(); 1904 subw(src, op); 1905 } 1906 template<typename T> 1907 void lock_andw(T src, const Operand& op) { 1908 masm.prefix_lock(); 1909 andw(src, op); 1910 } 1911 template<typename T> 1912 void lock_orw(T src, const Operand& op) { 1913 masm.prefix_lock(); 1914 orw(src, op); 1915 } 1916 template<typename T> 1917 void lock_xorw(T src, const Operand& op) { 1918 masm.prefix_lock(); 1919 xorw(src, op); 1920 } 1921 1922 // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems, 1923 // among other things. Do not optimize, replace by XADDL, or similar. 1924 template<typename T> 1925 void lock_addl(T src, const Operand& op) { 1926 masm.prefix_lock(); 1927 addl(src, op); 1928 } 1929 template<typename T> 1930 void lock_subl(T src, const Operand& op) { 1931 masm.prefix_lock(); 1932 subl(src, op); 1933 } 1934 template<typename T> 1935 void lock_andl(T src, const Operand& op) { 1936 masm.prefix_lock(); 1937 andl(src, op); 1938 } 1939 template<typename T> 1940 void lock_orl(T src, const Operand& op) { 1941 masm.prefix_lock(); 1942 orl(src, op); 1943 } 1944 template<typename T> 1945 void lock_xorl(T src, const Operand& op) { 1946 masm.prefix_lock(); 1947 xorl(src, op); 1948 } 1949 1950 void lock_cmpxchgb(Register src, const Operand& mem) { 1951 masm.prefix_lock(); 1952 switch (mem.kind()) { 1953 case Operand::MEM_REG_DISP: 1954 masm.cmpxchgb(src.encoding(), mem.disp(), mem.base()); 1955 break; 1956 case Operand::MEM_SCALE: 1957 masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 1958 break; 1959 default: 1960 MOZ_CRASH("unexpected operand kind"); 1961 } 1962 } 1963 void lock_cmpxchgw(Register src, const Operand& mem) { 1964 masm.prefix_lock(); 1965 switch (mem.kind()) { 1966 case Operand::MEM_REG_DISP: 1967 masm.cmpxchgw(src.encoding(), mem.disp(), mem.base()); 1968 break; 1969 case Operand::MEM_SCALE: 1970 masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 1971 break; 1972 default: 1973 MOZ_CRASH("unexpected operand kind"); 1974 } 1975 } 1976 void lock_cmpxchgl(Register src, const Operand& mem) { 1977 masm.prefix_lock(); 1978 switch (mem.kind()) { 1979 case Operand::MEM_REG_DISP: 1980 masm.cmpxchgl(src.encoding(), mem.disp(), mem.base()); 1981 break; 1982 case Operand::MEM_SCALE: 1983 masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 1984 break; 1985 default: 1986 MOZ_CRASH("unexpected operand kind"); 1987 } 1988 } 1989 1990 void xchgb(Register src, const Operand& mem) { 1991 switch (mem.kind()) { 1992 case Operand::MEM_REG_DISP: 1993 masm.xchgb_rm(src.encoding(), mem.disp(), mem.base()); 1994 break; 1995 case Operand::MEM_SCALE: 1996 masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 1997 break; 1998 default: 1999 MOZ_CRASH("unexpected operand kind"); 2000 } 2001 } 2002 void xchgw(Register src, const Operand& mem) { 2003 switch (mem.kind()) { 2004 case Operand::MEM_REG_DISP: 2005 masm.xchgw_rm(src.encoding(), mem.disp(), mem.base()); 2006 break; 2007 case Operand::MEM_SCALE: 2008 masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 2009 break; 2010 default: 2011 MOZ_CRASH("unexpected operand kind"); 2012 } 2013 } 2014 void xchgl(Register src, const Operand& mem) { 2015 switch (mem.kind()) { 2016 case Operand::MEM_REG_DISP: 2017 masm.xchgl_rm(src.encoding(), mem.disp(), mem.base()); 2018 break; 2019 case Operand::MEM_SCALE: 2020 masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 2021 break; 2022 default: 2023 MOZ_CRASH("unexpected operand kind"); 2024 } 2025 } 2026 2027 void lock_xaddb(Register srcdest, const Operand& mem) { 2028 switch (mem.kind()) { 2029 case Operand::MEM_REG_DISP: 2030 masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base()); 2031 break; 2032 case Operand::MEM_SCALE: 2033 masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 2034 break; 2035 default: 2036 MOZ_CRASH("unexpected operand kind"); 2037 } 2038 } 2039 void lock_xaddw(Register srcdest, const Operand& mem) { 2040 masm.prefix_16_for_32(); 2041 lock_xaddl(srcdest, mem); 2042 } 2043 void lock_xaddl(Register srcdest, const Operand& mem) { 2044 switch (mem.kind()) { 2045 case Operand::MEM_REG_DISP: 2046 masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base()); 2047 break; 2048 case Operand::MEM_SCALE: 2049 masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(), mem.index(), mem.scale()); 2050 break; 2051 default: 2052 MOZ_CRASH("unexpected operand kind"); 2053 } 2054 } 2055 2056 void push(const Imm32 imm) { 2057 masm.push_i(imm.value); 2058 } 2059 2060 void push(const Operand& src) { 2061 switch (src.kind()) { 2062 case Operand::REG: 2063 masm.push_r(src.reg()); 2064 break; 2065 case Operand::MEM_REG_DISP: 2066 masm.push_m(src.disp(), src.base()); 2067 break; 2068 default: 2069 MOZ_CRASH("unexpected operand kind"); 2070 } 2071 } 2072 void push(Register src) { 2073 masm.push_r(src.encoding()); 2074 } 2075 void push(const Address& src) { 2076 masm.push_m(src.offset, src.base.encoding()); 2077 } 2078 2079 void pop(const Operand& src) { 2080 switch (src.kind()) { 2081 case Operand::REG: 2082 masm.pop_r(src.reg()); 2083 break; 2084 case Operand::MEM_REG_DISP: 2085 masm.pop_m(src.disp(), src.base()); 2086 break; 2087 default: 2088 MOZ_CRASH("unexpected operand kind"); 2089 } 2090 } 2091 void pop(Register src) { 2092 masm.pop_r(src.encoding()); 2093 } 2094 void pop(const Address& src) { 2095 masm.pop_m(src.offset, src.base.encoding()); 2096 } 2097 2098 void pushFlags() { 2099 masm.push_flags(); 2100 } 2101 void popFlags() { 2102 masm.pop_flags(); 2103 } 2104 2105 #ifdef JS_CODEGEN_X86 2106 void pushAllRegs() { 2107 masm.pusha(); 2108 } 2109 void popAllRegs() { 2110 masm.popa(); 2111 } 2112 #endif 2113 2114 // Zero-extend byte to 32-bit integer. 2115 void movzbl(Register src, Register dest) { 2116 masm.movzbl_rr(src.encoding(), dest.encoding()); 2117 } 2118 2119 void cdq() { 2120 masm.cdq(); 2121 } 2122 void idiv(Register divisor) { 2123 masm.idivl_r(divisor.encoding()); 2124 } 2125 void udiv(Register divisor) { 2126 masm.divl_r(divisor.encoding()); 2127 } 2128 2129 void vpinsrb(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { 2130 MOZ_ASSERT(HasSSE41()); 2131 masm.vpinsrb_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); 2132 } 2133 void vpinsrw(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { 2134 masm.vpinsrw_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); 2135 } 2136 2137 void vpinsrd(unsigned lane, Register src1, FloatRegister src0, FloatRegister dest) { 2138 MOZ_ASSERT(HasSSE41()); 2139 masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding()); 2140 } 2141 2142 void vpextrb(unsigned lane, FloatRegister src, Register dest) { 2143 MOZ_ASSERT(HasSSE41()); 2144 masm.vpextrb_irr(lane, src.encoding(), dest.encoding()); 2145 } 2146 void vpextrw(unsigned lane, FloatRegister src, Register dest) { 2147 masm.vpextrw_irr(lane, src.encoding(), dest.encoding()); 2148 } 2149 void vpextrd(unsigned lane, FloatRegister src, Register dest) { 2150 MOZ_ASSERT(HasSSE41()); 2151 masm.vpextrd_irr(lane, src.encoding(), dest.encoding()); 2152 } 2153 void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2154 MOZ_ASSERT(HasSSE2()); 2155 masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding()); 2156 } 2157 void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2158 MOZ_ASSERT(HasSSE2()); 2159 masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding()); 2160 } 2161 void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) { 2162 MOZ_ASSERT(HasSSE2()); 2163 masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding()); 2164 } 2165 void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2166 MOZ_ASSERT(HasSSE2()); 2167 masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2168 } 2169 void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) { 2170 MOZ_ASSERT(HasSSE2()); 2171 masm.vpslld_ir(count.value, src0.encoding(), dest.encoding()); 2172 } 2173 void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2174 MOZ_ASSERT(HasSSE2()); 2175 masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2176 } 2177 void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) { 2178 MOZ_ASSERT(HasSSE2()); 2179 masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding()); 2180 } 2181 void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2182 MOZ_ASSERT(HasSSE2()); 2183 masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2184 } 2185 void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) { 2186 MOZ_ASSERT(HasSSE2()); 2187 masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding()); 2188 } 2189 2190 void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2191 MOZ_ASSERT(HasSSE2()); 2192 masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2193 } 2194 void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2195 MOZ_ASSERT(HasSSE2()); 2196 masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding()); 2197 } 2198 void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2199 MOZ_ASSERT(HasSSE2()); 2200 masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2201 } 2202 void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2203 MOZ_ASSERT(HasSSE2()); 2204 masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding()); 2205 } 2206 void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2207 MOZ_ASSERT(HasSSE2()); 2208 masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2209 } 2210 void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) { 2211 MOZ_ASSERT(HasSSE2()); 2212 masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding()); 2213 } 2214 2215 void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2216 MOZ_ASSERT(HasSSE2()); 2217 switch (src1.kind()) { 2218 case Operand::REG: 2219 masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding()); 2220 break; 2221 case Operand::MEM_REG_DISP: 2222 masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2223 break; 2224 case Operand::MEM_SCALE: 2225 masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); 2226 break; 2227 default: 2228 MOZ_CRASH("unexpected operand kind"); 2229 } 2230 } 2231 void vcvttsd2si(FloatRegister src, Register dest) { 2232 MOZ_ASSERT(HasSSE2()); 2233 masm.vcvttsd2si_rr(src.encoding(), dest.encoding()); 2234 } 2235 void vcvttss2si(FloatRegister src, Register dest) { 2236 MOZ_ASSERT(HasSSE2()); 2237 masm.vcvttss2si_rr(src.encoding(), dest.encoding()); 2238 } 2239 void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2240 MOZ_ASSERT(HasSSE2()); 2241 switch (src1.kind()) { 2242 case Operand::REG: 2243 masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding()); 2244 break; 2245 case Operand::MEM_REG_DISP: 2246 masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2247 break; 2248 case Operand::MEM_SCALE: 2249 masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(), src0.encoding(), dest.encoding()); 2250 break; 2251 default: 2252 MOZ_CRASH("unexpected operand kind"); 2253 } 2254 } 2255 void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) { 2256 MOZ_ASSERT(HasSSE2()); 2257 masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2258 } 2259 void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) { 2260 MOZ_ASSERT(HasSSE2()); 2261 masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2262 } 2263 void vcvttps2dq(FloatRegister src, FloatRegister dest) { 2264 MOZ_ASSERT(HasSSE2()); 2265 masm.vcvttps2dq_rr(src.encoding(), dest.encoding()); 2266 } 2267 void vcvtdq2ps(FloatRegister src, FloatRegister dest) { 2268 MOZ_ASSERT(HasSSE2()); 2269 masm.vcvtdq2ps_rr(src.encoding(), dest.encoding()); 2270 } 2271 void vmovmskpd(FloatRegister src, Register dest) { 2272 MOZ_ASSERT(HasSSE2()); 2273 masm.vmovmskpd_rr(src.encoding(), dest.encoding()); 2274 } 2275 void vmovmskps(FloatRegister src, Register dest) { 2276 MOZ_ASSERT(HasSSE2()); 2277 masm.vmovmskps_rr(src.encoding(), dest.encoding()); 2278 } 2279 void vptest(FloatRegister rhs, FloatRegister lhs) { 2280 MOZ_ASSERT(HasSSE41()); 2281 masm.vptest_rr(rhs.encoding(), lhs.encoding()); 2282 } 2283 void vucomisd(FloatRegister rhs, FloatRegister lhs) { 2284 MOZ_ASSERT(HasSSE2()); 2285 masm.vucomisd_rr(rhs.encoding(), lhs.encoding()); 2286 } 2287 void vucomiss(FloatRegister rhs, FloatRegister lhs) { 2288 MOZ_ASSERT(HasSSE2()); 2289 masm.vucomiss_rr(rhs.encoding(), lhs.encoding()); 2290 } 2291 2292 void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2293 MOZ_ASSERT(HasSSE2()); 2294 switch (rhs.kind()) { 2295 case Operand::FPREG: 2296 masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2297 break; 2298 case Operand::MEM_REG_DISP: 2299 masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2300 break; 2301 case Operand::MEM_ADDRESS32: 2302 masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2303 break; 2304 default: 2305 MOZ_CRASH("unexpected operand kind"); 2306 } 2307 } 2308 void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2309 MOZ_ASSERT(HasSSE2()); 2310 switch (rhs.kind()) { 2311 case Operand::FPREG: 2312 masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2313 break; 2314 case Operand::MEM_REG_DISP: 2315 masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2316 break; 2317 case Operand::MEM_ADDRESS32: 2318 masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2319 break; 2320 default: 2321 MOZ_CRASH("unexpected operand kind"); 2322 } 2323 } 2324 2325 void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2326 MOZ_ASSERT(HasSSE2()); 2327 switch (rhs.kind()) { 2328 case Operand::FPREG: 2329 masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2330 break; 2331 case Operand::MEM_REG_DISP: 2332 masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2333 break; 2334 case Operand::MEM_ADDRESS32: 2335 masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2336 break; 2337 default: 2338 MOZ_CRASH("unexpected operand kind"); 2339 } 2340 } 2341 void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2342 MOZ_ASSERT(HasSSE2()); 2343 switch (rhs.kind()) { 2344 case Operand::FPREG: 2345 masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2346 break; 2347 case Operand::MEM_REG_DISP: 2348 masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2349 break; 2350 case Operand::MEM_ADDRESS32: 2351 masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2352 break; 2353 default: 2354 MOZ_CRASH("unexpected operand kind"); 2355 } 2356 } 2357 2358 void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2359 MOZ_ASSERT(HasSSE2()); 2360 switch (rhs.kind()) { 2361 case Operand::FPREG: 2362 masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2363 break; 2364 case Operand::MEM_REG_DISP: 2365 masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2366 break; 2367 case Operand::MEM_ADDRESS32: 2368 masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2369 break; 2370 default: 2371 MOZ_CRASH("unexpected operand kind"); 2372 } 2373 } 2374 void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) { 2375 MOZ_ASSERT(HasSSE2()); 2376 switch (rhs.kind()) { 2377 case Operand::FPREG: 2378 masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding()); 2379 break; 2380 case Operand::MEM_REG_DISP: 2381 masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(), dest.encoding()); 2382 break; 2383 case Operand::MEM_ADDRESS32: 2384 masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding()); 2385 break; 2386 default: 2387 MOZ_CRASH("unexpected operand kind"); 2388 } 2389 } 2390 2391 void vcmpps(uint8_t order, Operand src1, FloatRegister src0, FloatRegister dest) { 2392 MOZ_ASSERT(HasSSE2()); 2393 // :TODO: (Bug 1132894) See LIRGeneratorX86Shared::lowerForFPU 2394 // FIXME: This logic belongs in the MacroAssembler. 2395 if (!HasAVX() && !src0.aliases(dest)) { 2396 if (src1.kind() == Operand::FPREG && 2397 dest.aliases(FloatRegister::FromCode(src1.fpu()))) 2398 { 2399 vmovdqa(src1, ScratchSimd128Reg); 2400 src1 = Operand(ScratchSimd128Reg); 2401 } 2402 vmovdqa(src0, dest); 2403 src0 = dest; 2404 } 2405 switch (src1.kind()) { 2406 case Operand::FPREG: 2407 masm.vcmpps_rr(order, src1.fpu(), src0.encoding(), dest.encoding()); 2408 break; 2409 case Operand::MEM_REG_DISP: 2410 masm.vcmpps_mr(order, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2411 break; 2412 case Operand::MEM_ADDRESS32: 2413 masm.vcmpps_mr(order, src1.address(), src0.encoding(), dest.encoding()); 2414 break; 2415 default: 2416 MOZ_CRASH("unexpected operand kind"); 2417 } 2418 } 2419 void vcmpeqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2420 vcmpps(X86Encoding::ConditionCmp_EQ, src1, src0, dest); 2421 } 2422 void vcmpltps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2423 vcmpps(X86Encoding::ConditionCmp_LT, src1, src0, dest); 2424 } 2425 void vcmpleps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2426 vcmpps(X86Encoding::ConditionCmp_LE, src1, src0, dest); 2427 } 2428 void vcmpunordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2429 vcmpps(X86Encoding::ConditionCmp_UNORD, src1, src0, dest); 2430 } 2431 void vcmpneqps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2432 vcmpps(X86Encoding::ConditionCmp_NEQ, src1, src0, dest); 2433 } 2434 void vcmpordps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2435 vcmpps(X86Encoding::ConditionCmp_ORD, src1, src0, dest); 2436 } 2437 void vrcpps(const Operand& src, FloatRegister dest) { 2438 MOZ_ASSERT(HasSSE2()); 2439 switch (src.kind()) { 2440 case Operand::FPREG: 2441 masm.vrcpps_rr(src.fpu(), dest.encoding()); 2442 break; 2443 case Operand::MEM_REG_DISP: 2444 masm.vrcpps_mr(src.disp(), src.base(), dest.encoding()); 2445 break; 2446 case Operand::MEM_ADDRESS32: 2447 masm.vrcpps_mr(src.address(), dest.encoding()); 2448 break; 2449 default: 2450 MOZ_CRASH("unexpected operand kind"); 2451 } 2452 } 2453 void vsqrtps(const Operand& src, FloatRegister dest) { 2454 MOZ_ASSERT(HasSSE2()); 2455 switch (src.kind()) { 2456 case Operand::FPREG: 2457 masm.vsqrtps_rr(src.fpu(), dest.encoding()); 2458 break; 2459 case Operand::MEM_REG_DISP: 2460 masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding()); 2461 break; 2462 case Operand::MEM_ADDRESS32: 2463 masm.vsqrtps_mr(src.address(), dest.encoding()); 2464 break; 2465 default: 2466 MOZ_CRASH("unexpected operand kind"); 2467 } 2468 } 2469 void vrsqrtps(const Operand& src, FloatRegister dest) { 2470 MOZ_ASSERT(HasSSE2()); 2471 switch (src.kind()) { 2472 case Operand::FPREG: 2473 masm.vrsqrtps_rr(src.fpu(), dest.encoding()); 2474 break; 2475 case Operand::MEM_REG_DISP: 2476 masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding()); 2477 break; 2478 case Operand::MEM_ADDRESS32: 2479 masm.vrsqrtps_mr(src.address(), dest.encoding()); 2480 break; 2481 default: 2482 MOZ_CRASH("unexpected operand kind"); 2483 } 2484 } 2485 void vmovd(Register src, FloatRegister dest) { 2486 MOZ_ASSERT(HasSSE2()); 2487 masm.vmovd_rr(src.encoding(), dest.encoding()); 2488 } 2489 void vmovd(FloatRegister src, Register dest) { 2490 MOZ_ASSERT(HasSSE2()); 2491 masm.vmovd_rr(src.encoding(), dest.encoding()); 2492 } 2493 void vmovd(const Operand& src, FloatRegister dest) { 2494 MOZ_ASSERT(HasSSE2()); 2495 switch (src.kind()) { 2496 case Operand::MEM_REG_DISP: 2497 masm.vmovd_mr(src.disp(), src.base(), dest.encoding()); 2498 break; 2499 case Operand::MEM_SCALE: 2500 masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 2501 break; 2502 default: 2503 MOZ_CRASH("unexpected operand kind"); 2504 } 2505 } 2506 void vmovd(FloatRegister src, const Operand& dest) { 2507 MOZ_ASSERT(HasSSE2()); 2508 switch (dest.kind()) { 2509 case Operand::MEM_REG_DISP: 2510 masm.vmovd_rm(src.encoding(), dest.disp(), dest.base()); 2511 break; 2512 case Operand::MEM_SCALE: 2513 masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 2514 break; 2515 case Operand::MEM_ADDRESS32: 2516 masm.vmovq_rm(src.encoding(), dest.address()); 2517 break; 2518 default: 2519 MOZ_CRASH("unexpected operand kind"); 2520 } 2521 } 2522 void vmovq(const Operand& src, FloatRegister dest) { 2523 MOZ_ASSERT(HasSSE2()); 2524 switch (src.kind()) { 2525 case Operand::MEM_REG_DISP: 2526 masm.vmovq_mr(src.disp(), src.base(), dest.encoding()); 2527 break; 2528 case Operand::MEM_SCALE: 2529 masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(), dest.encoding()); 2530 break; 2531 case Operand::MEM_ADDRESS32: 2532 masm.vmovq_mr(src.address(), dest.encoding()); 2533 break; 2534 default: 2535 MOZ_CRASH("unexpected operand kind"); 2536 } 2537 } 2538 void vmovq(FloatRegister src, const Operand& dest) { 2539 MOZ_ASSERT(HasSSE2()); 2540 switch (dest.kind()) { 2541 case Operand::MEM_REG_DISP: 2542 masm.vmovq_rm(src.encoding(), dest.disp(), dest.base()); 2543 break; 2544 case Operand::MEM_SCALE: 2545 masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(), dest.scale()); 2546 break; 2547 default: 2548 MOZ_CRASH("unexpected operand kind"); 2549 } 2550 } 2551 void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2552 MOZ_ASSERT(HasSSE2()); 2553 switch (src1.kind()) { 2554 case Operand::FPREG: 2555 masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2556 break; 2557 case Operand::MEM_REG_DISP: 2558 masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2559 break; 2560 case Operand::MEM_ADDRESS32: 2561 masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding()); 2562 break; 2563 default: 2564 MOZ_CRASH("unexpected operand kind"); 2565 } 2566 } 2567 void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2568 MOZ_ASSERT(HasSSE2()); 2569 switch (src1.kind()) { 2570 case Operand::FPREG: 2571 masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2572 break; 2573 case Operand::MEM_REG_DISP: 2574 masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2575 break; 2576 case Operand::MEM_ADDRESS32: 2577 masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding()); 2578 break; 2579 default: 2580 MOZ_CRASH("unexpected operand kind"); 2581 } 2582 } 2583 void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2584 MOZ_ASSERT(HasSSE2()); 2585 switch (src1.kind()) { 2586 case Operand::FPREG: 2587 masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2588 break; 2589 case Operand::MEM_REG_DISP: 2590 masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2591 break; 2592 case Operand::MEM_ADDRESS32: 2593 masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding()); 2594 break; 2595 default: 2596 MOZ_CRASH("unexpected operand kind"); 2597 } 2598 } 2599 void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2600 MOZ_ASSERT(HasSSE2()); 2601 switch (src1.kind()) { 2602 case Operand::FPREG: 2603 masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2604 break; 2605 case Operand::MEM_REG_DISP: 2606 masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2607 break; 2608 case Operand::MEM_ADDRESS32: 2609 masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding()); 2610 break; 2611 default: 2612 MOZ_CRASH("unexpected operand kind"); 2613 } 2614 } 2615 void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2616 MOZ_ASSERT(HasSSE2()); 2617 switch (src1.kind()) { 2618 case Operand::FPREG: 2619 masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2620 break; 2621 case Operand::MEM_REG_DISP: 2622 masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2623 break; 2624 case Operand::MEM_ADDRESS32: 2625 masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding()); 2626 break; 2627 default: 2628 MOZ_CRASH("unexpected operand kind"); 2629 } 2630 } 2631 void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2632 MOZ_ASSERT(HasSSE2()); 2633 switch (src1.kind()) { 2634 case Operand::FPREG: 2635 masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2636 break; 2637 case Operand::MEM_REG_DISP: 2638 masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2639 break; 2640 case Operand::MEM_ADDRESS32: 2641 masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding()); 2642 break; 2643 default: 2644 MOZ_CRASH("unexpected operand kind"); 2645 } 2646 } 2647 void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2648 MOZ_ASSERT(HasSSE2()); 2649 switch (src1.kind()) { 2650 case Operand::FPREG: 2651 masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2652 break; 2653 case Operand::MEM_REG_DISP: 2654 masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2655 break; 2656 case Operand::MEM_ADDRESS32: 2657 masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding()); 2658 break; 2659 default: 2660 MOZ_CRASH("unexpected operand kind"); 2661 } 2662 } 2663 void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2664 MOZ_ASSERT(HasSSE2()); 2665 switch (src1.kind()) { 2666 case Operand::FPREG: 2667 masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2668 break; 2669 case Operand::MEM_REG_DISP: 2670 masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2671 break; 2672 case Operand::MEM_ADDRESS32: 2673 masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding()); 2674 break; 2675 default: 2676 MOZ_CRASH("unexpected operand kind"); 2677 } 2678 } 2679 void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2680 MOZ_ASSERT(HasSSE2()); 2681 switch (src1.kind()) { 2682 case Operand::FPREG: 2683 masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2684 break; 2685 case Operand::MEM_REG_DISP: 2686 masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2687 break; 2688 case Operand::MEM_ADDRESS32: 2689 masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding()); 2690 break; 2691 default: 2692 MOZ_CRASH("unexpected operand kind"); 2693 } 2694 } 2695 void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2696 MOZ_ASSERT(HasSSE2()); 2697 switch (src1.kind()) { 2698 case Operand::FPREG: 2699 masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2700 break; 2701 case Operand::MEM_REG_DISP: 2702 masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2703 break; 2704 case Operand::MEM_ADDRESS32: 2705 masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding()); 2706 break; 2707 default: 2708 MOZ_CRASH("unexpected operand kind"); 2709 } 2710 } 2711 void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2712 MOZ_ASSERT(HasSSE2()); 2713 switch (src1.kind()) { 2714 case Operand::FPREG: 2715 masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2716 break; 2717 case Operand::MEM_REG_DISP: 2718 masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2719 break; 2720 case Operand::MEM_ADDRESS32: 2721 masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding()); 2722 break; 2723 default: 2724 MOZ_CRASH("unexpected operand kind"); 2725 } 2726 } 2727 void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2728 MOZ_ASSERT(HasSSE2()); 2729 switch (src1.kind()) { 2730 case Operand::FPREG: 2731 masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2732 break; 2733 case Operand::MEM_REG_DISP: 2734 masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2735 break; 2736 case Operand::MEM_ADDRESS32: 2737 masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding()); 2738 break; 2739 default: 2740 MOZ_CRASH("unexpected operand kind"); 2741 } 2742 } 2743 void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2744 MOZ_ASSERT(HasSSE2()); 2745 switch (src1.kind()) { 2746 case Operand::FPREG: 2747 masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2748 break; 2749 case Operand::MEM_REG_DISP: 2750 masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2751 break; 2752 case Operand::MEM_ADDRESS32: 2753 masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding()); 2754 break; 2755 default: 2756 MOZ_CRASH("unexpected operand kind"); 2757 } 2758 } 2759 void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2760 MOZ_ASSERT(HasSSE2()); 2761 switch (src1.kind()) { 2762 case Operand::FPREG: 2763 masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2764 break; 2765 case Operand::MEM_REG_DISP: 2766 masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2767 break; 2768 case Operand::MEM_ADDRESS32: 2769 masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding()); 2770 break; 2771 default: 2772 MOZ_CRASH("unexpected operand kind"); 2773 } 2774 } 2775 void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2776 MOZ_ASSERT(HasSSE2()); 2777 masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2778 } 2779 void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2780 MOZ_ASSERT(HasSSE2()); 2781 switch (src1.kind()) { 2782 case Operand::FPREG: 2783 masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2784 break; 2785 case Operand::MEM_REG_DISP: 2786 masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2787 break; 2788 default: 2789 MOZ_CRASH("unexpected operand kind"); 2790 } 2791 } 2792 void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2793 MOZ_ASSERT(HasSSE2()); 2794 switch (src1.kind()) { 2795 case Operand::FPREG: 2796 masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2797 break; 2798 case Operand::MEM_REG_DISP: 2799 masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2800 break; 2801 default: 2802 MOZ_CRASH("unexpected operand kind"); 2803 } 2804 } 2805 void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2806 MOZ_ASSERT(HasSSE41()); 2807 switch (src1.kind()) { 2808 case Operand::FPREG: 2809 masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2810 break; 2811 case Operand::MEM_REG_DISP: 2812 masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2813 break; 2814 case Operand::MEM_ADDRESS32: 2815 masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding()); 2816 break; 2817 default: 2818 MOZ_CRASH("unexpected operand kind"); 2819 } 2820 } 2821 void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2822 MOZ_ASSERT(HasSSE2()); 2823 switch (src1.kind()) { 2824 case Operand::FPREG: 2825 masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2826 break; 2827 case Operand::MEM_REG_DISP: 2828 masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2829 break; 2830 case Operand::MEM_ADDRESS32: 2831 masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding()); 2832 break; 2833 default: 2834 MOZ_CRASH("unexpected operand kind"); 2835 } 2836 } 2837 void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2838 MOZ_ASSERT(HasSSE2()); 2839 switch (src1.kind()) { 2840 case Operand::FPREG: 2841 masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2842 break; 2843 case Operand::MEM_REG_DISP: 2844 masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2845 break; 2846 case Operand::MEM_ADDRESS32: 2847 masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding()); 2848 break; 2849 default: 2850 MOZ_CRASH("unexpected operand kind"); 2851 } 2852 } 2853 void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2854 MOZ_ASSERT(HasSSE2()); 2855 switch (src1.kind()) { 2856 case Operand::FPREG: 2857 masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2858 break; 2859 case Operand::MEM_REG_DISP: 2860 masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2861 break; 2862 case Operand::MEM_ADDRESS32: 2863 masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding()); 2864 break; 2865 default: 2866 MOZ_CRASH("unexpected operand kind"); 2867 } 2868 } 2869 void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2870 MOZ_ASSERT(HasSSE2()); 2871 switch (src1.kind()) { 2872 case Operand::FPREG: 2873 masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2874 break; 2875 case Operand::MEM_REG_DISP: 2876 masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2877 break; 2878 case Operand::MEM_ADDRESS32: 2879 masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding()); 2880 break; 2881 default: 2882 MOZ_CRASH("unexpected operand kind"); 2883 } 2884 } 2885 void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2886 MOZ_ASSERT(HasSSE2()); 2887 switch (src1.kind()) { 2888 case Operand::FPREG: 2889 masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2890 break; 2891 case Operand::MEM_REG_DISP: 2892 masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2893 break; 2894 case Operand::MEM_ADDRESS32: 2895 masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding()); 2896 break; 2897 default: 2898 MOZ_CRASH("unexpected operand kind"); 2899 } 2900 } 2901 void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2902 MOZ_ASSERT(HasSSE2()); 2903 switch (src1.kind()) { 2904 case Operand::FPREG: 2905 masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2906 break; 2907 case Operand::MEM_REG_DISP: 2908 masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2909 break; 2910 case Operand::MEM_ADDRESS32: 2911 masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding()); 2912 break; 2913 default: 2914 MOZ_CRASH("unexpected operand kind"); 2915 } 2916 } 2917 void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2918 MOZ_ASSERT(HasSSE2()); 2919 switch (src1.kind()) { 2920 case Operand::FPREG: 2921 masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2922 break; 2923 case Operand::MEM_REG_DISP: 2924 masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2925 break; 2926 case Operand::MEM_ADDRESS32: 2927 masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding()); 2928 break; 2929 default: 2930 MOZ_CRASH("unexpected operand kind"); 2931 } 2932 } 2933 void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2934 // Negates bits of dest and then applies AND 2935 MOZ_ASSERT(HasSSE2()); 2936 switch (src1.kind()) { 2937 case Operand::FPREG: 2938 masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2939 break; 2940 case Operand::MEM_REG_DISP: 2941 masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2942 break; 2943 case Operand::MEM_ADDRESS32: 2944 masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding()); 2945 break; 2946 default: 2947 MOZ_CRASH("unexpected operand kind"); 2948 } 2949 } 2950 void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2951 MOZ_ASSERT(HasSSE2()); 2952 switch (src1.kind()) { 2953 case Operand::FPREG: 2954 masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2955 break; 2956 case Operand::MEM_REG_DISP: 2957 masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2958 break; 2959 case Operand::MEM_ADDRESS32: 2960 masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding()); 2961 break; 2962 default: 2963 MOZ_CRASH("unexpected operand kind"); 2964 } 2965 } 2966 void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2967 MOZ_ASSERT(HasSSE2()); 2968 switch (src1.kind()) { 2969 case Operand::FPREG: 2970 masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2971 break; 2972 case Operand::MEM_REG_DISP: 2973 masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2974 break; 2975 case Operand::MEM_ADDRESS32: 2976 masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding()); 2977 break; 2978 default: 2979 MOZ_CRASH("unexpected operand kind"); 2980 } 2981 } 2982 void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 2983 MOZ_ASSERT(HasSSE2()); 2984 masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding()); 2985 } 2986 void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) { 2987 MOZ_ASSERT(HasSSE2()); 2988 switch (src1.kind()) { 2989 case Operand::FPREG: 2990 masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding()); 2991 break; 2992 case Operand::MEM_REG_DISP: 2993 masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 2994 break; 2995 case Operand::MEM_ADDRESS32: 2996 masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding()); 2997 break; 2998 default: 2999 MOZ_CRASH("unexpected operand kind"); 3000 } 3001 } 3002 void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3003 MOZ_ASSERT(HasSSE2()); 3004 masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3005 } 3006 void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3007 MOZ_ASSERT(HasSSE2()); 3008 switch (src1.kind()) { 3009 case Operand::FPREG: 3010 masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3011 break; 3012 case Operand::MEM_REG_DISP: 3013 masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3014 break; 3015 case Operand::MEM_ADDRESS32: 3016 masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding()); 3017 break; 3018 default: 3019 MOZ_CRASH("unexpected operand kind"); 3020 } 3021 } 3022 void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3023 MOZ_ASSERT(HasSSE2()); 3024 masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3025 } 3026 void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3027 MOZ_ASSERT(HasSSE2()); 3028 switch (src1.kind()) { 3029 case Operand::FPREG: 3030 masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3031 break; 3032 case Operand::MEM_REG_DISP: 3033 masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3034 break; 3035 case Operand::MEM_ADDRESS32: 3036 masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding()); 3037 break; 3038 default: 3039 MOZ_CRASH("unexpected operand kind"); 3040 } 3041 } 3042 void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3043 MOZ_ASSERT(HasSSE2()); 3044 masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3045 } 3046 void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3047 MOZ_ASSERT(HasSSE2()); 3048 switch (src1.kind()) { 3049 case Operand::FPREG: 3050 masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3051 break; 3052 case Operand::MEM_REG_DISP: 3053 masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3054 break; 3055 case Operand::MEM_ADDRESS32: 3056 masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding()); 3057 break; 3058 default: 3059 MOZ_CRASH("unexpected operand kind"); 3060 } 3061 } 3062 3063 void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) { 3064 MOZ_ASSERT(HasSSE2()); 3065 masm.vpshufd_irr(mask, src.encoding(), dest.encoding()); 3066 } 3067 void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) { 3068 MOZ_ASSERT(HasSSE2()); 3069 switch (src1.kind()) { 3070 case Operand::FPREG: 3071 masm.vpshufd_irr(mask, src1.fpu(), dest.encoding()); 3072 break; 3073 case Operand::MEM_REG_DISP: 3074 masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding()); 3075 break; 3076 case Operand::MEM_ADDRESS32: 3077 masm.vpshufd_imr(mask, src1.address(), dest.encoding()); 3078 break; 3079 default: 3080 MOZ_CRASH("unexpected operand kind"); 3081 } 3082 } 3083 3084 void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) { 3085 MOZ_ASSERT(HasSSE2()); 3086 masm.vpshuflw_irr(mask, src.encoding(), dest.encoding()); 3087 } 3088 void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) { 3089 MOZ_ASSERT(HasSSE2()); 3090 masm.vpshufhw_irr(mask, src.encoding(), dest.encoding()); 3091 } 3092 void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) { 3093 MOZ_ASSERT(HasSSSE3()); 3094 masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding()); 3095 } 3096 void vmovddup(FloatRegister src, FloatRegister dest) { 3097 MOZ_ASSERT(HasSSE3()); 3098 masm.vmovddup_rr(src.encoding(), dest.encoding()); 3099 } 3100 void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3101 MOZ_ASSERT(HasSSE2()); 3102 masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3103 } 3104 void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3105 MOZ_ASSERT(HasSSE2()); 3106 masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3107 } 3108 void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3109 MOZ_ASSERT(HasSSE2()); 3110 masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3111 } 3112 void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3113 MOZ_ASSERT(HasSSE2()); 3114 switch (src1.kind()) { 3115 case Operand::FPREG: 3116 masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3117 break; 3118 case Operand::MEM_REG_DISP: 3119 masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3120 break; 3121 case Operand::MEM_ADDRESS32: 3122 masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding()); 3123 break; 3124 default: 3125 MOZ_CRASH("unexpected operand kind"); 3126 } 3127 } 3128 void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3129 MOZ_ASSERT(HasSSE2()); 3130 masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3131 } 3132 void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3133 MOZ_ASSERT(HasSSE2()); 3134 switch (src1.kind()) { 3135 case Operand::FPREG: 3136 masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3137 break; 3138 case Operand::MEM_REG_DISP: 3139 masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3140 break; 3141 case Operand::MEM_ADDRESS32: 3142 masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding()); 3143 break; 3144 default: 3145 MOZ_CRASH("unexpected operand kind"); 3146 } 3147 } 3148 void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3149 MOZ_ASSERT(HasSSE2()); 3150 masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 3151 } 3152 void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { 3153 MOZ_ASSERT(HasSSE2()); 3154 switch (src1.kind()) { 3155 case Operand::FPREG: 3156 masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 3157 break; 3158 case Operand::MEM_REG_DISP: 3159 masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3160 break; 3161 case Operand::MEM_ADDRESS32: 3162 masm.vshufps_imr(mask, src1.address(), src0.encoding(), dest.encoding()); 3163 break; 3164 default: 3165 MOZ_CRASH("unexpected operand kind"); 3166 } 3167 } 3168 void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3169 MOZ_ASSERT(HasSSE2()); 3170 masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3171 } 3172 void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3173 MOZ_ASSERT(HasSSE2()); 3174 masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3175 } 3176 void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3177 MOZ_ASSERT(HasSSE2()); 3178 switch (src1.kind()) { 3179 case Operand::FPREG: 3180 masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3181 break; 3182 case Operand::MEM_REG_DISP: 3183 masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3184 break; 3185 case Operand::MEM_ADDRESS32: 3186 masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding()); 3187 break; 3188 default: 3189 MOZ_CRASH("unexpected operand kind"); 3190 } 3191 } 3192 void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3193 MOZ_ASSERT(HasSSE2()); 3194 switch (src1.kind()) { 3195 case Operand::FPREG: 3196 masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3197 break; 3198 case Operand::MEM_REG_DISP: 3199 masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3200 break; 3201 case Operand::MEM_ADDRESS32: 3202 masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding()); 3203 break; 3204 default: 3205 MOZ_CRASH("unexpected operand kind"); 3206 } 3207 } 3208 void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3209 MOZ_ASSERT(HasSSE2()); 3210 masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3211 } 3212 void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3213 MOZ_ASSERT(HasSSE2()); 3214 masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3215 } 3216 void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3217 MOZ_ASSERT(HasSSE2()); 3218 switch (src1.kind()) { 3219 case Operand::FPREG: 3220 masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3221 break; 3222 case Operand::MEM_REG_DISP: 3223 masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3224 break; 3225 default: 3226 MOZ_CRASH("unexpected operand kind"); 3227 } 3228 } 3229 void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3230 MOZ_ASSERT(HasSSE2()); 3231 switch (src1.kind()) { 3232 case Operand::FPREG: 3233 masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3234 break; 3235 case Operand::MEM_REG_DISP: 3236 masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3237 break; 3238 default: 3239 MOZ_CRASH("unexpected operand kind"); 3240 } 3241 } 3242 void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3243 MOZ_ASSERT(HasSSE2()); 3244 masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3245 } 3246 void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3247 MOZ_ASSERT(HasSSE2()); 3248 switch (src1.kind()) { 3249 case Operand::FPREG: 3250 masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3251 break; 3252 case Operand::MEM_REG_DISP: 3253 masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3254 break; 3255 default: 3256 MOZ_CRASH("unexpected operand kind"); 3257 } 3258 } 3259 void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3260 MOZ_ASSERT(HasSSE2()); 3261 switch (src1.kind()) { 3262 case Operand::FPREG: 3263 masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3264 break; 3265 case Operand::MEM_REG_DISP: 3266 masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3267 break; 3268 default: 3269 MOZ_CRASH("unexpected operand kind"); 3270 } 3271 } 3272 void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3273 MOZ_ASSERT(HasSSE2()); 3274 masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3275 } 3276 void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3277 MOZ_ASSERT(HasSSE2()); 3278 masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3279 } 3280 void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3281 MOZ_ASSERT(HasSSE2()); 3282 masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3283 } 3284 void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3285 MOZ_ASSERT(HasSSE2()); 3286 switch (src1.kind()) { 3287 case Operand::FPREG: 3288 masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3289 break; 3290 case Operand::MEM_REG_DISP: 3291 masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3292 break; 3293 default: 3294 MOZ_CRASH("unexpected operand kind"); 3295 } 3296 } 3297 void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3298 MOZ_ASSERT(HasSSE2()); 3299 switch (src1.kind()) { 3300 case Operand::FPREG: 3301 masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3302 break; 3303 case Operand::MEM_REG_DISP: 3304 masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3305 break; 3306 default: 3307 MOZ_CRASH("unexpected operand kind"); 3308 } 3309 } 3310 void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3311 MOZ_ASSERT(HasSSE2()); 3312 masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3313 } 3314 void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3315 MOZ_ASSERT(HasSSE2()); 3316 masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3317 } 3318 void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3319 MOZ_ASSERT(HasSSE2()); 3320 masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3321 } 3322 void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3323 MOZ_ASSERT(HasSSE2()); 3324 masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3325 } 3326 void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3327 MOZ_ASSERT(HasSSE2()); 3328 masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3329 } 3330 void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3331 MOZ_ASSERT(HasSSE2()); 3332 masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3333 } 3334 void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3335 MOZ_ASSERT(HasSSE2()); 3336 masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3337 } 3338 void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3339 MOZ_ASSERT(HasSSE2()); 3340 masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3341 } 3342 void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3343 MOZ_ASSERT(HasSSE41()); 3344 masm.vroundsd_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); 3345 } 3346 void vroundss(X86Encoding::RoundingMode mode, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3347 MOZ_ASSERT(HasSSE41()); 3348 masm.vroundss_irr(mode, src1.encoding(), src0.encoding(), dest.encoding()); 3349 } 3350 unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane, unsigned zeroMask = 0) 3351 { 3352 // Note that the sourceLane bits are ignored in the case of a source 3353 // memory operand, and the source is the given 32-bits memory location. 3354 MOZ_ASSERT(zeroMask < 16); 3355 unsigned ret = zeroMask ; 3356 ret |= destLane << 4; 3357 ret |= sourceLane << 6; 3358 MOZ_ASSERT(ret < 256); 3359 return ret; 3360 } 3361 void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3362 MOZ_ASSERT(HasSSE41()); 3363 masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 3364 } 3365 void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { 3366 MOZ_ASSERT(HasSSE41()); 3367 switch (src1.kind()) { 3368 case Operand::FPREG: 3369 masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 3370 break; 3371 case Operand::MEM_REG_DISP: 3372 masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3373 break; 3374 default: 3375 MOZ_CRASH("unexpected operand kind"); 3376 } 3377 } 3378 unsigned blendpsMask(bool x, bool y, bool z, bool w) { 3379 return (x << 0) | (y << 1) | (z << 2) | (w << 3); 3380 } 3381 void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3382 MOZ_ASSERT(HasSSE41()); 3383 masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding()); 3384 } 3385 void vblendps(unsigned mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { 3386 MOZ_ASSERT(HasSSE41()); 3387 switch (src1.kind()) { 3388 case Operand::FPREG: 3389 masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding()); 3390 break; 3391 case Operand::MEM_REG_DISP: 3392 masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3393 break; 3394 default: 3395 MOZ_CRASH("unexpected operand kind"); 3396 } 3397 } 3398 void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3399 MOZ_ASSERT(HasSSE41()); 3400 masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(), dest.encoding()); 3401 } 3402 void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0, FloatRegister dest) { 3403 MOZ_ASSERT(HasSSE41()); 3404 switch (src1.kind()) { 3405 case Operand::FPREG: 3406 masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(), dest.encoding()); 3407 break; 3408 case Operand::MEM_REG_DISP: 3409 masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3410 break; 3411 default: 3412 MOZ_CRASH("unexpected operand kind"); 3413 } 3414 } 3415 void vmovsldup(FloatRegister src, FloatRegister dest) { 3416 MOZ_ASSERT(HasSSE3()); 3417 masm.vmovsldup_rr(src.encoding(), dest.encoding()); 3418 } 3419 void vmovsldup(const Operand& src, FloatRegister dest) { 3420 MOZ_ASSERT(HasSSE3()); 3421 switch (src.kind()) { 3422 case Operand::FPREG: 3423 masm.vmovsldup_rr(src.fpu(), dest.encoding()); 3424 break; 3425 case Operand::MEM_REG_DISP: 3426 masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding()); 3427 break; 3428 default: 3429 MOZ_CRASH("unexpected operand kind"); 3430 } 3431 } 3432 void vmovshdup(FloatRegister src, FloatRegister dest) { 3433 MOZ_ASSERT(HasSSE3()); 3434 masm.vmovshdup_rr(src.encoding(), dest.encoding()); 3435 } 3436 void vmovshdup(const Operand& src, FloatRegister dest) { 3437 MOZ_ASSERT(HasSSE3()); 3438 switch (src.kind()) { 3439 case Operand::FPREG: 3440 masm.vmovshdup_rr(src.fpu(), dest.encoding()); 3441 break; 3442 case Operand::MEM_REG_DISP: 3443 masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding()); 3444 break; 3445 default: 3446 MOZ_CRASH("unexpected operand kind"); 3447 } 3448 } 3449 void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3450 MOZ_ASSERT(HasSSE2()); 3451 masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3452 } 3453 void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3454 MOZ_ASSERT(HasSSE2()); 3455 switch (src1.kind()) { 3456 case Operand::FPREG: 3457 masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3458 break; 3459 case Operand::MEM_REG_DISP: 3460 masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3461 break; 3462 default: 3463 MOZ_CRASH("unexpected operand kind"); 3464 } 3465 } 3466 void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3467 MOZ_ASSERT(HasSSE2()); 3468 masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3469 } 3470 void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3471 MOZ_ASSERT(HasSSE2()); 3472 masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3473 } 3474 void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) { 3475 MOZ_ASSERT(HasSSE2()); 3476 switch (src1.kind()) { 3477 case Operand::FPREG: 3478 masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding()); 3479 break; 3480 case Operand::MEM_REG_DISP: 3481 masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(), dest.encoding()); 3482 break; 3483 default: 3484 MOZ_CRASH("unexpected operand kind"); 3485 } 3486 } 3487 void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) { 3488 MOZ_ASSERT(HasSSE2()); 3489 masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding()); 3490 } 3491 void fisttp(const Operand& dest) { 3492 MOZ_ASSERT(HasSSE3()); 3493 switch (dest.kind()) { 3494 case Operand::MEM_REG_DISP: 3495 masm.fisttp_m(dest.disp(), dest.base()); 3496 break; 3497 default: 3498 MOZ_CRASH("unexpected operand kind"); 3499 } 3500 } 3501 void fistp(const Operand& dest) { 3502 switch (dest.kind()) { 3503 case Operand::MEM_REG_DISP: 3504 masm.fistp_m(dest.disp(), dest.base()); 3505 break; 3506 default: 3507 MOZ_CRASH("unexpected operand kind"); 3508 } 3509 } 3510 void fnstcw(const Operand& dest) { 3511 switch (dest.kind()) { 3512 case Operand::MEM_REG_DISP: 3513 masm.fnstcw_m(dest.disp(), dest.base()); 3514 break; 3515 default: 3516 MOZ_CRASH("unexpected operand kind"); 3517 } 3518 } 3519 void fldcw(const Operand& dest) { 3520 switch (dest.kind()) { 3521 case Operand::MEM_REG_DISP: 3522 masm.fldcw_m(dest.disp(), dest.base()); 3523 break; 3524 default: 3525 MOZ_CRASH("unexpected operand kind"); 3526 } 3527 } 3528 void fnstsw(const Operand& dest) { 3529 switch (dest.kind()) { 3530 case Operand::MEM_REG_DISP: 3531 masm.fnstsw_m(dest.disp(), dest.base()); 3532 break; 3533 default: 3534 MOZ_CRASH("unexpected operand kind"); 3535 } 3536 } 3537 void fld(const Operand& dest) { 3538 switch (dest.kind()) { 3539 case Operand::MEM_REG_DISP: 3540 masm.fld_m(dest.disp(), dest.base()); 3541 break; 3542 default: 3543 MOZ_CRASH("unexpected operand kind"); 3544 } 3545 } 3546 void fld32(const Operand& dest) { 3547 switch (dest.kind()) { 3548 case Operand::MEM_REG_DISP: 3549 masm.fld32_m(dest.disp(), dest.base()); 3550 break; 3551 default: 3552 MOZ_CRASH("unexpected operand kind"); 3553 } 3554 } 3555 void fstp(const Operand& src) { 3556 switch (src.kind()) { 3557 case Operand::MEM_REG_DISP: 3558 masm.fstp_m(src.disp(), src.base()); 3559 break; 3560 default: 3561 MOZ_CRASH("unexpected operand kind"); 3562 } 3563 } 3564 void fstp32(const Operand& src) { 3565 switch (src.kind()) { 3566 case Operand::MEM_REG_DISP: 3567 masm.fstp32_m(src.disp(), src.base()); 3568 break; 3569 default: 3570 MOZ_CRASH("unexpected operand kind"); 3571 } 3572 } 3573 3574 // Defined for compatibility with ARM's assembler 3575 uint32_t actualIndex(uint32_t x) { 3576 return x; 3577 } 3578 3579 void flushBuffer() { 3580 } 3581 3582 // Patching. 3583 3584 static size_t PatchWrite_NearCallSize() { 3585 return 5; 3586 } 3587 static uintptr_t GetPointer(uint8_t* instPtr) { 3588 uintptr_t* ptr = ((uintptr_t*) instPtr) - 1; 3589 return *ptr; 3590 } 3591 // Write a relative call at the start location |dataLabel|. 3592 // Note that this DOES NOT patch data that comes before |label|. 3593 static void PatchWrite_NearCall(CodeLocationLabel startLabel, CodeLocationLabel target) { 3594 uint8_t* start = startLabel.raw(); 3595 *start = 0xE8; 3596 ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize(); 3597 MOZ_ASSERT(int32_t(offset) == offset); 3598 *((int32_t*) (start + 1)) = offset; 3599 } 3600 3601 static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) { 3602 *((int32_t*) dataLabel.raw() - 1) = toWrite.value; 3603 } 3604 3605 static void PatchDataWithValueCheck(CodeLocationLabel data, PatchedImmPtr newData, 3606 PatchedImmPtr expectedData) { 3607 // The pointer given is a pointer to *after* the data. 3608 uintptr_t* ptr = ((uintptr_t*) data.raw()) - 1; 3609 MOZ_ASSERT(*ptr == (uintptr_t)expectedData.value); 3610 *ptr = (uintptr_t)newData.value; 3611 } 3612 static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData, ImmPtr expectedData) { 3613 PatchDataWithValueCheck(data, PatchedImmPtr(newData.value), PatchedImmPtr(expectedData.value)); 3614 } 3615 3616 static void PatchInstructionImmediate(uint8_t* code, PatchedImmPtr imm) { 3617 MOZ_CRASH("Unused."); 3618 } 3619 3620 static uint32_t NopSize() { 3621 return 1; 3622 } 3623 static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) { 3624 MOZ_CRASH("nextInstruction NYI on x86"); 3625 } 3626 3627 // Toggle a jmp or cmp emitted by toggledJump(). 3628 static void ToggleToJmp(CodeLocationLabel inst) { 3629 uint8_t* ptr = (uint8_t*)inst.raw(); 3630 MOZ_ASSERT(*ptr == 0x3D); 3631 *ptr = 0xE9; 3632 } 3633 static void ToggleToCmp(CodeLocationLabel inst) { 3634 uint8_t* ptr = (uint8_t*)inst.raw(); 3635 MOZ_ASSERT(*ptr == 0xE9); 3636 *ptr = 0x3D; 3637 } 3638 static void ToggleCall(CodeLocationLabel inst, bool enabled) { 3639 uint8_t* ptr = (uint8_t*)inst.raw(); 3640 MOZ_ASSERT(*ptr == 0x3D || // CMP 3641 *ptr == 0xE8); // CALL 3642 *ptr = enabled ? 0xE8 : 0x3D; 3643 } 3644 3645 MOZ_COLD void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, 3646 const Disassembler::HeapAccess& heapAccess); 3647 }; 3648 3649 } // namespace jit 3650 } // namespace js 3651 3652 #endif /* jit_x86_shared_Assembler_x86_shared_h */ 3653