1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- 2 * vim: set ts=8 sts=2 et sw=2 tw=80: 3 * This Source Code Form is subject to the terms of the Mozilla Public 4 * License, v. 2.0. If a copy of the MPL was not distributed with this 5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 7 #ifndef jit_x64_MacroAssembler_x64_h 8 #define jit_x64_MacroAssembler_x64_h 9 10 #include "jit/JitFrames.h" 11 #include "jit/MoveResolver.h" 12 #include "jit/x86-shared/MacroAssembler-x86-shared.h" 13 #include "js/HeapAPI.h" 14 #include "vm/BigIntType.h" // JS::BigInt 15 16 namespace js { 17 namespace jit { 18 19 struct ImmShiftedTag : public ImmWord { ImmShiftedTagImmShiftedTag20 explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {} 21 ImmShiftedTagImmShiftedTag22 explicit ImmShiftedTag(JSValueType type) 23 : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {} 24 }; 25 26 struct ImmTag : public Imm32 { ImmTagImmTag27 explicit ImmTag(JSValueTag tag) : Imm32(tag) {} 28 }; 29 30 // ScratchTagScope and ScratchTagScopeRelease are used to manage the tag 31 // register for splitTagForTest(), which has different register management on 32 // different platforms. On 64-bit platforms it requires a scratch register that 33 // does not interfere with other operations; on 32-bit platforms it uses a 34 // register that is already part of the Value. 35 // 36 // The ScratchTagScope RAII type acquires the appropriate register; a reference 37 // to a variable of this type is then passed to splitTagForTest(). 38 // 39 // On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register 40 // available in a dynamic scope during compilation. However it is important to 41 // remember that that does not preserve the register value in any way, so this 42 // RAII type should only be used along paths that eventually branch past further 43 // uses of the extracted tag value. 44 // 45 // On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not 46 // manage a register, it only aliases a register in the ValueOperand. 47 48 class ScratchTagScope : public ScratchRegisterScope { 49 public: ScratchTagScope(MacroAssembler & masm,const ValueOperand &)50 ScratchTagScope(MacroAssembler& masm, const ValueOperand&) 51 : ScratchRegisterScope(masm) {} 52 }; 53 54 class ScratchTagScopeRelease { 55 ScratchTagScope* ts_; 56 57 public: ScratchTagScopeRelease(ScratchTagScope * ts)58 explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) { 59 ts_->release(); 60 } ~ScratchTagScopeRelease()61 ~ScratchTagScopeRelease() { ts_->reacquire(); } 62 }; 63 64 class MacroAssemblerX64 : public MacroAssemblerX86Shared { 65 private: 66 // Perform a downcast. Should be removed by Bug 996602. 67 MacroAssembler& asMasm(); 68 const MacroAssembler& asMasm() const; 69 70 void bindOffsets(const MacroAssemblerX86Shared::UsesVector&); 71 72 public: 73 using MacroAssemblerX86Shared::load32; 74 using MacroAssemblerX86Shared::store16; 75 using MacroAssemblerX86Shared::store32; 76 77 MacroAssemblerX64() = default; 78 79 // The buffer is about to be linked, make sure any constant pools or excess 80 // bookkeeping has been flushed to the instruction stream. 81 void finish(); 82 83 ///////////////////////////////////////////////////////////////// 84 // X64 helpers. 85 ///////////////////////////////////////////////////////////////// writeDataRelocation(const Value & val)86 void writeDataRelocation(const Value& val) { 87 // Raw GC pointer relocations and Value relocations both end up in 88 // Assembler::TraceDataRelocations. 89 if (val.isGCThing()) { 90 gc::Cell* cell = val.toGCThing(); 91 if (cell && gc::IsInsideNursery(cell)) { 92 embedsNurseryPointers_ = true; 93 } 94 dataRelocations_.writeUnsigned(masm.currentOffset()); 95 } 96 } 97 98 // Refers to the upper 32 bits of a 64-bit Value operand. 99 // On x86_64, the upper 32 bits do not necessarily only contain the type. ToUpper32(Operand base)100 Operand ToUpper32(Operand base) { 101 switch (base.kind()) { 102 case Operand::MEM_REG_DISP: 103 return Operand(Register::FromCode(base.base()), base.disp() + 4); 104 105 case Operand::MEM_SCALE: 106 return Operand(Register::FromCode(base.base()), 107 Register::FromCode(base.index()), base.scale(), 108 base.disp() + 4); 109 110 default: 111 MOZ_CRASH("unexpected operand kind"); 112 } 113 } ToUpper32(const Address & address)114 static inline Operand ToUpper32(const Address& address) { 115 return Operand(address.base, address.offset + 4); 116 } ToUpper32(const BaseIndex & address)117 static inline Operand ToUpper32(const BaseIndex& address) { 118 return Operand(address.base, address.index, address.scale, 119 address.offset + 4); 120 } 121 Upper32Of(JSValueShiftedTag tag)122 uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); } 123 GetShiftedTag(JSValueType type)124 JSValueShiftedTag GetShiftedTag(JSValueType type) { 125 return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type); 126 } 127 128 ///////////////////////////////////////////////////////////////// 129 // X86/X64-common interface. 130 ///////////////////////////////////////////////////////////////// ToPayload(Address value)131 Address ToPayload(Address value) { return value; } 132 storeValue(ValueOperand val,Operand dest)133 void storeValue(ValueOperand val, Operand dest) { 134 movq(val.valueReg(), dest); 135 } storeValue(ValueOperand val,const Address & dest)136 void storeValue(ValueOperand val, const Address& dest) { 137 storeValue(val, Operand(dest)); 138 } 139 template <typename T> storeValue(JSValueType type,Register reg,const T & dest)140 void storeValue(JSValueType type, Register reg, const T& dest) { 141 // Value types with 32-bit payloads can be emitted as two 32-bit moves. 142 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { 143 movl(reg, Operand(dest)); 144 movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest))); 145 } else { 146 ScratchRegisterScope scratch(asMasm()); 147 #ifdef NIGHTLY_BUILD 148 // Bug 1485209 - Diagnostic assert for constructing Values with 149 // nullptr or misaligned (eg poisoned) JSObject/JSString pointers. 150 if (type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING) { 151 Label crash, ok; 152 testPtr(reg, Imm32(js::gc::CellAlignMask)); 153 j(Assembler::NonZero, &crash); 154 testPtr(reg, reg); 155 j(Assembler::NonZero, &ok); 156 bind(&crash); 157 breakpoint(); 158 bind(&ok); 159 } 160 #endif 161 boxValue(type, reg, scratch); 162 movq(scratch, Operand(dest)); 163 } 164 } 165 template <typename T> storeValue(const Value & val,const T & dest)166 void storeValue(const Value& val, const T& dest) { 167 ScratchRegisterScope scratch(asMasm()); 168 if (val.isGCThing()) { 169 movWithPatch(ImmWord(val.asRawBits()), scratch); 170 writeDataRelocation(val); 171 } else { 172 mov(ImmWord(val.asRawBits()), scratch); 173 } 174 movq(scratch, Operand(dest)); 175 } storeValue(ValueOperand val,BaseIndex dest)176 void storeValue(ValueOperand val, BaseIndex dest) { 177 storeValue(val, Operand(dest)); 178 } storeValue(const Address & src,const Address & dest,Register temp)179 void storeValue(const Address& src, const Address& dest, Register temp) { 180 loadPtr(src, temp); 181 storePtr(temp, dest); 182 } loadValue(Operand src,ValueOperand val)183 void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); } loadValue(Address src,ValueOperand val)184 void loadValue(Address src, ValueOperand val) { 185 loadValue(Operand(src), val); 186 } loadValue(const BaseIndex & src,ValueOperand val)187 void loadValue(const BaseIndex& src, ValueOperand val) { 188 loadValue(Operand(src), val); 189 } loadUnalignedValue(const Address & src,ValueOperand dest)190 void loadUnalignedValue(const Address& src, ValueOperand dest) { 191 loadValue(src, dest); 192 } tagValue(JSValueType type,Register payload,ValueOperand dest)193 void tagValue(JSValueType type, Register payload, ValueOperand dest) { 194 ScratchRegisterScope scratch(asMasm()); 195 MOZ_ASSERT(dest.valueReg() != scratch); 196 if (payload != dest.valueReg()) { 197 movq(payload, dest.valueReg()); 198 } 199 mov(ImmShiftedTag(type), scratch); 200 orq(scratch, dest.valueReg()); 201 } pushValue(ValueOperand val)202 void pushValue(ValueOperand val) { push(val.valueReg()); } popValue(ValueOperand val)203 void popValue(ValueOperand val) { pop(val.valueReg()); } pushValue(const Value & val)204 void pushValue(const Value& val) { 205 if (val.isGCThing()) { 206 ScratchRegisterScope scratch(asMasm()); 207 movWithPatch(ImmWord(val.asRawBits()), scratch); 208 writeDataRelocation(val); 209 push(scratch); 210 } else { 211 push(ImmWord(val.asRawBits())); 212 } 213 } pushValue(JSValueType type,Register reg)214 void pushValue(JSValueType type, Register reg) { 215 ScratchRegisterScope scratch(asMasm()); 216 boxValue(type, reg, scratch); 217 push(scratch); 218 } pushValue(const Address & addr)219 void pushValue(const Address& addr) { push(Operand(addr)); } 220 221 void boxValue(JSValueType type, Register src, Register dest); 222 testUndefined(Condition cond,Register tag)223 Condition testUndefined(Condition cond, Register tag) { 224 MOZ_ASSERT(cond == Equal || cond == NotEqual); 225 cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED)); 226 return cond; 227 } testInt32(Condition cond,Register tag)228 Condition testInt32(Condition cond, Register tag) { 229 MOZ_ASSERT(cond == Equal || cond == NotEqual); 230 cmp32(tag, ImmTag(JSVAL_TAG_INT32)); 231 return cond; 232 } testBoolean(Condition cond,Register tag)233 Condition testBoolean(Condition cond, Register tag) { 234 MOZ_ASSERT(cond == Equal || cond == NotEqual); 235 cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN)); 236 return cond; 237 } testNull(Condition cond,Register tag)238 Condition testNull(Condition cond, Register tag) { 239 MOZ_ASSERT(cond == Equal || cond == NotEqual); 240 cmp32(tag, ImmTag(JSVAL_TAG_NULL)); 241 return cond; 242 } testString(Condition cond,Register tag)243 Condition testString(Condition cond, Register tag) { 244 MOZ_ASSERT(cond == Equal || cond == NotEqual); 245 cmp32(tag, ImmTag(JSVAL_TAG_STRING)); 246 return cond; 247 } testSymbol(Condition cond,Register tag)248 Condition testSymbol(Condition cond, Register tag) { 249 MOZ_ASSERT(cond == Equal || cond == NotEqual); 250 cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL)); 251 return cond; 252 } testBigInt(Condition cond,Register tag)253 Condition testBigInt(Condition cond, Register tag) { 254 MOZ_ASSERT(cond == Equal || cond == NotEqual); 255 cmp32(tag, ImmTag(JSVAL_TAG_BIGINT)); 256 return cond; 257 } testObject(Condition cond,Register tag)258 Condition testObject(Condition cond, Register tag) { 259 MOZ_ASSERT(cond == Equal || cond == NotEqual); 260 cmp32(tag, ImmTag(JSVAL_TAG_OBJECT)); 261 return cond; 262 } testDouble(Condition cond,Register tag)263 Condition testDouble(Condition cond, Register tag) { 264 MOZ_ASSERT(cond == Equal || cond == NotEqual); 265 cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE)); 266 return cond == Equal ? BelowOrEqual : Above; 267 } testNumber(Condition cond,Register tag)268 Condition testNumber(Condition cond, Register tag) { 269 MOZ_ASSERT(cond == Equal || cond == NotEqual); 270 cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag)); 271 return cond == Equal ? BelowOrEqual : Above; 272 } testGCThing(Condition cond,Register tag)273 Condition testGCThing(Condition cond, Register tag) { 274 MOZ_ASSERT(cond == Equal || cond == NotEqual); 275 cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag)); 276 return cond == Equal ? AboveOrEqual : Below; 277 } 278 testMagic(Condition cond,Register tag)279 Condition testMagic(Condition cond, Register tag) { 280 MOZ_ASSERT(cond == Equal || cond == NotEqual); 281 cmp32(tag, ImmTag(JSVAL_TAG_MAGIC)); 282 return cond; 283 } testError(Condition cond,Register tag)284 Condition testError(Condition cond, Register tag) { 285 return testMagic(cond, tag); 286 } testPrimitive(Condition cond,Register tag)287 Condition testPrimitive(Condition cond, Register tag) { 288 MOZ_ASSERT(cond == Equal || cond == NotEqual); 289 cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag)); 290 return cond == Equal ? Below : AboveOrEqual; 291 } 292 testUndefined(Condition cond,const ValueOperand & src)293 Condition testUndefined(Condition cond, const ValueOperand& src) { 294 ScratchRegisterScope scratch(asMasm()); 295 splitTag(src, scratch); 296 return testUndefined(cond, scratch); 297 } testInt32(Condition cond,const ValueOperand & src)298 Condition testInt32(Condition cond, const ValueOperand& src) { 299 ScratchRegisterScope scratch(asMasm()); 300 splitTag(src, scratch); 301 return testInt32(cond, scratch); 302 } testBoolean(Condition cond,const ValueOperand & src)303 Condition testBoolean(Condition cond, const ValueOperand& src) { 304 ScratchRegisterScope scratch(asMasm()); 305 splitTag(src, scratch); 306 return testBoolean(cond, scratch); 307 } testDouble(Condition cond,const ValueOperand & src)308 Condition testDouble(Condition cond, const ValueOperand& src) { 309 ScratchRegisterScope scratch(asMasm()); 310 splitTag(src, scratch); 311 return testDouble(cond, scratch); 312 } testNumber(Condition cond,const ValueOperand & src)313 Condition testNumber(Condition cond, const ValueOperand& src) { 314 ScratchRegisterScope scratch(asMasm()); 315 splitTag(src, scratch); 316 return testNumber(cond, scratch); 317 } testNull(Condition cond,const ValueOperand & src)318 Condition testNull(Condition cond, const ValueOperand& src) { 319 ScratchRegisterScope scratch(asMasm()); 320 splitTag(src, scratch); 321 return testNull(cond, scratch); 322 } testString(Condition cond,const ValueOperand & src)323 Condition testString(Condition cond, const ValueOperand& src) { 324 ScratchRegisterScope scratch(asMasm()); 325 splitTag(src, scratch); 326 return testString(cond, scratch); 327 } testSymbol(Condition cond,const ValueOperand & src)328 Condition testSymbol(Condition cond, const ValueOperand& src) { 329 ScratchRegisterScope scratch(asMasm()); 330 splitTag(src, scratch); 331 return testSymbol(cond, scratch); 332 } testBigInt(Condition cond,const ValueOperand & src)333 Condition testBigInt(Condition cond, const ValueOperand& src) { 334 ScratchRegisterScope scratch(asMasm()); 335 splitTag(src, scratch); 336 return testBigInt(cond, scratch); 337 } testObject(Condition cond,const ValueOperand & src)338 Condition testObject(Condition cond, const ValueOperand& src) { 339 ScratchRegisterScope scratch(asMasm()); 340 splitTag(src, scratch); 341 return testObject(cond, scratch); 342 } testGCThing(Condition cond,const ValueOperand & src)343 Condition testGCThing(Condition cond, const ValueOperand& src) { 344 ScratchRegisterScope scratch(asMasm()); 345 splitTag(src, scratch); 346 return testGCThing(cond, scratch); 347 } testPrimitive(Condition cond,const ValueOperand & src)348 Condition testPrimitive(Condition cond, const ValueOperand& src) { 349 ScratchRegisterScope scratch(asMasm()); 350 splitTag(src, scratch); 351 return testPrimitive(cond, scratch); 352 } 353 testUndefined(Condition cond,const Address & src)354 Condition testUndefined(Condition cond, const Address& src) { 355 cmp32(ToUpper32(src), 356 Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED)))); 357 return cond; 358 } testInt32(Condition cond,const Address & src)359 Condition testInt32(Condition cond, const Address& src) { 360 cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32)))); 361 return cond; 362 } testBoolean(Condition cond,const Address & src)363 Condition testBoolean(Condition cond, const Address& src) { 364 cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN)))); 365 return cond; 366 } testDouble(Condition cond,const Address & src)367 Condition testDouble(Condition cond, const Address& src) { 368 ScratchRegisterScope scratch(asMasm()); 369 splitTag(src, scratch); 370 return testDouble(cond, scratch); 371 } testNumber(Condition cond,const Address & src)372 Condition testNumber(Condition cond, const Address& src) { 373 ScratchRegisterScope scratch(asMasm()); 374 splitTag(src, scratch); 375 return testNumber(cond, scratch); 376 } testNull(Condition cond,const Address & src)377 Condition testNull(Condition cond, const Address& src) { 378 cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL)))); 379 return cond; 380 } testString(Condition cond,const Address & src)381 Condition testString(Condition cond, const Address& src) { 382 ScratchRegisterScope scratch(asMasm()); 383 splitTag(src, scratch); 384 return testString(cond, scratch); 385 } testSymbol(Condition cond,const Address & src)386 Condition testSymbol(Condition cond, const Address& src) { 387 ScratchRegisterScope scratch(asMasm()); 388 splitTag(src, scratch); 389 return testSymbol(cond, scratch); 390 } testBigInt(Condition cond,const Address & src)391 Condition testBigInt(Condition cond, const Address& src) { 392 ScratchRegisterScope scratch(asMasm()); 393 splitTag(src, scratch); 394 return testBigInt(cond, scratch); 395 } testObject(Condition cond,const Address & src)396 Condition testObject(Condition cond, const Address& src) { 397 ScratchRegisterScope scratch(asMasm()); 398 splitTag(src, scratch); 399 return testObject(cond, scratch); 400 } testPrimitive(Condition cond,const Address & src)401 Condition testPrimitive(Condition cond, const Address& src) { 402 ScratchRegisterScope scratch(asMasm()); 403 splitTag(src, scratch); 404 return testPrimitive(cond, scratch); 405 } testGCThing(Condition cond,const Address & src)406 Condition testGCThing(Condition cond, const Address& src) { 407 ScratchRegisterScope scratch(asMasm()); 408 splitTag(src, scratch); 409 return testGCThing(cond, scratch); 410 } testMagic(Condition cond,const Address & src)411 Condition testMagic(Condition cond, const Address& src) { 412 ScratchRegisterScope scratch(asMasm()); 413 splitTag(src, scratch); 414 return testMagic(cond, scratch); 415 } 416 testUndefined(Condition cond,const BaseIndex & src)417 Condition testUndefined(Condition cond, const BaseIndex& src) { 418 ScratchRegisterScope scratch(asMasm()); 419 splitTag(src, scratch); 420 return testUndefined(cond, scratch); 421 } testNull(Condition cond,const BaseIndex & src)422 Condition testNull(Condition cond, const BaseIndex& src) { 423 ScratchRegisterScope scratch(asMasm()); 424 splitTag(src, scratch); 425 return testNull(cond, scratch); 426 } testBoolean(Condition cond,const BaseIndex & src)427 Condition testBoolean(Condition cond, const BaseIndex& src) { 428 ScratchRegisterScope scratch(asMasm()); 429 splitTag(src, scratch); 430 return testBoolean(cond, scratch); 431 } testString(Condition cond,const BaseIndex & src)432 Condition testString(Condition cond, const BaseIndex& src) { 433 ScratchRegisterScope scratch(asMasm()); 434 splitTag(src, scratch); 435 return testString(cond, scratch); 436 } testSymbol(Condition cond,const BaseIndex & src)437 Condition testSymbol(Condition cond, const BaseIndex& src) { 438 ScratchRegisterScope scratch(asMasm()); 439 splitTag(src, scratch); 440 return testSymbol(cond, scratch); 441 } testBigInt(Condition cond,const BaseIndex & src)442 Condition testBigInt(Condition cond, const BaseIndex& src) { 443 ScratchRegisterScope scratch(asMasm()); 444 splitTag(src, scratch); 445 return testBigInt(cond, scratch); 446 } testInt32(Condition cond,const BaseIndex & src)447 Condition testInt32(Condition cond, const BaseIndex& src) { 448 ScratchRegisterScope scratch(asMasm()); 449 splitTag(src, scratch); 450 return testInt32(cond, scratch); 451 } testObject(Condition cond,const BaseIndex & src)452 Condition testObject(Condition cond, const BaseIndex& src) { 453 ScratchRegisterScope scratch(asMasm()); 454 splitTag(src, scratch); 455 return testObject(cond, scratch); 456 } testDouble(Condition cond,const BaseIndex & src)457 Condition testDouble(Condition cond, const BaseIndex& src) { 458 ScratchRegisterScope scratch(asMasm()); 459 splitTag(src, scratch); 460 return testDouble(cond, scratch); 461 } testMagic(Condition cond,const BaseIndex & src)462 Condition testMagic(Condition cond, const BaseIndex& src) { 463 ScratchRegisterScope scratch(asMasm()); 464 splitTag(src, scratch); 465 return testMagic(cond, scratch); 466 } testGCThing(Condition cond,const BaseIndex & src)467 Condition testGCThing(Condition cond, const BaseIndex& src) { 468 ScratchRegisterScope scratch(asMasm()); 469 splitTag(src, scratch); 470 return testGCThing(cond, scratch); 471 } 472 isMagic(Condition cond,const ValueOperand & src,JSWhyMagic why)473 Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) { 474 uint64_t magic = MagicValue(why).asRawBits(); 475 cmpPtr(src.valueReg(), ImmWord(magic)); 476 return cond; 477 } 478 cmpPtr(Register lhs,const ImmWord rhs)479 void cmpPtr(Register lhs, const ImmWord rhs) { 480 ScratchRegisterScope scratch(asMasm()); 481 MOZ_ASSERT(lhs != scratch); 482 if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) { 483 cmpPtr(lhs, Imm32(int32_t(rhs.value))); 484 } else { 485 movePtr(rhs, scratch); 486 cmpPtr(lhs, scratch); 487 } 488 } cmpPtr(Register lhs,const ImmPtr rhs)489 void cmpPtr(Register lhs, const ImmPtr rhs) { 490 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); 491 } cmpPtr(Register lhs,const ImmGCPtr rhs)492 void cmpPtr(Register lhs, const ImmGCPtr rhs) { 493 ScratchRegisterScope scratch(asMasm()); 494 MOZ_ASSERT(lhs != scratch); 495 movePtr(rhs, scratch); 496 cmpPtr(lhs, scratch); 497 } cmpPtr(Register lhs,const Imm32 rhs)498 void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); } cmpPtr(const Operand & lhs,const ImmGCPtr rhs)499 void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) { 500 ScratchRegisterScope scratch(asMasm()); 501 MOZ_ASSERT(!lhs.containsReg(scratch)); 502 movePtr(rhs, scratch); 503 cmpPtr(lhs, scratch); 504 } cmpPtr(const Operand & lhs,const ImmWord rhs)505 void cmpPtr(const Operand& lhs, const ImmWord rhs) { 506 if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) { 507 cmpPtr(lhs, Imm32((int32_t)rhs.value)); 508 } else { 509 ScratchRegisterScope scratch(asMasm()); 510 movePtr(rhs, scratch); 511 cmpPtr(lhs, scratch); 512 } 513 } cmpPtr(const Operand & lhs,const ImmPtr rhs)514 void cmpPtr(const Operand& lhs, const ImmPtr rhs) { 515 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); 516 } cmpPtr(const Address & lhs,const ImmGCPtr rhs)517 void cmpPtr(const Address& lhs, const ImmGCPtr rhs) { 518 cmpPtr(Operand(lhs), rhs); 519 } cmpPtr(const Address & lhs,const ImmWord rhs)520 void cmpPtr(const Address& lhs, const ImmWord rhs) { 521 cmpPtr(Operand(lhs), rhs); 522 } cmpPtr(const Address & lhs,const ImmPtr rhs)523 void cmpPtr(const Address& lhs, const ImmPtr rhs) { 524 cmpPtr(lhs, ImmWord(uintptr_t(rhs.value))); 525 } cmpPtr(const Operand & lhs,Register rhs)526 void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); } cmpPtr(Register lhs,const Operand & rhs)527 void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); } cmpPtr(const Operand & lhs,const Imm32 rhs)528 void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); } cmpPtr(const Address & lhs,Register rhs)529 void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); } cmpPtr(Register lhs,Register rhs)530 void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); } testPtr(Register lhs,Register rhs)531 void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); } testPtr(Register lhs,Imm32 rhs)532 void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); } testPtr(const Operand & lhs,Imm32 rhs)533 void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); } 534 535 ///////////////////////////////////////////////////////////////// 536 // Common interface. 537 ///////////////////////////////////////////////////////////////// 538 movePtr(Register src,Register dest)539 void movePtr(Register src, Register dest) { movq(src, dest); } movePtr(Register src,const Operand & dest)540 void movePtr(Register src, const Operand& dest) { movq(src, dest); } movePtr(ImmWord imm,Register dest)541 void movePtr(ImmWord imm, Register dest) { mov(imm, dest); } movePtr(ImmPtr imm,Register dest)542 void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); } movePtr(wasm::SymbolicAddress imm,Register dest)543 void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); } movePtr(ImmGCPtr imm,Register dest)544 void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); } loadPtr(AbsoluteAddress address,Register dest)545 void loadPtr(AbsoluteAddress address, Register dest) { 546 if (X86Encoding::IsAddressImmediate(address.addr)) { 547 movq(Operand(address), dest); 548 } else { 549 ScratchRegisterScope scratch(asMasm()); 550 mov(ImmPtr(address.addr), scratch); 551 loadPtr(Address(scratch, 0x0), dest); 552 } 553 } loadPtr(const Address & address,Register dest)554 void loadPtr(const Address& address, Register dest) { 555 movq(Operand(address), dest); 556 } load64(const Address & address,Register dest)557 void load64(const Address& address, Register dest) { 558 movq(Operand(address), dest); 559 } loadPtr(const Operand & src,Register dest)560 void loadPtr(const Operand& src, Register dest) { movq(src, dest); } loadPtr(const BaseIndex & src,Register dest)561 void loadPtr(const BaseIndex& src, Register dest) { 562 movq(Operand(src), dest); 563 } loadPrivate(const Address & src,Register dest)564 void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); } load32(AbsoluteAddress address,Register dest)565 void load32(AbsoluteAddress address, Register dest) { 566 if (X86Encoding::IsAddressImmediate(address.addr)) { 567 movl(Operand(address), dest); 568 } else { 569 ScratchRegisterScope scratch(asMasm()); 570 mov(ImmPtr(address.addr), scratch); 571 load32(Address(scratch, 0x0), dest); 572 } 573 } load64(const Address & address,Register64 dest)574 void load64(const Address& address, Register64 dest) { 575 movq(Operand(address), dest.reg); 576 } load64(const BaseIndex & address,Register64 dest)577 void load64(const BaseIndex& address, Register64 dest) { 578 movq(Operand(address), dest.reg); 579 } 580 template <typename S> load64Unaligned(const S & src,Register64 dest)581 void load64Unaligned(const S& src, Register64 dest) { 582 load64(src, dest); 583 } 584 template <typename T> storePtr(ImmWord imm,T address)585 void storePtr(ImmWord imm, T address) { 586 if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) { 587 movq(Imm32((int32_t)imm.value), Operand(address)); 588 } else { 589 ScratchRegisterScope scratch(asMasm()); 590 mov(imm, scratch); 591 movq(scratch, Operand(address)); 592 } 593 } 594 template <typename T> storePtr(ImmPtr imm,T address)595 void storePtr(ImmPtr imm, T address) { 596 storePtr(ImmWord(uintptr_t(imm.value)), address); 597 } 598 template <typename T> storePtr(ImmGCPtr imm,T address)599 void storePtr(ImmGCPtr imm, T address) { 600 ScratchRegisterScope scratch(asMasm()); 601 movq(imm, scratch); 602 movq(scratch, Operand(address)); 603 } storePtr(Register src,const Address & address)604 void storePtr(Register src, const Address& address) { 605 movq(src, Operand(address)); 606 } store64(Register src,const Address & address)607 void store64(Register src, const Address& address) { 608 movq(src, Operand(address)); 609 } storePtr(Register src,const BaseIndex & address)610 void storePtr(Register src, const BaseIndex& address) { 611 movq(src, Operand(address)); 612 } storePtr(Register src,const Operand & dest)613 void storePtr(Register src, const Operand& dest) { movq(src, dest); } storePtr(Register src,AbsoluteAddress address)614 void storePtr(Register src, AbsoluteAddress address) { 615 if (X86Encoding::IsAddressImmediate(address.addr)) { 616 movq(src, Operand(address)); 617 } else { 618 ScratchRegisterScope scratch(asMasm()); 619 mov(ImmPtr(address.addr), scratch); 620 storePtr(src, Address(scratch, 0x0)); 621 } 622 } store32(Register src,AbsoluteAddress address)623 void store32(Register src, AbsoluteAddress address) { 624 if (X86Encoding::IsAddressImmediate(address.addr)) { 625 movl(src, Operand(address)); 626 } else { 627 ScratchRegisterScope scratch(asMasm()); 628 mov(ImmPtr(address.addr), scratch); 629 store32(src, Address(scratch, 0x0)); 630 } 631 } store16(Register src,AbsoluteAddress address)632 void store16(Register src, AbsoluteAddress address) { 633 if (X86Encoding::IsAddressImmediate(address.addr)) { 634 movw(src, Operand(address)); 635 } else { 636 ScratchRegisterScope scratch(asMasm()); 637 mov(ImmPtr(address.addr), scratch); 638 store16(src, Address(scratch, 0x0)); 639 } 640 } store64(Register64 src,Address address)641 void store64(Register64 src, Address address) { storePtr(src.reg, address); } store64(Register64 src,const BaseIndex & address)642 void store64(Register64 src, const BaseIndex& address) { 643 storePtr(src.reg, address); 644 } store64(Imm64 imm,Address address)645 void store64(Imm64 imm, Address address) { 646 storePtr(ImmWord(imm.value), address); 647 } store64(Imm64 imm,const BaseIndex & address)648 void store64(Imm64 imm, const BaseIndex& address) { 649 storePtr(ImmWord(imm.value), address); 650 } 651 template <typename S, typename T> store64Unaligned(const S & src,const T & dest)652 void store64Unaligned(const S& src, const T& dest) { 653 store64(src, dest); 654 } 655 splitTag(Register src,Register dest)656 void splitTag(Register src, Register dest) { 657 if (src != dest) { 658 movq(src, dest); 659 } 660 shrq(Imm32(JSVAL_TAG_SHIFT), dest); 661 } splitTag(const ValueOperand & operand,Register dest)662 void splitTag(const ValueOperand& operand, Register dest) { 663 splitTag(operand.valueReg(), dest); 664 } splitTag(const Operand & operand,Register dest)665 void splitTag(const Operand& operand, Register dest) { 666 movq(operand, dest); 667 shrq(Imm32(JSVAL_TAG_SHIFT), dest); 668 } splitTag(const Address & operand,Register dest)669 void splitTag(const Address& operand, Register dest) { 670 splitTag(Operand(operand), dest); 671 } splitTag(const BaseIndex & operand,Register dest)672 void splitTag(const BaseIndex& operand, Register dest) { 673 splitTag(Operand(operand), dest); 674 } 675 676 // Extracts the tag of a value and places it in tag. splitTagForTest(const ValueOperand & value,ScratchTagScope & tag)677 void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) { 678 splitTag(value, tag); 679 } cmpTag(const ValueOperand & operand,ImmTag tag)680 void cmpTag(const ValueOperand& operand, ImmTag tag) { 681 ScratchTagScope reg(asMasm(), operand); 682 splitTagForTest(operand, reg); 683 cmp32(reg, tag); 684 } 685 testMagic(Condition cond,const ValueOperand & src)686 Condition testMagic(Condition cond, const ValueOperand& src) { 687 ScratchTagScope scratch(asMasm(), src); 688 splitTagForTest(src, scratch); 689 return testMagic(cond, scratch); 690 } testError(Condition cond,const ValueOperand & src)691 Condition testError(Condition cond, const ValueOperand& src) { 692 return testMagic(cond, src); 693 } 694 testNullSet(Condition cond,const ValueOperand & value,Register dest)695 void testNullSet(Condition cond, const ValueOperand& value, Register dest) { 696 cond = testNull(cond, value); 697 emitSet(cond, dest); 698 } 699 testObjectSet(Condition cond,const ValueOperand & value,Register dest)700 void testObjectSet(Condition cond, const ValueOperand& value, Register dest) { 701 cond = testObject(cond, value); 702 emitSet(cond, dest); 703 } 704 testUndefinedSet(Condition cond,const ValueOperand & value,Register dest)705 void testUndefinedSet(Condition cond, const ValueOperand& value, 706 Register dest) { 707 cond = testUndefined(cond, value); 708 emitSet(cond, dest); 709 } 710 boxDouble(FloatRegister src,const ValueOperand & dest,FloatRegister)711 void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) { 712 vmovq(src, dest.valueReg()); 713 } boxNonDouble(JSValueType type,Register src,const ValueOperand & dest)714 void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) { 715 MOZ_ASSERT(src != dest.valueReg()); 716 boxValue(type, src, dest.valueReg()); 717 } 718 719 // Note that the |dest| register here may be ScratchReg, so we shouldn't 720 // use it. unboxInt32(const ValueOperand & src,Register dest)721 void unboxInt32(const ValueOperand& src, Register dest) { 722 movl(src.valueReg(), dest); 723 } unboxInt32(const Operand & src,Register dest)724 void unboxInt32(const Operand& src, Register dest) { movl(src, dest); } unboxInt32(const Address & src,Register dest)725 void unboxInt32(const Address& src, Register dest) { 726 unboxInt32(Operand(src), dest); 727 } unboxInt32(const BaseIndex & src,Register dest)728 void unboxInt32(const BaseIndex& src, Register dest) { 729 unboxInt32(Operand(src), dest); 730 } 731 template <typename T> unboxDouble(const T & src,FloatRegister dest)732 void unboxDouble(const T& src, FloatRegister dest) { 733 loadDouble(Operand(src), dest); 734 } 735 unboxArgObjMagic(const ValueOperand & src,Register dest)736 void unboxArgObjMagic(const ValueOperand& src, Register dest) { 737 unboxArgObjMagic(Operand(src.valueReg()), dest); 738 } unboxArgObjMagic(const Operand & src,Register dest)739 void unboxArgObjMagic(const Operand& src, Register dest) { 740 mov(ImmWord(0), dest); 741 } unboxArgObjMagic(const Address & src,Register dest)742 void unboxArgObjMagic(const Address& src, Register dest) { 743 unboxArgObjMagic(Operand(src), dest); 744 } 745 unboxBoolean(const ValueOperand & src,Register dest)746 void unboxBoolean(const ValueOperand& src, Register dest) { 747 movl(src.valueReg(), dest); 748 } unboxBoolean(const Operand & src,Register dest)749 void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); } unboxBoolean(const Address & src,Register dest)750 void unboxBoolean(const Address& src, Register dest) { 751 unboxBoolean(Operand(src), dest); 752 } unboxBoolean(const BaseIndex & src,Register dest)753 void unboxBoolean(const BaseIndex& src, Register dest) { 754 unboxBoolean(Operand(src), dest); 755 } 756 unboxMagic(const ValueOperand & src,Register dest)757 void unboxMagic(const ValueOperand& src, Register dest) { 758 movl(src.valueReg(), dest); 759 } 760 unboxDouble(const ValueOperand & src,FloatRegister dest)761 void unboxDouble(const ValueOperand& src, FloatRegister dest) { 762 vmovq(src.valueReg(), dest); 763 } 764 notBoolean(const ValueOperand & val)765 void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); } 766 unboxNonDouble(const ValueOperand & src,Register dest,JSValueType type)767 void unboxNonDouble(const ValueOperand& src, Register dest, 768 JSValueType type) { 769 MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); 770 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { 771 movl(src.valueReg(), dest); 772 return; 773 } 774 if (src.valueReg() == dest) { 775 ScratchRegisterScope scratch(asMasm()); 776 mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); 777 xorq(scratch, dest); 778 } else { 779 mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest); 780 xorq(src.valueReg(), dest); 781 } 782 } unboxNonDouble(const Operand & src,Register dest,JSValueType type)783 void unboxNonDouble(const Operand& src, Register dest, JSValueType type) { 784 MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE); 785 if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) { 786 movl(src, dest); 787 return; 788 } 789 // Explicitly permits |dest| to be used in |src|. 790 ScratchRegisterScope scratch(asMasm()); 791 MOZ_ASSERT(dest != scratch); 792 if (src.containsReg(dest)) { 793 mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch); 794 // If src is already a register, then src and dest are the same 795 // thing and we don't need to move anything into dest. 796 if (src.kind() != Operand::REG) { 797 movq(src, dest); 798 } 799 xorq(scratch, dest); 800 } else { 801 mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest); 802 xorq(src, dest); 803 } 804 } unboxNonDouble(const Address & src,Register dest,JSValueType type)805 void unboxNonDouble(const Address& src, Register dest, JSValueType type) { 806 unboxNonDouble(Operand(src), dest, type); 807 } unboxNonDouble(const BaseIndex & src,Register dest,JSValueType type)808 void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) { 809 unboxNonDouble(Operand(src), dest, type); 810 } 811 unboxString(const ValueOperand & src,Register dest)812 void unboxString(const ValueOperand& src, Register dest) { 813 unboxNonDouble(src, dest, JSVAL_TYPE_STRING); 814 } unboxString(const Operand & src,Register dest)815 void unboxString(const Operand& src, Register dest) { 816 unboxNonDouble(src, dest, JSVAL_TYPE_STRING); 817 } unboxString(const Address & src,Register dest)818 void unboxString(const Address& src, Register dest) { 819 unboxNonDouble(src, dest, JSVAL_TYPE_STRING); 820 } 821 unboxSymbol(const ValueOperand & src,Register dest)822 void unboxSymbol(const ValueOperand& src, Register dest) { 823 unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); 824 } unboxSymbol(const Operand & src,Register dest)825 void unboxSymbol(const Operand& src, Register dest) { 826 unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL); 827 } 828 unboxBigInt(const ValueOperand & src,Register dest)829 void unboxBigInt(const ValueOperand& src, Register dest) { 830 unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); 831 } unboxBigInt(const Operand & src,Register dest)832 void unboxBigInt(const Operand& src, Register dest) { 833 unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); 834 } unboxBigInt(const Address & src,Register dest)835 void unboxBigInt(const Address& src, Register dest) { 836 unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT); 837 } 838 unboxObject(const ValueOperand & src,Register dest)839 void unboxObject(const ValueOperand& src, Register dest) { 840 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 841 } unboxObject(const Operand & src,Register dest)842 void unboxObject(const Operand& src, Register dest) { 843 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 844 } unboxObject(const Address & src,Register dest)845 void unboxObject(const Address& src, Register dest) { 846 unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT); 847 } unboxObject(const BaseIndex & src,Register dest)848 void unboxObject(const BaseIndex& src, Register dest) { 849 unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT); 850 } 851 852 template <typename T> unboxObjectOrNull(const T & src,Register dest)853 void unboxObjectOrNull(const T& src, Register dest) { 854 unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT); 855 ScratchRegisterScope scratch(asMasm()); 856 mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch); 857 andq(scratch, dest); 858 } 859 860 // This should only be used for GC barrier code, to unbox a GC thing Value. 861 // It's fine there because we don't depend on the actual Value type (all Cells 862 // are treated the same way). In almost all other cases this would be 863 // Spectre-unsafe - use unboxNonDouble and friends instead. unboxGCThingForGCBarrier(const Address & src,Register dest)864 void unboxGCThingForGCBarrier(const Address& src, Register dest) { 865 movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest); 866 andq(Operand(src), dest); 867 } unboxGCThingForGCBarrier(const ValueOperand & src,Register dest)868 void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) { 869 MOZ_ASSERT(src.valueReg() != dest); 870 movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest); 871 andq(src.valueReg(), dest); 872 } 873 874 inline void fallibleUnboxPtrImpl(const Operand& src, Register dest, 875 JSValueType type, Label* fail); 876 877 // Extended unboxing API. If the payload is already in a register, returns 878 // that register. Otherwise, provides a move to the given scratch register, 879 // and returns that. extractObject(const Address & address,Register scratch)880 MOZ_MUST_USE Register extractObject(const Address& address, 881 Register scratch) { 882 MOZ_ASSERT(scratch != ScratchReg); 883 unboxObject(address, scratch); 884 return scratch; 885 } extractObject(const ValueOperand & value,Register scratch)886 MOZ_MUST_USE Register extractObject(const ValueOperand& value, 887 Register scratch) { 888 MOZ_ASSERT(scratch != ScratchReg); 889 unboxObject(value, scratch); 890 return scratch; 891 } extractSymbol(const ValueOperand & value,Register scratch)892 MOZ_MUST_USE Register extractSymbol(const ValueOperand& value, 893 Register scratch) { 894 MOZ_ASSERT(scratch != ScratchReg); 895 unboxSymbol(value, scratch); 896 return scratch; 897 } extractInt32(const ValueOperand & value,Register scratch)898 MOZ_MUST_USE Register extractInt32(const ValueOperand& value, 899 Register scratch) { 900 MOZ_ASSERT(scratch != ScratchReg); 901 unboxInt32(value, scratch); 902 return scratch; 903 } extractBoolean(const ValueOperand & value,Register scratch)904 MOZ_MUST_USE Register extractBoolean(const ValueOperand& value, 905 Register scratch) { 906 MOZ_ASSERT(scratch != ScratchReg); 907 unboxBoolean(value, scratch); 908 return scratch; 909 } extractTag(const Address & address,Register scratch)910 MOZ_MUST_USE Register extractTag(const Address& address, Register scratch) { 911 MOZ_ASSERT(scratch != ScratchReg); 912 loadPtr(address, scratch); 913 splitTag(scratch, scratch); 914 return scratch; 915 } extractTag(const ValueOperand & value,Register scratch)916 MOZ_MUST_USE Register extractTag(const ValueOperand& value, 917 Register scratch) { 918 MOZ_ASSERT(scratch != ScratchReg); 919 splitTag(value, scratch); 920 return scratch; 921 } 922 923 inline void unboxValue(const ValueOperand& src, AnyRegister dest, 924 JSValueType type); 925 926 // These two functions use the low 32-bits of the full value register. boolValueToDouble(const ValueOperand & operand,FloatRegister dest)927 void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) { 928 convertInt32ToDouble(operand.valueReg(), dest); 929 } int32ValueToDouble(const ValueOperand & operand,FloatRegister dest)930 void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) { 931 convertInt32ToDouble(operand.valueReg(), dest); 932 } 933 boolValueToFloat32(const ValueOperand & operand,FloatRegister dest)934 void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) { 935 convertInt32ToFloat32(operand.valueReg(), dest); 936 } int32ValueToFloat32(const ValueOperand & operand,FloatRegister dest)937 void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) { 938 convertInt32ToFloat32(operand.valueReg(), dest); 939 } 940 941 void loadConstantDouble(double d, FloatRegister dest); 942 void loadConstantFloat32(float f, FloatRegister dest); 943 944 void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest); 945 void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest); 946 void vpandSimd128(const SimdConstant& v, FloatRegister dest); 947 void rightShiftInt64x2(Imm32 count, FloatRegister src, FloatRegister dest); 948 loadWasmGlobalPtr(uint32_t globalDataOffset,Register dest)949 void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest) { 950 loadPtr(Address(WasmTlsReg, 951 offsetof(wasm::TlsData, globalArea) + globalDataOffset), 952 dest); 953 } loadWasmPinnedRegsFromTls()954 void loadWasmPinnedRegsFromTls() { 955 loadPtr(Address(WasmTlsReg, offsetof(wasm::TlsData, memoryBase)), HeapReg); 956 } 957 958 public: testInt32Truthy(bool truthy,const ValueOperand & operand)959 Condition testInt32Truthy(bool truthy, const ValueOperand& operand) { 960 test32(operand.valueReg(), operand.valueReg()); 961 return truthy ? NonZero : Zero; 962 } testStringTruthy(bool truthy,const ValueOperand & value)963 Condition testStringTruthy(bool truthy, const ValueOperand& value) { 964 ScratchRegisterScope scratch(asMasm()); 965 unboxString(value, scratch); 966 cmp32(Operand(scratch, JSString::offsetOfLength()), Imm32(0)); 967 return truthy ? Assembler::NotEqual : Assembler::Equal; 968 } testBigIntTruthy(bool truthy,const ValueOperand & value)969 Condition testBigIntTruthy(bool truthy, const ValueOperand& value) { 970 ScratchRegisterScope scratch(asMasm()); 971 unboxBigInt(value, scratch); 972 cmp32(Operand(scratch, BigInt::offsetOfDigitLength()), Imm32(0)); 973 return truthy ? Assembler::NotEqual : Assembler::Equal; 974 } 975 976 template <typename T> 977 inline void loadInt32OrDouble(const T& src, FloatRegister dest); 978 979 template <typename T> loadUnboxedValue(const T & src,MIRType type,AnyRegister dest)980 void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) { 981 if (dest.isFloat()) { 982 loadInt32OrDouble(src, dest.fpu()); 983 } else if (type == MIRType::ObjectOrNull) { 984 unboxObjectOrNull(src, dest.gpr()); 985 } else { 986 unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type)); 987 } 988 } 989 990 template <typename T> storeUnboxedPayload(ValueOperand value,T address,size_t nbytes,JSValueType type)991 void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes, 992 JSValueType type) { 993 switch (nbytes) { 994 case 8: { 995 ScratchRegisterScope scratch(asMasm()); 996 unboxNonDouble(value, scratch, type); 997 storePtr(scratch, address); 998 if (type == JSVAL_TYPE_OBJECT) { 999 // Ideally we would call unboxObjectOrNull, but we need an extra 1000 // scratch register for that. So unbox as object, then clear the 1001 // object-or-null bit. 1002 mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch); 1003 andq(scratch, Operand(address)); 1004 } 1005 return; 1006 } 1007 case 4: 1008 store32(value.valueReg(), address); 1009 return; 1010 case 1: 1011 store8(value.valueReg(), address); 1012 return; 1013 default: 1014 MOZ_CRASH("Bad payload width"); 1015 } 1016 } 1017 loadInstructionPointerAfterCall(Register dest)1018 void loadInstructionPointerAfterCall(Register dest) { 1019 loadPtr(Address(StackPointer, 0x0), dest); 1020 } 1021 convertUInt32ToDouble(Register src,FloatRegister dest)1022 void convertUInt32ToDouble(Register src, FloatRegister dest) { 1023 // Zero the output register to break dependencies, see convertInt32ToDouble. 1024 zeroDouble(dest); 1025 1026 vcvtsq2sd(src, dest, dest); 1027 } 1028 convertUInt32ToFloat32(Register src,FloatRegister dest)1029 void convertUInt32ToFloat32(Register src, FloatRegister dest) { 1030 // Zero the output register to break dependencies, see convertInt32ToDouble. 1031 zeroDouble(dest); 1032 1033 vcvtsq2ss(src, dest, dest); 1034 } 1035 1036 inline void incrementInt32Value(const Address& addr); 1037 1038 inline void ensureDouble(const ValueOperand& source, FloatRegister dest, 1039 Label* failure); 1040 1041 public: 1042 void handleFailureWithHandlerTail(void* handler, Label* profilerExitTail); 1043 1044 // Instrumentation for entering and leaving the profiler. 1045 void profilerEnterFrame(Register framePtr, Register scratch); 1046 void profilerExitFrame(); 1047 }; 1048 1049 using MacroAssemblerSpecific = MacroAssemblerX64; 1050 1051 } // namespace jit 1052 } // namespace js 1053 1054 #endif /* jit_x64_MacroAssembler_x64_h */ 1055