1 // Copyright (c) 1994-2006 Sun Microsystems Inc.
2 // All Rights Reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions
6 // are met:
7 //
8 // - Redistributions of source code must retain the above copyright notice,
9 // this list of conditions and the following disclaimer.
10 //
11 // - Redistribution in binary form must reproduce the above copyright
12 // notice, this list of conditions and the following disclaimer in the
13 // documentation and/or other materials provided with the
14 // distribution.
15 //
16 // - Neither the name of Sun Microsystems or the names of contributors may
17 // be used to endorse or promote products derived from this software without
18 // specific prior written permission.
19 //
20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 // COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
26 // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
27 // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 // HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
29 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
31 // OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 // The original source code covered by the above license above has been modified
34 // significantly by Google Inc.
35 // Copyright 2012 the V8 project authors. All rights reserved.
36
37 #include "src/codegen/ia32/assembler-ia32.h"
38
39 #include <cstring>
40
41 #if V8_TARGET_ARCH_IA32
42
43 #if V8_LIBC_MSVCRT
44 #include <intrin.h> // _xgetbv()
45 #endif
46 #if V8_OS_MACOSX
47 #include <sys/sysctl.h>
48 #endif
49
50 #include "src/base/bits.h"
51 #include "src/base/cpu.h"
52 #include "src/codegen/assembler-inl.h"
53 #include "src/codegen/macro-assembler.h"
54 #include "src/codegen/string-constants.h"
55 #include "src/deoptimizer/deoptimizer.h"
56 #include "src/diagnostics/disassembler.h"
57 #include "src/init/v8.h"
58 #include "src/numbers/conversions-inl.h"
59
60 namespace v8 {
61 namespace internal {
62
EmbeddedNumber(double value)63 Immediate Immediate::EmbeddedNumber(double value) {
64 int32_t smi;
65 if (DoubleToSmiInteger(value, &smi)) return Immediate(Smi::FromInt(smi));
66 Immediate result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
67 result.is_heap_object_request_ = true;
68 result.value_.heap_object_request = HeapObjectRequest(value);
69 return result;
70 }
71
EmbeddedStringConstant(const StringConstantBase * str)72 Immediate Immediate::EmbeddedStringConstant(const StringConstantBase* str) {
73 Immediate result(0, RelocInfo::FULL_EMBEDDED_OBJECT);
74 result.is_heap_object_request_ = true;
75 result.value_.heap_object_request = HeapObjectRequest(str);
76 return result;
77 }
78
79 // -----------------------------------------------------------------------------
80 // Implementation of CpuFeatures
81
82 namespace {
83
84 #if !V8_LIBC_MSVCRT
85
_xgetbv(unsigned int xcr)86 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
87 unsigned eax, edx;
88 // Check xgetbv; this uses a .byte sequence instead of the instruction
89 // directly because older assemblers do not include support for xgetbv and
90 // there is no easy way to conditionally compile based on the assembler
91 // used.
92 __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
93 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
94 }
95
96 #define _XCR_XFEATURE_ENABLED_MASK 0
97
98 #endif // !V8_LIBC_MSVCRT
99
OSHasAVXSupport()100 bool OSHasAVXSupport() {
101 #if V8_OS_MACOSX
102 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
103 // caused by ISRs, so we detect that here and disable AVX in that case.
104 char buffer[128];
105 size_t buffer_size = arraysize(buffer);
106 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
107 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
108 FATAL("V8 failed to get kernel version");
109 }
110 // The buffer now contains a string of the form XX.YY.ZZ, where
111 // XX is the major kernel version component.
112 char* period_pos = strchr(buffer, '.');
113 DCHECK_NOT_NULL(period_pos);
114 *period_pos = '\0';
115 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
116 if (kernel_version_major <= 13) return false;
117 #endif // V8_OS_MACOSX
118 // Check whether OS claims to support AVX.
119 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
120 return (feature_mask & 0x6) == 0x6;
121 }
122
123 #undef _XCR_XFEATURE_ENABLED_MASK
124
125 } // namespace
126
ProbeImpl(bool cross_compile)127 void CpuFeatures::ProbeImpl(bool cross_compile) {
128 base::CPU cpu;
129 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
130 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
131
132 // Only use statically determined features for cross compile (snapshot).
133 if (cross_compile) return;
134
135 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
136 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
137 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
138 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
139 OSHasAVXSupport()) {
140 supported_ |= 1u << AVX;
141 }
142 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
143 OSHasAVXSupport()) {
144 supported_ |= 1u << FMA3;
145 }
146 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
147 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
148 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
149 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
150 if (strcmp(FLAG_mcpu, "auto") == 0) {
151 if (cpu.is_atom()) supported_ |= 1u << ATOM;
152 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
153 supported_ |= 1u << ATOM;
154 }
155 }
156
PrintTarget()157 void CpuFeatures::PrintTarget() {}
PrintFeatures()158 void CpuFeatures::PrintFeatures() {
159 printf(
160 "SSE3=%d SSSE3=%d SSE4_1=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d "
161 "POPCNT=%d ATOM=%d\n",
162 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
163 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(AVX),
164 CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
165 CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
166 CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
167 }
168
169 // -----------------------------------------------------------------------------
170 // Implementation of Displacement
171
init(Label * L,Type type)172 void Displacement::init(Label* L, Type type) {
173 DCHECK(!L->is_bound());
174 int next = 0;
175 if (L->is_linked()) {
176 next = L->pos();
177 DCHECK_GT(next, 0); // Displacements must be at positions > 0
178 }
179 // Ensure that we _never_ overflow the next field.
180 DCHECK(NextField::is_valid(Assembler::kMaximalBufferSize));
181 data_ = NextField::encode(next) | TypeField::encode(type);
182 }
183
184 // -----------------------------------------------------------------------------
185 // Implementation of RelocInfo
186
187 const int RelocInfo::kApplyMask =
188 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
189 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
190 RelocInfo::ModeMask(RelocInfo::OFF_HEAP_TARGET) |
191 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY);
192
IsCodedSpecially()193 bool RelocInfo::IsCodedSpecially() {
194 // The deserializer needs to know whether a pointer is specially coded. Being
195 // specially coded on IA32 means that it is a relative address, as used by
196 // branch instructions. These are also the ones that need changing when a
197 // code object moves.
198 return RelocInfo::ModeMask(rmode_) & kApplyMask;
199 }
200
IsInConstantPool()201 bool RelocInfo::IsInConstantPool() { return false; }
202
wasm_call_tag() const203 uint32_t RelocInfo::wasm_call_tag() const {
204 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
205 return ReadUnalignedValue<uint32_t>(pc_);
206 }
207
208 // -----------------------------------------------------------------------------
209 // Implementation of Operand
210
Operand(Register base,int32_t disp,RelocInfo::Mode rmode)211 Operand::Operand(Register base, int32_t disp, RelocInfo::Mode rmode) {
212 // [base + disp/r]
213 if (disp == 0 && RelocInfo::IsNone(rmode) && base != ebp) {
214 // [base]
215 set_modrm(0, base);
216 if (base == esp) set_sib(times_1, esp, base);
217 } else if (is_int8(disp) && RelocInfo::IsNone(rmode)) {
218 // [base + disp8]
219 set_modrm(1, base);
220 if (base == esp) set_sib(times_1, esp, base);
221 set_disp8(disp);
222 } else {
223 // [base + disp/r]
224 set_modrm(2, base);
225 if (base == esp) set_sib(times_1, esp, base);
226 set_dispr(disp, rmode);
227 }
228 }
229
Operand(Register base,Register index,ScaleFactor scale,int32_t disp,RelocInfo::Mode rmode)230 Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp,
231 RelocInfo::Mode rmode) {
232 DCHECK(index != esp); // illegal addressing mode
233 // [base + index*scale + disp/r]
234 if (disp == 0 && RelocInfo::IsNone(rmode) && base != ebp) {
235 // [base + index*scale]
236 set_modrm(0, esp);
237 set_sib(scale, index, base);
238 } else if (is_int8(disp) && RelocInfo::IsNone(rmode)) {
239 // [base + index*scale + disp8]
240 set_modrm(1, esp);
241 set_sib(scale, index, base);
242 set_disp8(disp);
243 } else {
244 // [base + index*scale + disp/r]
245 set_modrm(2, esp);
246 set_sib(scale, index, base);
247 set_dispr(disp, rmode);
248 }
249 }
250
Operand(Register index,ScaleFactor scale,int32_t disp,RelocInfo::Mode rmode)251 Operand::Operand(Register index, ScaleFactor scale, int32_t disp,
252 RelocInfo::Mode rmode) {
253 DCHECK(index != esp); // illegal addressing mode
254 // [index*scale + disp/r]
255 set_modrm(0, esp);
256 set_sib(scale, index, ebp);
257 set_dispr(disp, rmode);
258 }
259
is_reg_only() const260 bool Operand::is_reg_only() const {
261 return (buf_[0] & 0xF8) == 0xC0; // Addressing mode is register only.
262 }
263
reg() const264 Register Operand::reg() const {
265 DCHECK(is_reg_only());
266 return Register::from_code(buf_[0] & 0x07);
267 }
268
AllocateAndInstallRequestedHeapObjects(Isolate * isolate)269 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
270 DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
271 for (auto& request : heap_object_requests_) {
272 Handle<HeapObject> object;
273 switch (request.kind()) {
274 case HeapObjectRequest::kHeapNumber:
275 object = isolate->factory()->NewHeapNumber<AllocationType::kOld>(
276 request.heap_number());
277 break;
278 case HeapObjectRequest::kStringConstant: {
279 const StringConstantBase* str = request.string();
280 CHECK_NOT_NULL(str);
281 object = str->AllocateStringConstant(isolate);
282 break;
283 }
284 }
285 Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
286 WriteUnalignedValue(pc, object);
287 }
288 }
289
290 // -----------------------------------------------------------------------------
291 // Implementation of Assembler.
292
293 // Emit a single byte. Must always be inlined.
294 #define EMIT(x) *pc_++ = (x)
295
Assembler(const AssemblerOptions & options,std::unique_ptr<AssemblerBuffer> buffer)296 Assembler::Assembler(const AssemblerOptions& options,
297 std::unique_ptr<AssemblerBuffer> buffer)
298 : AssemblerBase(options, std::move(buffer)) {
299 reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
300 }
301
GetCode(Isolate * isolate,CodeDesc * desc,SafepointTableBuilder * safepoint_table_builder,int handler_table_offset)302 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
303 SafepointTableBuilder* safepoint_table_builder,
304 int handler_table_offset) {
305 // As a crutch to avoid having to add manual Align calls wherever we use a
306 // raw workflow to create Code objects (mostly in tests), add another Align
307 // call here. It does no harm - the end of the Code object is aligned to the
308 // (larger) kCodeAlignment anyways.
309 // TODO(jgruber): Consider moving responsibility for proper alignment to
310 // metadata table builders (safepoint, handler, constant pool, code
311 // comments).
312 DataAlign(Code::kMetadataAlignment);
313
314 const int code_comments_size = WriteCodeComments();
315
316 // Finalize code (at this point overflow() may be true, but the gap ensures
317 // that we are still not overlapping instructions and relocation info).
318 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
319
320 AllocateAndInstallRequestedHeapObjects(isolate);
321
322 // Set up code descriptor.
323 // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
324 // this point to make CodeDesc initialization less fiddly.
325
326 static constexpr int kConstantPoolSize = 0;
327 const int instruction_size = pc_offset();
328 const int code_comments_offset = instruction_size - code_comments_size;
329 const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
330 const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
331 ? constant_pool_offset
332 : handler_table_offset;
333 const int safepoint_table_offset =
334 (safepoint_table_builder == kNoSafepointTable)
335 ? handler_table_offset2
336 : safepoint_table_builder->GetCodeOffset();
337 const int reloc_info_offset =
338 static_cast<int>(reloc_info_writer.pos() - buffer_->start());
339 CodeDesc::Initialize(desc, this, safepoint_table_offset,
340 handler_table_offset2, constant_pool_offset,
341 code_comments_offset, reloc_info_offset);
342 }
343
FinalizeJumpOptimizationInfo()344 void Assembler::FinalizeJumpOptimizationInfo() {
345 // Collection stage
346 auto jump_opt = jump_optimization_info();
347 if (jump_opt && jump_opt->is_collecting()) {
348 auto& bitmap = jump_opt->farjmp_bitmap();
349 int num = static_cast<int>(farjmp_positions_.size());
350 if (num && bitmap.empty()) {
351 bool can_opt = false;
352
353 bitmap.resize((num + 31) / 32, 0);
354 for (int i = 0; i < num; i++) {
355 int disp_pos = farjmp_positions_[i];
356 int disp = long_at(disp_pos);
357 if (is_int8(disp)) {
358 bitmap[i / 32] |= 1 << (i & 31);
359 can_opt = true;
360 }
361 }
362 if (can_opt) {
363 jump_opt->set_optimizable();
364 }
365 }
366 }
367 }
368
Align(int m)369 void Assembler::Align(int m) {
370 DCHECK(base::bits::IsPowerOfTwo(m));
371 int mask = m - 1;
372 int addr = pc_offset();
373 Nop((m - (addr & mask)) & mask);
374 }
375
IsNop(Address addr)376 bool Assembler::IsNop(Address addr) {
377 byte* a = reinterpret_cast<byte*>(addr);
378 while (*a == 0x66) a++;
379 if (*a == 0x90) return true;
380 if (a[0] == 0xF && a[1] == 0x1F) return true;
381 return false;
382 }
383
Nop(int bytes)384 void Assembler::Nop(int bytes) {
385 EnsureSpace ensure_space(this);
386 // Multi byte nops from http://support.amd.com/us/Processor_TechDocs/40546.pdf
387 while (bytes > 0) {
388 switch (bytes) {
389 case 2:
390 EMIT(0x66);
391 V8_FALLTHROUGH;
392 case 1:
393 EMIT(0x90);
394 return;
395 case 3:
396 EMIT(0xF);
397 EMIT(0x1F);
398 EMIT(0);
399 return;
400 case 4:
401 EMIT(0xF);
402 EMIT(0x1F);
403 EMIT(0x40);
404 EMIT(0);
405 return;
406 case 6:
407 EMIT(0x66);
408 V8_FALLTHROUGH;
409 case 5:
410 EMIT(0xF);
411 EMIT(0x1F);
412 EMIT(0x44);
413 EMIT(0);
414 EMIT(0);
415 return;
416 case 7:
417 EMIT(0xF);
418 EMIT(0x1F);
419 EMIT(0x80);
420 EMIT(0);
421 EMIT(0);
422 EMIT(0);
423 EMIT(0);
424 return;
425 default:
426 case 11:
427 EMIT(0x66);
428 bytes--;
429 V8_FALLTHROUGH;
430 case 10:
431 EMIT(0x66);
432 bytes--;
433 V8_FALLTHROUGH;
434 case 9:
435 EMIT(0x66);
436 bytes--;
437 V8_FALLTHROUGH;
438 case 8:
439 EMIT(0xF);
440 EMIT(0x1F);
441 EMIT(0x84);
442 EMIT(0);
443 EMIT(0);
444 EMIT(0);
445 EMIT(0);
446 EMIT(0);
447 bytes -= 8;
448 }
449 }
450 }
451
CodeTargetAlign()452 void Assembler::CodeTargetAlign() {
453 Align(16); // Preferred alignment of jump targets on ia32.
454 }
455
cpuid()456 void Assembler::cpuid() {
457 EnsureSpace ensure_space(this);
458 EMIT(0x0F);
459 EMIT(0xA2);
460 }
461
pushad()462 void Assembler::pushad() {
463 EnsureSpace ensure_space(this);
464 EMIT(0x60);
465 }
466
popad()467 void Assembler::popad() {
468 EnsureSpace ensure_space(this);
469 EMIT(0x61);
470 }
471
pushfd()472 void Assembler::pushfd() {
473 EnsureSpace ensure_space(this);
474 EMIT(0x9C);
475 }
476
popfd()477 void Assembler::popfd() {
478 EnsureSpace ensure_space(this);
479 EMIT(0x9D);
480 }
481
push(const Immediate & x)482 void Assembler::push(const Immediate& x) {
483 EnsureSpace ensure_space(this);
484 if (x.is_int8()) {
485 EMIT(0x6A);
486 EMIT(x.immediate());
487 } else {
488 EMIT(0x68);
489 emit(x);
490 }
491 }
492
push_imm32(int32_t imm32)493 void Assembler::push_imm32(int32_t imm32) {
494 EnsureSpace ensure_space(this);
495 EMIT(0x68);
496 emit(imm32);
497 }
498
push(Register src)499 void Assembler::push(Register src) {
500 EnsureSpace ensure_space(this);
501 EMIT(0x50 | src.code());
502 }
503
push(Operand src)504 void Assembler::push(Operand src) {
505 EnsureSpace ensure_space(this);
506 EMIT(0xFF);
507 emit_operand(esi, src);
508 }
509
pop(Register dst)510 void Assembler::pop(Register dst) {
511 DCHECK_NOT_NULL(reloc_info_writer.last_pc());
512 EnsureSpace ensure_space(this);
513 EMIT(0x58 | dst.code());
514 }
515
pop(Operand dst)516 void Assembler::pop(Operand dst) {
517 EnsureSpace ensure_space(this);
518 EMIT(0x8F);
519 emit_operand(eax, dst);
520 }
521
leave()522 void Assembler::leave() {
523 EnsureSpace ensure_space(this);
524 EMIT(0xC9);
525 }
526
mov_b(Register dst,Operand src)527 void Assembler::mov_b(Register dst, Operand src) {
528 CHECK(dst.is_byte_register());
529 EnsureSpace ensure_space(this);
530 EMIT(0x8A);
531 emit_operand(dst, src);
532 }
533
mov_b(Operand dst,const Immediate & src)534 void Assembler::mov_b(Operand dst, const Immediate& src) {
535 EnsureSpace ensure_space(this);
536 EMIT(0xC6);
537 emit_operand(eax, dst);
538 EMIT(static_cast<int8_t>(src.immediate()));
539 }
540
mov_b(Operand dst,Register src)541 void Assembler::mov_b(Operand dst, Register src) {
542 CHECK(src.is_byte_register());
543 EnsureSpace ensure_space(this);
544 EMIT(0x88);
545 emit_operand(src, dst);
546 }
547
mov_w(Register dst,Operand src)548 void Assembler::mov_w(Register dst, Operand src) {
549 EnsureSpace ensure_space(this);
550 EMIT(0x66);
551 EMIT(0x8B);
552 emit_operand(dst, src);
553 }
554
mov_w(Operand dst,Register src)555 void Assembler::mov_w(Operand dst, Register src) {
556 EnsureSpace ensure_space(this);
557 EMIT(0x66);
558 EMIT(0x89);
559 emit_operand(src, dst);
560 }
561
mov_w(Operand dst,const Immediate & src)562 void Assembler::mov_w(Operand dst, const Immediate& src) {
563 EnsureSpace ensure_space(this);
564 EMIT(0x66);
565 EMIT(0xC7);
566 emit_operand(eax, dst);
567 EMIT(static_cast<int8_t>(src.immediate() & 0xFF));
568 EMIT(static_cast<int8_t>(src.immediate() >> 8));
569 }
570
mov(Register dst,int32_t imm32)571 void Assembler::mov(Register dst, int32_t imm32) {
572 EnsureSpace ensure_space(this);
573 EMIT(0xB8 | dst.code());
574 emit(imm32);
575 }
576
mov(Register dst,const Immediate & x)577 void Assembler::mov(Register dst, const Immediate& x) {
578 EnsureSpace ensure_space(this);
579 EMIT(0xB8 | dst.code());
580 emit(x);
581 }
582
mov(Register dst,Handle<HeapObject> handle)583 void Assembler::mov(Register dst, Handle<HeapObject> handle) {
584 EnsureSpace ensure_space(this);
585 EMIT(0xB8 | dst.code());
586 emit(handle);
587 }
588
mov(Register dst,Operand src)589 void Assembler::mov(Register dst, Operand src) {
590 EnsureSpace ensure_space(this);
591 EMIT(0x8B);
592 emit_operand(dst, src);
593 }
594
mov(Register dst,Register src)595 void Assembler::mov(Register dst, Register src) {
596 EnsureSpace ensure_space(this);
597 EMIT(0x89);
598 EMIT(0xC0 | src.code() << 3 | dst.code());
599 }
600
mov(Operand dst,const Immediate & x)601 void Assembler::mov(Operand dst, const Immediate& x) {
602 EnsureSpace ensure_space(this);
603 EMIT(0xC7);
604 emit_operand(eax, dst);
605 emit(x);
606 }
607
mov(Operand dst,Address src,RelocInfo::Mode rmode)608 void Assembler::mov(Operand dst, Address src, RelocInfo::Mode rmode) {
609 EnsureSpace ensure_space(this);
610 EMIT(0xC7);
611 emit_operand(eax, dst);
612 emit(src, rmode);
613 }
614
mov(Operand dst,Handle<HeapObject> handle)615 void Assembler::mov(Operand dst, Handle<HeapObject> handle) {
616 EnsureSpace ensure_space(this);
617 EMIT(0xC7);
618 emit_operand(eax, dst);
619 emit(handle);
620 }
621
mov(Operand dst,Register src)622 void Assembler::mov(Operand dst, Register src) {
623 EnsureSpace ensure_space(this);
624 EMIT(0x89);
625 emit_operand(src, dst);
626 }
627
movsx_b(Register dst,Operand src)628 void Assembler::movsx_b(Register dst, Operand src) {
629 DCHECK_IMPLIES(src.is_reg_only(), src.reg().is_byte_register());
630 EnsureSpace ensure_space(this);
631 EMIT(0x0F);
632 EMIT(0xBE);
633 emit_operand(dst, src);
634 }
635
movsx_w(Register dst,Operand src)636 void Assembler::movsx_w(Register dst, Operand src) {
637 EnsureSpace ensure_space(this);
638 EMIT(0x0F);
639 EMIT(0xBF);
640 emit_operand(dst, src);
641 }
642
movzx_b(Register dst,Operand src)643 void Assembler::movzx_b(Register dst, Operand src) {
644 DCHECK_IMPLIES(src.is_reg_only(), src.reg().is_byte_register());
645 EnsureSpace ensure_space(this);
646 EMIT(0x0F);
647 EMIT(0xB6);
648 emit_operand(dst, src);
649 }
650
movzx_w(Register dst,Operand src)651 void Assembler::movzx_w(Register dst, Operand src) {
652 EnsureSpace ensure_space(this);
653 EMIT(0x0F);
654 EMIT(0xB7);
655 emit_operand(dst, src);
656 }
657
movq(XMMRegister dst,Operand src)658 void Assembler::movq(XMMRegister dst, Operand src) {
659 EnsureSpace ensure_space(this);
660 EMIT(0xF3);
661 EMIT(0x0F);
662 EMIT(0x7E);
663 emit_operand(dst, src);
664 }
665
cmov(Condition cc,Register dst,Operand src)666 void Assembler::cmov(Condition cc, Register dst, Operand src) {
667 EnsureSpace ensure_space(this);
668 // Opcode: 0f 40 + cc /r.
669 EMIT(0x0F);
670 EMIT(0x40 + cc);
671 emit_operand(dst, src);
672 }
673
cld()674 void Assembler::cld() {
675 EnsureSpace ensure_space(this);
676 EMIT(0xFC);
677 }
678
rep_movs()679 void Assembler::rep_movs() {
680 EnsureSpace ensure_space(this);
681 EMIT(0xF3);
682 EMIT(0xA5);
683 }
684
rep_stos()685 void Assembler::rep_stos() {
686 EnsureSpace ensure_space(this);
687 EMIT(0xF3);
688 EMIT(0xAB);
689 }
690
stos()691 void Assembler::stos() {
692 EnsureSpace ensure_space(this);
693 EMIT(0xAB);
694 }
695
xadd(Operand dst,Register src)696 void Assembler::xadd(Operand dst, Register src) {
697 EnsureSpace ensure_space(this);
698 EMIT(0x0F);
699 EMIT(0xC1);
700 emit_operand(src, dst);
701 }
702
xadd_b(Operand dst,Register src)703 void Assembler::xadd_b(Operand dst, Register src) {
704 DCHECK(src.is_byte_register());
705 EnsureSpace ensure_space(this);
706 EMIT(0x0F);
707 EMIT(0xC0);
708 emit_operand(src, dst);
709 }
710
xadd_w(Operand dst,Register src)711 void Assembler::xadd_w(Operand dst, Register src) {
712 EnsureSpace ensure_space(this);
713 EMIT(0x66);
714 EMIT(0x0F);
715 EMIT(0xC1);
716 emit_operand(src, dst);
717 }
718
xchg(Register dst,Register src)719 void Assembler::xchg(Register dst, Register src) {
720 EnsureSpace ensure_space(this);
721 if (src == eax || dst == eax) { // Single-byte encoding.
722 EMIT(0x90 | (src == eax ? dst.code() : src.code()));
723 } else {
724 EMIT(0x87);
725 EMIT(0xC0 | src.code() << 3 | dst.code());
726 }
727 }
728
xchg(Register dst,Operand src)729 void Assembler::xchg(Register dst, Operand src) {
730 EnsureSpace ensure_space(this);
731 EMIT(0x87);
732 emit_operand(dst, src);
733 }
734
xchg_b(Register reg,Operand op)735 void Assembler::xchg_b(Register reg, Operand op) {
736 DCHECK(reg.is_byte_register());
737 EnsureSpace ensure_space(this);
738 EMIT(0x86);
739 emit_operand(reg, op);
740 }
741
xchg_w(Register reg,Operand op)742 void Assembler::xchg_w(Register reg, Operand op) {
743 EnsureSpace ensure_space(this);
744 EMIT(0x66);
745 EMIT(0x87);
746 emit_operand(reg, op);
747 }
748
lock()749 void Assembler::lock() {
750 EnsureSpace ensure_space(this);
751 EMIT(0xF0);
752 }
753
cmpxchg(Operand dst,Register src)754 void Assembler::cmpxchg(Operand dst, Register src) {
755 EnsureSpace ensure_space(this);
756 EMIT(0x0F);
757 EMIT(0xB1);
758 emit_operand(src, dst);
759 }
760
cmpxchg_b(Operand dst,Register src)761 void Assembler::cmpxchg_b(Operand dst, Register src) {
762 DCHECK(src.is_byte_register());
763 EnsureSpace ensure_space(this);
764 EMIT(0x0F);
765 EMIT(0xB0);
766 emit_operand(src, dst);
767 }
768
cmpxchg_w(Operand dst,Register src)769 void Assembler::cmpxchg_w(Operand dst, Register src) {
770 EnsureSpace ensure_space(this);
771 EMIT(0x66);
772 EMIT(0x0F);
773 EMIT(0xB1);
774 emit_operand(src, dst);
775 }
776
cmpxchg8b(Operand dst)777 void Assembler::cmpxchg8b(Operand dst) {
778 EnsureSpace enure_space(this);
779 EMIT(0x0F);
780 EMIT(0xC7);
781 emit_operand(ecx, dst);
782 }
783
mfence()784 void Assembler::mfence() {
785 EnsureSpace ensure_space(this);
786 EMIT(0x0F);
787 EMIT(0xAE);
788 EMIT(0xF0);
789 }
790
lfence()791 void Assembler::lfence() {
792 EnsureSpace ensure_space(this);
793 EMIT(0x0F);
794 EMIT(0xAE);
795 EMIT(0xE8);
796 }
797
pause()798 void Assembler::pause() {
799 EnsureSpace ensure_space(this);
800 EMIT(0xF3);
801 EMIT(0x90);
802 }
803
adc(Register dst,int32_t imm32)804 void Assembler::adc(Register dst, int32_t imm32) {
805 EnsureSpace ensure_space(this);
806 emit_arith(2, Operand(dst), Immediate(imm32));
807 }
808
adc(Register dst,Operand src)809 void Assembler::adc(Register dst, Operand src) {
810 EnsureSpace ensure_space(this);
811 EMIT(0x13);
812 emit_operand(dst, src);
813 }
814
add(Register dst,Operand src)815 void Assembler::add(Register dst, Operand src) {
816 EnsureSpace ensure_space(this);
817 EMIT(0x03);
818 emit_operand(dst, src);
819 }
820
add(Operand dst,Register src)821 void Assembler::add(Operand dst, Register src) {
822 EnsureSpace ensure_space(this);
823 EMIT(0x01);
824 emit_operand(src, dst);
825 }
826
add(Operand dst,const Immediate & x)827 void Assembler::add(Operand dst, const Immediate& x) {
828 DCHECK_NOT_NULL(reloc_info_writer.last_pc());
829 EnsureSpace ensure_space(this);
830 emit_arith(0, dst, x);
831 }
832
and_(Register dst,int32_t imm32)833 void Assembler::and_(Register dst, int32_t imm32) {
834 and_(dst, Immediate(imm32));
835 }
836
and_(Register dst,const Immediate & x)837 void Assembler::and_(Register dst, const Immediate& x) {
838 EnsureSpace ensure_space(this);
839 emit_arith(4, Operand(dst), x);
840 }
841
and_(Register dst,Operand src)842 void Assembler::and_(Register dst, Operand src) {
843 EnsureSpace ensure_space(this);
844 EMIT(0x23);
845 emit_operand(dst, src);
846 }
847
and_(Operand dst,const Immediate & x)848 void Assembler::and_(Operand dst, const Immediate& x) {
849 EnsureSpace ensure_space(this);
850 emit_arith(4, dst, x);
851 }
852
and_(Operand dst,Register src)853 void Assembler::and_(Operand dst, Register src) {
854 EnsureSpace ensure_space(this);
855 EMIT(0x21);
856 emit_operand(src, dst);
857 }
858
cmpb(Operand op,Immediate imm8)859 void Assembler::cmpb(Operand op, Immediate imm8) {
860 DCHECK(imm8.is_int8() || imm8.is_uint8());
861 EnsureSpace ensure_space(this);
862 if (op.is_reg(eax)) {
863 EMIT(0x3C);
864 } else {
865 EMIT(0x80);
866 emit_operand(edi, op); // edi == 7
867 }
868 emit_b(imm8);
869 }
870
cmpb(Operand op,Register reg)871 void Assembler::cmpb(Operand op, Register reg) {
872 CHECK(reg.is_byte_register());
873 EnsureSpace ensure_space(this);
874 EMIT(0x38);
875 emit_operand(reg, op);
876 }
877
cmpb(Register reg,Operand op)878 void Assembler::cmpb(Register reg, Operand op) {
879 CHECK(reg.is_byte_register());
880 EnsureSpace ensure_space(this);
881 EMIT(0x3A);
882 emit_operand(reg, op);
883 }
884
cmpw(Operand op,Immediate imm16)885 void Assembler::cmpw(Operand op, Immediate imm16) {
886 DCHECK(imm16.is_int16() || imm16.is_uint16());
887 EnsureSpace ensure_space(this);
888 EMIT(0x66);
889 EMIT(0x81);
890 emit_operand(edi, op);
891 emit_w(imm16);
892 }
893
cmpw(Register reg,Operand op)894 void Assembler::cmpw(Register reg, Operand op) {
895 EnsureSpace ensure_space(this);
896 EMIT(0x66);
897 EMIT(0x3B);
898 emit_operand(reg, op);
899 }
900
cmpw(Operand op,Register reg)901 void Assembler::cmpw(Operand op, Register reg) {
902 EnsureSpace ensure_space(this);
903 EMIT(0x66);
904 EMIT(0x39);
905 emit_operand(reg, op);
906 }
907
cmp(Register reg,int32_t imm32)908 void Assembler::cmp(Register reg, int32_t imm32) {
909 EnsureSpace ensure_space(this);
910 emit_arith(7, Operand(reg), Immediate(imm32));
911 }
912
cmp(Register reg,Handle<HeapObject> handle)913 void Assembler::cmp(Register reg, Handle<HeapObject> handle) {
914 EnsureSpace ensure_space(this);
915 emit_arith(7, Operand(reg), Immediate(handle));
916 }
917
cmp(Register reg,Operand op)918 void Assembler::cmp(Register reg, Operand op) {
919 EnsureSpace ensure_space(this);
920 EMIT(0x3B);
921 emit_operand(reg, op);
922 }
923
cmp(Operand op,Register reg)924 void Assembler::cmp(Operand op, Register reg) {
925 EnsureSpace ensure_space(this);
926 EMIT(0x39);
927 emit_operand(reg, op);
928 }
929
cmp(Operand op,const Immediate & imm)930 void Assembler::cmp(Operand op, const Immediate& imm) {
931 EnsureSpace ensure_space(this);
932 emit_arith(7, op, imm);
933 }
934
cmp(Operand op,Handle<HeapObject> handle)935 void Assembler::cmp(Operand op, Handle<HeapObject> handle) {
936 EnsureSpace ensure_space(this);
937 emit_arith(7, op, Immediate(handle));
938 }
939
cmpb_al(Operand op)940 void Assembler::cmpb_al(Operand op) {
941 EnsureSpace ensure_space(this);
942 EMIT(0x38); // CMP r/m8, r8
943 emit_operand(eax, op); // eax has same code as register al.
944 }
945
cmpw_ax(Operand op)946 void Assembler::cmpw_ax(Operand op) {
947 EnsureSpace ensure_space(this);
948 EMIT(0x66);
949 EMIT(0x39); // CMP r/m16, r16
950 emit_operand(eax, op); // eax has same code as register ax.
951 }
952
dec_b(Register dst)953 void Assembler::dec_b(Register dst) {
954 CHECK(dst.is_byte_register());
955 EnsureSpace ensure_space(this);
956 EMIT(0xFE);
957 EMIT(0xC8 | dst.code());
958 }
959
dec_b(Operand dst)960 void Assembler::dec_b(Operand dst) {
961 EnsureSpace ensure_space(this);
962 EMIT(0xFE);
963 emit_operand(ecx, dst);
964 }
965
dec(Register dst)966 void Assembler::dec(Register dst) {
967 EnsureSpace ensure_space(this);
968 EMIT(0x48 | dst.code());
969 }
970
dec(Operand dst)971 void Assembler::dec(Operand dst) {
972 EnsureSpace ensure_space(this);
973 EMIT(0xFF);
974 emit_operand(ecx, dst);
975 }
976
cdq()977 void Assembler::cdq() {
978 EnsureSpace ensure_space(this);
979 EMIT(0x99);
980 }
981
idiv(Operand src)982 void Assembler::idiv(Operand src) {
983 EnsureSpace ensure_space(this);
984 EMIT(0xF7);
985 emit_operand(edi, src);
986 }
987
div(Operand src)988 void Assembler::div(Operand src) {
989 EnsureSpace ensure_space(this);
990 EMIT(0xF7);
991 emit_operand(esi, src);
992 }
993
imul(Register reg)994 void Assembler::imul(Register reg) {
995 EnsureSpace ensure_space(this);
996 EMIT(0xF7);
997 EMIT(0xE8 | reg.code());
998 }
999
imul(Register dst,Operand src)1000 void Assembler::imul(Register dst, Operand src) {
1001 EnsureSpace ensure_space(this);
1002 EMIT(0x0F);
1003 EMIT(0xAF);
1004 emit_operand(dst, src);
1005 }
1006
imul(Register dst,Register src,int32_t imm32)1007 void Assembler::imul(Register dst, Register src, int32_t imm32) {
1008 imul(dst, Operand(src), imm32);
1009 }
1010
imul(Register dst,Operand src,int32_t imm32)1011 void Assembler::imul(Register dst, Operand src, int32_t imm32) {
1012 EnsureSpace ensure_space(this);
1013 if (is_int8(imm32)) {
1014 EMIT(0x6B);
1015 emit_operand(dst, src);
1016 EMIT(imm32);
1017 } else {
1018 EMIT(0x69);
1019 emit_operand(dst, src);
1020 emit(imm32);
1021 }
1022 }
1023
inc(Register dst)1024 void Assembler::inc(Register dst) {
1025 EnsureSpace ensure_space(this);
1026 EMIT(0x40 | dst.code());
1027 }
1028
inc(Operand dst)1029 void Assembler::inc(Operand dst) {
1030 EnsureSpace ensure_space(this);
1031 EMIT(0xFF);
1032 emit_operand(eax, dst);
1033 }
1034
lea(Register dst,Operand src)1035 void Assembler::lea(Register dst, Operand src) {
1036 EnsureSpace ensure_space(this);
1037 EMIT(0x8D);
1038 emit_operand(dst, src);
1039 }
1040
mul(Register src)1041 void Assembler::mul(Register src) {
1042 EnsureSpace ensure_space(this);
1043 EMIT(0xF7);
1044 EMIT(0xE0 | src.code());
1045 }
1046
neg(Register dst)1047 void Assembler::neg(Register dst) {
1048 EnsureSpace ensure_space(this);
1049 EMIT(0xF7);
1050 EMIT(0xD8 | dst.code());
1051 }
1052
neg(Operand dst)1053 void Assembler::neg(Operand dst) {
1054 EnsureSpace ensure_space(this);
1055 EMIT(0xF7);
1056 emit_operand(ebx, dst);
1057 }
1058
not_(Register dst)1059 void Assembler::not_(Register dst) {
1060 EnsureSpace ensure_space(this);
1061 EMIT(0xF7);
1062 EMIT(0xD0 | dst.code());
1063 }
1064
not_(Operand dst)1065 void Assembler::not_(Operand dst) {
1066 EnsureSpace ensure_space(this);
1067 EMIT(0xF7);
1068 emit_operand(edx, dst);
1069 }
1070
or_(Register dst,int32_t imm32)1071 void Assembler::or_(Register dst, int32_t imm32) {
1072 EnsureSpace ensure_space(this);
1073 emit_arith(1, Operand(dst), Immediate(imm32));
1074 }
1075
or_(Register dst,Operand src)1076 void Assembler::or_(Register dst, Operand src) {
1077 EnsureSpace ensure_space(this);
1078 EMIT(0x0B);
1079 emit_operand(dst, src);
1080 }
1081
or_(Operand dst,const Immediate & x)1082 void Assembler::or_(Operand dst, const Immediate& x) {
1083 EnsureSpace ensure_space(this);
1084 emit_arith(1, dst, x);
1085 }
1086
or_(Operand dst,Register src)1087 void Assembler::or_(Operand dst, Register src) {
1088 EnsureSpace ensure_space(this);
1089 EMIT(0x09);
1090 emit_operand(src, dst);
1091 }
1092
rcl(Register dst,uint8_t imm8)1093 void Assembler::rcl(Register dst, uint8_t imm8) {
1094 EnsureSpace ensure_space(this);
1095 DCHECK(is_uint5(imm8)); // illegal shift count
1096 if (imm8 == 1) {
1097 EMIT(0xD1);
1098 EMIT(0xD0 | dst.code());
1099 } else {
1100 EMIT(0xC1);
1101 EMIT(0xD0 | dst.code());
1102 EMIT(imm8);
1103 }
1104 }
1105
rcr(Register dst,uint8_t imm8)1106 void Assembler::rcr(Register dst, uint8_t imm8) {
1107 EnsureSpace ensure_space(this);
1108 DCHECK(is_uint5(imm8)); // illegal shift count
1109 if (imm8 == 1) {
1110 EMIT(0xD1);
1111 EMIT(0xD8 | dst.code());
1112 } else {
1113 EMIT(0xC1);
1114 EMIT(0xD8 | dst.code());
1115 EMIT(imm8);
1116 }
1117 }
1118
rol(Operand dst,uint8_t imm8)1119 void Assembler::rol(Operand dst, uint8_t imm8) {
1120 EnsureSpace ensure_space(this);
1121 DCHECK(is_uint5(imm8)); // illegal shift count
1122 if (imm8 == 1) {
1123 EMIT(0xD1);
1124 emit_operand(eax, dst);
1125 } else {
1126 EMIT(0xC1);
1127 emit_operand(eax, dst);
1128 EMIT(imm8);
1129 }
1130 }
1131
rol_cl(Operand dst)1132 void Assembler::rol_cl(Operand dst) {
1133 EnsureSpace ensure_space(this);
1134 EMIT(0xD3);
1135 emit_operand(eax, dst);
1136 }
1137
ror(Operand dst,uint8_t imm8)1138 void Assembler::ror(Operand dst, uint8_t imm8) {
1139 EnsureSpace ensure_space(this);
1140 DCHECK(is_uint5(imm8)); // illegal shift count
1141 if (imm8 == 1) {
1142 EMIT(0xD1);
1143 emit_operand(ecx, dst);
1144 } else {
1145 EMIT(0xC1);
1146 emit_operand(ecx, dst);
1147 EMIT(imm8);
1148 }
1149 }
1150
ror_cl(Operand dst)1151 void Assembler::ror_cl(Operand dst) {
1152 EnsureSpace ensure_space(this);
1153 EMIT(0xD3);
1154 emit_operand(ecx, dst);
1155 }
1156
sar(Operand dst,uint8_t imm8)1157 void Assembler::sar(Operand dst, uint8_t imm8) {
1158 EnsureSpace ensure_space(this);
1159 DCHECK(is_uint5(imm8)); // illegal shift count
1160 if (imm8 == 1) {
1161 EMIT(0xD1);
1162 emit_operand(edi, dst);
1163 } else {
1164 EMIT(0xC1);
1165 emit_operand(edi, dst);
1166 EMIT(imm8);
1167 }
1168 }
1169
sar_cl(Operand dst)1170 void Assembler::sar_cl(Operand dst) {
1171 EnsureSpace ensure_space(this);
1172 EMIT(0xD3);
1173 emit_operand(edi, dst);
1174 }
1175
sbb(Register dst,Operand src)1176 void Assembler::sbb(Register dst, Operand src) {
1177 EnsureSpace ensure_space(this);
1178 EMIT(0x1B);
1179 emit_operand(dst, src);
1180 }
1181
shld(Register dst,Register src,uint8_t shift)1182 void Assembler::shld(Register dst, Register src, uint8_t shift) {
1183 DCHECK(is_uint5(shift));
1184 EnsureSpace ensure_space(this);
1185 EMIT(0x0F);
1186 EMIT(0xA4);
1187 emit_operand(src, Operand(dst));
1188 EMIT(shift);
1189 }
1190
shld_cl(Register dst,Register src)1191 void Assembler::shld_cl(Register dst, Register src) {
1192 EnsureSpace ensure_space(this);
1193 EMIT(0x0F);
1194 EMIT(0xA5);
1195 emit_operand(src, Operand(dst));
1196 }
1197
shl(Operand dst,uint8_t imm8)1198 void Assembler::shl(Operand dst, uint8_t imm8) {
1199 EnsureSpace ensure_space(this);
1200 DCHECK(is_uint5(imm8)); // illegal shift count
1201 if (imm8 == 1) {
1202 EMIT(0xD1);
1203 emit_operand(esp, dst);
1204 } else {
1205 EMIT(0xC1);
1206 emit_operand(esp, dst);
1207 EMIT(imm8);
1208 }
1209 }
1210
shl_cl(Operand dst)1211 void Assembler::shl_cl(Operand dst) {
1212 EnsureSpace ensure_space(this);
1213 EMIT(0xD3);
1214 emit_operand(esp, dst);
1215 }
1216
shr(Operand dst,uint8_t imm8)1217 void Assembler::shr(Operand dst, uint8_t imm8) {
1218 EnsureSpace ensure_space(this);
1219 DCHECK(is_uint5(imm8)); // illegal shift count
1220 if (imm8 == 1) {
1221 EMIT(0xD1);
1222 emit_operand(ebp, dst);
1223 } else {
1224 EMIT(0xC1);
1225 emit_operand(ebp, dst);
1226 EMIT(imm8);
1227 }
1228 }
1229
shr_cl(Operand dst)1230 void Assembler::shr_cl(Operand dst) {
1231 EnsureSpace ensure_space(this);
1232 EMIT(0xD3);
1233 emit_operand(ebp, dst);
1234 }
1235
shrd(Register dst,Register src,uint8_t shift)1236 void Assembler::shrd(Register dst, Register src, uint8_t shift) {
1237 DCHECK(is_uint5(shift));
1238 EnsureSpace ensure_space(this);
1239 EMIT(0x0F);
1240 EMIT(0xAC);
1241 emit_operand(src, Operand(dst));
1242 EMIT(shift);
1243 }
1244
shrd_cl(Operand dst,Register src)1245 void Assembler::shrd_cl(Operand dst, Register src) {
1246 EnsureSpace ensure_space(this);
1247 EMIT(0x0F);
1248 EMIT(0xAD);
1249 emit_operand(src, dst);
1250 }
1251
sub(Operand dst,const Immediate & x)1252 void Assembler::sub(Operand dst, const Immediate& x) {
1253 EnsureSpace ensure_space(this);
1254 emit_arith(5, dst, x);
1255 }
1256
sub(Register dst,Operand src)1257 void Assembler::sub(Register dst, Operand src) {
1258 EnsureSpace ensure_space(this);
1259 EMIT(0x2B);
1260 emit_operand(dst, src);
1261 }
1262
sub(Operand dst,Register src)1263 void Assembler::sub(Operand dst, Register src) {
1264 EnsureSpace ensure_space(this);
1265 EMIT(0x29);
1266 emit_operand(src, dst);
1267 }
1268
sub_sp_32(uint32_t imm)1269 void Assembler::sub_sp_32(uint32_t imm) {
1270 EnsureSpace ensure_space(this);
1271 EMIT(0x81); // using a literal 32-bit immediate.
1272 static constexpr Register ireg = Register::from_code(5);
1273 emit_operand(ireg, Operand(esp));
1274 emit(imm);
1275 }
1276
test(Register reg,const Immediate & imm)1277 void Assembler::test(Register reg, const Immediate& imm) {
1278 if (imm.is_uint8()) {
1279 test_b(reg, imm);
1280 return;
1281 }
1282
1283 EnsureSpace ensure_space(this);
1284 // This is not using emit_arith because test doesn't support
1285 // sign-extension of 8-bit operands.
1286 if (reg == eax) {
1287 EMIT(0xA9);
1288 } else {
1289 EMIT(0xF7);
1290 EMIT(0xC0 | reg.code());
1291 }
1292 emit(imm);
1293 }
1294
test(Register reg,Operand op)1295 void Assembler::test(Register reg, Operand op) {
1296 EnsureSpace ensure_space(this);
1297 EMIT(0x85);
1298 emit_operand(reg, op);
1299 }
1300
test_b(Register reg,Operand op)1301 void Assembler::test_b(Register reg, Operand op) {
1302 CHECK(reg.is_byte_register());
1303 EnsureSpace ensure_space(this);
1304 EMIT(0x84);
1305 emit_operand(reg, op);
1306 }
1307
test(Operand op,const Immediate & imm)1308 void Assembler::test(Operand op, const Immediate& imm) {
1309 if (op.is_reg_only()) {
1310 test(op.reg(), imm);
1311 return;
1312 }
1313 if (imm.is_uint8()) {
1314 return test_b(op, imm);
1315 }
1316 EnsureSpace ensure_space(this);
1317 EMIT(0xF7);
1318 emit_operand(eax, op);
1319 emit(imm);
1320 }
1321
test_b(Register reg,Immediate imm8)1322 void Assembler::test_b(Register reg, Immediate imm8) {
1323 DCHECK(imm8.is_uint8());
1324 EnsureSpace ensure_space(this);
1325 // Only use test against byte for registers that have a byte
1326 // variant: eax, ebx, ecx, and edx.
1327 if (reg == eax) {
1328 EMIT(0xA8);
1329 emit_b(imm8);
1330 } else if (reg.is_byte_register()) {
1331 emit_arith_b(0xF6, 0xC0, reg, static_cast<uint8_t>(imm8.immediate()));
1332 } else {
1333 EMIT(0x66);
1334 EMIT(0xF7);
1335 EMIT(0xC0 | reg.code());
1336 emit_w(imm8);
1337 }
1338 }
1339
test_b(Operand op,Immediate imm8)1340 void Assembler::test_b(Operand op, Immediate imm8) {
1341 if (op.is_reg_only()) {
1342 test_b(op.reg(), imm8);
1343 return;
1344 }
1345 EnsureSpace ensure_space(this);
1346 EMIT(0xF6);
1347 emit_operand(eax, op);
1348 emit_b(imm8);
1349 }
1350
test_w(Register reg,Immediate imm16)1351 void Assembler::test_w(Register reg, Immediate imm16) {
1352 DCHECK(imm16.is_int16() || imm16.is_uint16());
1353 EnsureSpace ensure_space(this);
1354 if (reg == eax) {
1355 EMIT(0xA9);
1356 emit_w(imm16);
1357 } else {
1358 EMIT(0x66);
1359 EMIT(0xF7);
1360 EMIT(0xC0 | reg.code());
1361 emit_w(imm16);
1362 }
1363 }
1364
test_w(Register reg,Operand op)1365 void Assembler::test_w(Register reg, Operand op) {
1366 EnsureSpace ensure_space(this);
1367 EMIT(0x66);
1368 EMIT(0x85);
1369 emit_operand(reg, op);
1370 }
1371
test_w(Operand op,Immediate imm16)1372 void Assembler::test_w(Operand op, Immediate imm16) {
1373 DCHECK(imm16.is_int16() || imm16.is_uint16());
1374 if (op.is_reg_only()) {
1375 test_w(op.reg(), imm16);
1376 return;
1377 }
1378 EnsureSpace ensure_space(this);
1379 EMIT(0x66);
1380 EMIT(0xF7);
1381 emit_operand(eax, op);
1382 emit_w(imm16);
1383 }
1384
xor_(Register dst,int32_t imm32)1385 void Assembler::xor_(Register dst, int32_t imm32) {
1386 EnsureSpace ensure_space(this);
1387 emit_arith(6, Operand(dst), Immediate(imm32));
1388 }
1389
xor_(Register dst,Operand src)1390 void Assembler::xor_(Register dst, Operand src) {
1391 EnsureSpace ensure_space(this);
1392 EMIT(0x33);
1393 emit_operand(dst, src);
1394 }
1395
xor_(Operand dst,Register src)1396 void Assembler::xor_(Operand dst, Register src) {
1397 EnsureSpace ensure_space(this);
1398 EMIT(0x31);
1399 emit_operand(src, dst);
1400 }
1401
xor_(Operand dst,const Immediate & x)1402 void Assembler::xor_(Operand dst, const Immediate& x) {
1403 EnsureSpace ensure_space(this);
1404 emit_arith(6, dst, x);
1405 }
1406
bswap(Register dst)1407 void Assembler::bswap(Register dst) {
1408 EnsureSpace ensure_space(this);
1409 EMIT(0x0F);
1410 EMIT(0xC8 + dst.code());
1411 }
1412
bt(Operand dst,Register src)1413 void Assembler::bt(Operand dst, Register src) {
1414 EnsureSpace ensure_space(this);
1415 EMIT(0x0F);
1416 EMIT(0xA3);
1417 emit_operand(src, dst);
1418 }
1419
bts(Operand dst,Register src)1420 void Assembler::bts(Operand dst, Register src) {
1421 EnsureSpace ensure_space(this);
1422 EMIT(0x0F);
1423 EMIT(0xAB);
1424 emit_operand(src, dst);
1425 }
1426
bsr(Register dst,Operand src)1427 void Assembler::bsr(Register dst, Operand src) {
1428 EnsureSpace ensure_space(this);
1429 EMIT(0x0F);
1430 EMIT(0xBD);
1431 emit_operand(dst, src);
1432 }
1433
bsf(Register dst,Operand src)1434 void Assembler::bsf(Register dst, Operand src) {
1435 EnsureSpace ensure_space(this);
1436 EMIT(0x0F);
1437 EMIT(0xBC);
1438 emit_operand(dst, src);
1439 }
1440
hlt()1441 void Assembler::hlt() {
1442 EnsureSpace ensure_space(this);
1443 EMIT(0xF4);
1444 }
1445
int3()1446 void Assembler::int3() {
1447 EnsureSpace ensure_space(this);
1448 EMIT(0xCC);
1449 }
1450
nop()1451 void Assembler::nop() {
1452 EnsureSpace ensure_space(this);
1453 EMIT(0x90);
1454 }
1455
ret(int imm16)1456 void Assembler::ret(int imm16) {
1457 EnsureSpace ensure_space(this);
1458 DCHECK(is_uint16(imm16));
1459 if (imm16 == 0) {
1460 EMIT(0xC3);
1461 } else {
1462 EMIT(0xC2);
1463 EMIT(imm16 & 0xFF);
1464 EMIT((imm16 >> 8) & 0xFF);
1465 }
1466 }
1467
ud2()1468 void Assembler::ud2() {
1469 EnsureSpace ensure_space(this);
1470 EMIT(0x0F);
1471 EMIT(0x0B);
1472 }
1473
1474 // Labels refer to positions in the (to be) generated code.
1475 // There are bound, linked, and unused labels.
1476 //
1477 // Bound labels refer to known positions in the already
1478 // generated code. pos() is the position the label refers to.
1479 //
1480 // Linked labels refer to unknown positions in the code
1481 // to be generated; pos() is the position of the 32bit
1482 // Displacement of the last instruction using the label.
1483
print(const Label * L)1484 void Assembler::print(const Label* L) {
1485 if (L->is_unused()) {
1486 PrintF("unused label\n");
1487 } else if (L->is_bound()) {
1488 PrintF("bound label to %d\n", L->pos());
1489 } else if (L->is_linked()) {
1490 Label l;
1491 l.link_to(L->pos());
1492 PrintF("unbound label");
1493 while (l.is_linked()) {
1494 Displacement disp = disp_at(&l);
1495 PrintF("@ %d ", l.pos());
1496 disp.print();
1497 PrintF("\n");
1498 disp.next(&l);
1499 }
1500 } else {
1501 PrintF("label in inconsistent state (pos = %d)\n", L->pos_);
1502 }
1503 }
1504
bind_to(Label * L,int pos)1505 void Assembler::bind_to(Label* L, int pos) {
1506 EnsureSpace ensure_space(this);
1507 DCHECK(0 <= pos && pos <= pc_offset()); // must have a valid binding position
1508 while (L->is_linked()) {
1509 Displacement disp = disp_at(L);
1510 int fixup_pos = L->pos();
1511 if (disp.type() == Displacement::CODE_ABSOLUTE) {
1512 long_at_put(fixup_pos, reinterpret_cast<int>(buffer_start_ + pos));
1513 internal_reference_positions_.push_back(fixup_pos);
1514 } else if (disp.type() == Displacement::CODE_RELATIVE) {
1515 // Relative to Code heap object pointer.
1516 long_at_put(fixup_pos, pos + Code::kHeaderSize - kHeapObjectTag);
1517 } else {
1518 if (disp.type() == Displacement::UNCONDITIONAL_JUMP) {
1519 DCHECK_EQ(byte_at(fixup_pos - 1), 0xE9); // jmp expected
1520 }
1521 // Relative address, relative to point after address.
1522 int imm32 = pos - (fixup_pos + sizeof(int32_t));
1523 long_at_put(fixup_pos, imm32);
1524 }
1525 disp.next(L);
1526 }
1527 while (L->is_near_linked()) {
1528 int fixup_pos = L->near_link_pos();
1529 int offset_to_next =
1530 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
1531 DCHECK_LE(offset_to_next, 0);
1532 // Relative address, relative to point after address.
1533 int disp = pos - fixup_pos - sizeof(int8_t);
1534 CHECK(0 <= disp && disp <= 127);
1535 set_byte_at(fixup_pos, disp);
1536 if (offset_to_next < 0) {
1537 L->link_to(fixup_pos + offset_to_next, Label::kNear);
1538 } else {
1539 L->UnuseNear();
1540 }
1541 }
1542
1543 // Optimization stage
1544 auto jump_opt = jump_optimization_info();
1545 if (jump_opt && jump_opt->is_optimizing()) {
1546 auto it = label_farjmp_maps_.find(L);
1547 if (it != label_farjmp_maps_.end()) {
1548 auto& pos_vector = it->second;
1549 for (auto fixup_pos : pos_vector) {
1550 int disp = pos - (fixup_pos + sizeof(int8_t));
1551 CHECK(is_int8(disp));
1552 set_byte_at(fixup_pos, disp);
1553 }
1554 label_farjmp_maps_.erase(it);
1555 }
1556 }
1557 L->bind_to(pos);
1558 }
1559
bind(Label * L)1560 void Assembler::bind(Label* L) {
1561 EnsureSpace ensure_space(this);
1562 DCHECK(!L->is_bound()); // label can only be bound once
1563 bind_to(L, pc_offset());
1564 }
1565
record_farjmp_position(Label * L,int pos)1566 void Assembler::record_farjmp_position(Label* L, int pos) {
1567 auto& pos_vector = label_farjmp_maps_[L];
1568 pos_vector.push_back(pos);
1569 }
1570
is_optimizable_farjmp(int idx)1571 bool Assembler::is_optimizable_farjmp(int idx) {
1572 if (predictable_code_size()) return false;
1573
1574 auto jump_opt = jump_optimization_info();
1575 CHECK(jump_opt->is_optimizing());
1576
1577 auto& bitmap = jump_opt->farjmp_bitmap();
1578 CHECK(idx < static_cast<int>(bitmap.size() * 32));
1579 return !!(bitmap[idx / 32] & (1 << (idx & 31)));
1580 }
1581
call(Label * L)1582 void Assembler::call(Label* L) {
1583 EnsureSpace ensure_space(this);
1584 if (L->is_bound()) {
1585 const int long_size = 5;
1586 int offs = L->pos() - pc_offset();
1587 DCHECK_LE(offs, 0);
1588 // 1110 1000 #32-bit disp.
1589 EMIT(0xE8);
1590 emit(offs - long_size);
1591 } else {
1592 // 1110 1000 #32-bit disp.
1593 EMIT(0xE8);
1594 emit_disp(L, Displacement::OTHER);
1595 }
1596 }
1597
call(Address entry,RelocInfo::Mode rmode)1598 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
1599 EnsureSpace ensure_space(this);
1600 DCHECK(!RelocInfo::IsCodeTarget(rmode));
1601 EMIT(0xE8);
1602 if (RelocInfo::IsRuntimeEntry(rmode)) {
1603 emit(entry, rmode);
1604 } else {
1605 emit(entry - (reinterpret_cast<Address>(pc_) + sizeof(int32_t)), rmode);
1606 }
1607 }
1608
wasm_call(Address entry,RelocInfo::Mode rmode)1609 void Assembler::wasm_call(Address entry, RelocInfo::Mode rmode) {
1610 EnsureSpace ensure_space(this);
1611 EMIT(0xE8);
1612 emit(entry, rmode);
1613 }
1614
call(Operand adr)1615 void Assembler::call(Operand adr) {
1616 EnsureSpace ensure_space(this);
1617 EMIT(0xFF);
1618 emit_operand(edx, adr);
1619 }
1620
call(Handle<Code> code,RelocInfo::Mode rmode)1621 void Assembler::call(Handle<Code> code, RelocInfo::Mode rmode) {
1622 EnsureSpace ensure_space(this);
1623 DCHECK(RelocInfo::IsCodeTarget(rmode));
1624 DCHECK(code->IsExecutable());
1625 EMIT(0xE8);
1626 emit(code, rmode);
1627 }
1628
jmp_rel(int offset)1629 void Assembler::jmp_rel(int offset) {
1630 EnsureSpace ensure_space(this);
1631 const int short_size = 2;
1632 const int long_size = 5;
1633 if (is_int8(offset - short_size)) {
1634 // 1110 1011 #8-bit disp.
1635 EMIT(0xEB);
1636 EMIT((offset - short_size) & 0xFF);
1637 } else {
1638 // 1110 1001 #32-bit disp.
1639 EMIT(0xE9);
1640 emit(offset - long_size);
1641 }
1642 }
1643
jmp(Label * L,Label::Distance distance)1644 void Assembler::jmp(Label* L, Label::Distance distance) {
1645 if (L->is_bound()) {
1646 int offset = L->pos() - pc_offset();
1647 DCHECK_LE(offset, 0); // backward jump.
1648 jmp_rel(offset);
1649 return;
1650 }
1651
1652 EnsureSpace ensure_space(this);
1653 if (distance == Label::kNear) {
1654 EMIT(0xEB);
1655 emit_near_disp(L);
1656 } else {
1657 auto jump_opt = jump_optimization_info();
1658 if (V8_UNLIKELY(jump_opt)) {
1659 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1660 EMIT(0xEB);
1661 record_farjmp_position(L, pc_offset());
1662 EMIT(0);
1663 return;
1664 }
1665 if (jump_opt->is_collecting()) {
1666 farjmp_positions_.push_back(pc_offset() + 1);
1667 }
1668 }
1669 // 1110 1001 #32-bit disp.
1670 EMIT(0xE9);
1671 emit_disp(L, Displacement::UNCONDITIONAL_JUMP);
1672 }
1673 }
1674
jmp(Address entry,RelocInfo::Mode rmode)1675 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1676 EnsureSpace ensure_space(this);
1677 DCHECK(!RelocInfo::IsCodeTarget(rmode));
1678 EMIT(0xE9);
1679 if (RelocInfo::IsRuntimeEntry(rmode) || RelocInfo::IsWasmCall(rmode)) {
1680 emit(entry, rmode);
1681 } else {
1682 emit(entry - (reinterpret_cast<Address>(pc_) + sizeof(int32_t)), rmode);
1683 }
1684 }
1685
jmp(Operand adr)1686 void Assembler::jmp(Operand adr) {
1687 EnsureSpace ensure_space(this);
1688 EMIT(0xFF);
1689 emit_operand(esp, adr);
1690 }
1691
jmp(Handle<Code> code,RelocInfo::Mode rmode)1692 void Assembler::jmp(Handle<Code> code, RelocInfo::Mode rmode) {
1693 EnsureSpace ensure_space(this);
1694 DCHECK(RelocInfo::IsCodeTarget(rmode));
1695 EMIT(0xE9);
1696 emit(code, rmode);
1697 }
1698
j(Condition cc,Label * L,Label::Distance distance)1699 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1700 EnsureSpace ensure_space(this);
1701 DCHECK(0 <= cc && static_cast<int>(cc) < 16);
1702 if (L->is_bound()) {
1703 const int short_size = 2;
1704 const int long_size = 6;
1705 int offs = L->pos() - pc_offset();
1706 DCHECK_LE(offs, 0);
1707 if (is_int8(offs - short_size)) {
1708 // 0111 tttn #8-bit disp
1709 EMIT(0x70 | cc);
1710 EMIT((offs - short_size) & 0xFF);
1711 } else {
1712 // 0000 1111 1000 tttn #32-bit disp
1713 EMIT(0x0F);
1714 EMIT(0x80 | cc);
1715 emit(offs - long_size);
1716 }
1717 } else if (distance == Label::kNear) {
1718 EMIT(0x70 | cc);
1719 emit_near_disp(L);
1720 } else {
1721 auto jump_opt = jump_optimization_info();
1722 if (V8_UNLIKELY(jump_opt)) {
1723 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1724 // 0111 tttn #8-bit disp
1725 EMIT(0x70 | cc);
1726 record_farjmp_position(L, pc_offset());
1727 EMIT(0);
1728 return;
1729 }
1730 if (jump_opt->is_collecting()) {
1731 farjmp_positions_.push_back(pc_offset() + 2);
1732 }
1733 }
1734 // 0000 1111 1000 tttn #32-bit disp
1735 // Note: could eliminate cond. jumps to this jump if condition
1736 // is the same however, seems to be rather unlikely case.
1737 EMIT(0x0F);
1738 EMIT(0x80 | cc);
1739 emit_disp(L, Displacement::OTHER);
1740 }
1741 }
1742
j(Condition cc,byte * entry,RelocInfo::Mode rmode)1743 void Assembler::j(Condition cc, byte* entry, RelocInfo::Mode rmode) {
1744 EnsureSpace ensure_space(this);
1745 DCHECK((0 <= cc) && (static_cast<int>(cc) < 16));
1746 // 0000 1111 1000 tttn #32-bit disp.
1747 EMIT(0x0F);
1748 EMIT(0x80 | cc);
1749 if (RelocInfo::IsRuntimeEntry(rmode)) {
1750 emit(reinterpret_cast<uint32_t>(entry), rmode);
1751 } else {
1752 emit(entry - (pc_ + sizeof(int32_t)), rmode);
1753 }
1754 }
1755
j(Condition cc,Handle<Code> code,RelocInfo::Mode rmode)1756 void Assembler::j(Condition cc, Handle<Code> code, RelocInfo::Mode rmode) {
1757 EnsureSpace ensure_space(this);
1758 // 0000 1111 1000 tttn #32-bit disp
1759 EMIT(0x0F);
1760 EMIT(0x80 | cc);
1761 emit(code, rmode);
1762 }
1763
1764 // FPU instructions.
1765
fld(int i)1766 void Assembler::fld(int i) {
1767 EnsureSpace ensure_space(this);
1768 emit_farith(0xD9, 0xC0, i);
1769 }
1770
fstp(int i)1771 void Assembler::fstp(int i) {
1772 EnsureSpace ensure_space(this);
1773 emit_farith(0xDD, 0xD8, i);
1774 }
1775
fld1()1776 void Assembler::fld1() {
1777 EnsureSpace ensure_space(this);
1778 EMIT(0xD9);
1779 EMIT(0xE8);
1780 }
1781
fldpi()1782 void Assembler::fldpi() {
1783 EnsureSpace ensure_space(this);
1784 EMIT(0xD9);
1785 EMIT(0xEB);
1786 }
1787
fldz()1788 void Assembler::fldz() {
1789 EnsureSpace ensure_space(this);
1790 EMIT(0xD9);
1791 EMIT(0xEE);
1792 }
1793
fldln2()1794 void Assembler::fldln2() {
1795 EnsureSpace ensure_space(this);
1796 EMIT(0xD9);
1797 EMIT(0xED);
1798 }
1799
fld_s(Operand adr)1800 void Assembler::fld_s(Operand adr) {
1801 EnsureSpace ensure_space(this);
1802 EMIT(0xD9);
1803 emit_operand(eax, adr);
1804 }
1805
fld_d(Operand adr)1806 void Assembler::fld_d(Operand adr) {
1807 EnsureSpace ensure_space(this);
1808 EMIT(0xDD);
1809 emit_operand(eax, adr);
1810 }
1811
fstp_s(Operand adr)1812 void Assembler::fstp_s(Operand adr) {
1813 EnsureSpace ensure_space(this);
1814 EMIT(0xD9);
1815 emit_operand(ebx, adr);
1816 }
1817
fst_s(Operand adr)1818 void Assembler::fst_s(Operand adr) {
1819 EnsureSpace ensure_space(this);
1820 EMIT(0xD9);
1821 emit_operand(edx, adr);
1822 }
1823
fstp_d(Operand adr)1824 void Assembler::fstp_d(Operand adr) {
1825 EnsureSpace ensure_space(this);
1826 EMIT(0xDD);
1827 emit_operand(ebx, adr);
1828 }
1829
fst_d(Operand adr)1830 void Assembler::fst_d(Operand adr) {
1831 EnsureSpace ensure_space(this);
1832 EMIT(0xDD);
1833 emit_operand(edx, adr);
1834 }
1835
fild_s(Operand adr)1836 void Assembler::fild_s(Operand adr) {
1837 EnsureSpace ensure_space(this);
1838 EMIT(0xDB);
1839 emit_operand(eax, adr);
1840 }
1841
fild_d(Operand adr)1842 void Assembler::fild_d(Operand adr) {
1843 EnsureSpace ensure_space(this);
1844 EMIT(0xDF);
1845 emit_operand(ebp, adr);
1846 }
1847
fistp_s(Operand adr)1848 void Assembler::fistp_s(Operand adr) {
1849 EnsureSpace ensure_space(this);
1850 EMIT(0xDB);
1851 emit_operand(ebx, adr);
1852 }
1853
fisttp_s(Operand adr)1854 void Assembler::fisttp_s(Operand adr) {
1855 DCHECK(IsEnabled(SSE3));
1856 EnsureSpace ensure_space(this);
1857 EMIT(0xDB);
1858 emit_operand(ecx, adr);
1859 }
1860
fisttp_d(Operand adr)1861 void Assembler::fisttp_d(Operand adr) {
1862 DCHECK(IsEnabled(SSE3));
1863 EnsureSpace ensure_space(this);
1864 EMIT(0xDD);
1865 emit_operand(ecx, adr);
1866 }
1867
fist_s(Operand adr)1868 void Assembler::fist_s(Operand adr) {
1869 EnsureSpace ensure_space(this);
1870 EMIT(0xDB);
1871 emit_operand(edx, adr);
1872 }
1873
fistp_d(Operand adr)1874 void Assembler::fistp_d(Operand adr) {
1875 EnsureSpace ensure_space(this);
1876 EMIT(0xDF);
1877 emit_operand(edi, adr);
1878 }
1879
fabs()1880 void Assembler::fabs() {
1881 EnsureSpace ensure_space(this);
1882 EMIT(0xD9);
1883 EMIT(0xE1);
1884 }
1885
fchs()1886 void Assembler::fchs() {
1887 EnsureSpace ensure_space(this);
1888 EMIT(0xD9);
1889 EMIT(0xE0);
1890 }
1891
fcos()1892 void Assembler::fcos() {
1893 EnsureSpace ensure_space(this);
1894 EMIT(0xD9);
1895 EMIT(0xFF);
1896 }
1897
fsin()1898 void Assembler::fsin() {
1899 EnsureSpace ensure_space(this);
1900 EMIT(0xD9);
1901 EMIT(0xFE);
1902 }
1903
fptan()1904 void Assembler::fptan() {
1905 EnsureSpace ensure_space(this);
1906 EMIT(0xD9);
1907 EMIT(0xF2);
1908 }
1909
fyl2x()1910 void Assembler::fyl2x() {
1911 EnsureSpace ensure_space(this);
1912 EMIT(0xD9);
1913 EMIT(0xF1);
1914 }
1915
f2xm1()1916 void Assembler::f2xm1() {
1917 EnsureSpace ensure_space(this);
1918 EMIT(0xD9);
1919 EMIT(0xF0);
1920 }
1921
fscale()1922 void Assembler::fscale() {
1923 EnsureSpace ensure_space(this);
1924 EMIT(0xD9);
1925 EMIT(0xFD);
1926 }
1927
fninit()1928 void Assembler::fninit() {
1929 EnsureSpace ensure_space(this);
1930 EMIT(0xDB);
1931 EMIT(0xE3);
1932 }
1933
fadd(int i)1934 void Assembler::fadd(int i) {
1935 EnsureSpace ensure_space(this);
1936 emit_farith(0xDC, 0xC0, i);
1937 }
1938
fadd_i(int i)1939 void Assembler::fadd_i(int i) {
1940 EnsureSpace ensure_space(this);
1941 emit_farith(0xD8, 0xC0, i);
1942 }
1943
fsub(int i)1944 void Assembler::fsub(int i) {
1945 EnsureSpace ensure_space(this);
1946 emit_farith(0xDC, 0xE8, i);
1947 }
1948
fsub_i(int i)1949 void Assembler::fsub_i(int i) {
1950 EnsureSpace ensure_space(this);
1951 emit_farith(0xD8, 0xE0, i);
1952 }
1953
fisub_s(Operand adr)1954 void Assembler::fisub_s(Operand adr) {
1955 EnsureSpace ensure_space(this);
1956 EMIT(0xDA);
1957 emit_operand(esp, adr);
1958 }
1959
fmul_i(int i)1960 void Assembler::fmul_i(int i) {
1961 EnsureSpace ensure_space(this);
1962 emit_farith(0xD8, 0xC8, i);
1963 }
1964
fmul(int i)1965 void Assembler::fmul(int i) {
1966 EnsureSpace ensure_space(this);
1967 emit_farith(0xDC, 0xC8, i);
1968 }
1969
fdiv(int i)1970 void Assembler::fdiv(int i) {
1971 EnsureSpace ensure_space(this);
1972 emit_farith(0xDC, 0xF8, i);
1973 }
1974
fdiv_i(int i)1975 void Assembler::fdiv_i(int i) {
1976 EnsureSpace ensure_space(this);
1977 emit_farith(0xD8, 0xF0, i);
1978 }
1979
faddp(int i)1980 void Assembler::faddp(int i) {
1981 EnsureSpace ensure_space(this);
1982 emit_farith(0xDE, 0xC0, i);
1983 }
1984
fsubp(int i)1985 void Assembler::fsubp(int i) {
1986 EnsureSpace ensure_space(this);
1987 emit_farith(0xDE, 0xE8, i);
1988 }
1989
fsubrp(int i)1990 void Assembler::fsubrp(int i) {
1991 EnsureSpace ensure_space(this);
1992 emit_farith(0xDE, 0xE0, i);
1993 }
1994
fmulp(int i)1995 void Assembler::fmulp(int i) {
1996 EnsureSpace ensure_space(this);
1997 emit_farith(0xDE, 0xC8, i);
1998 }
1999
fdivp(int i)2000 void Assembler::fdivp(int i) {
2001 EnsureSpace ensure_space(this);
2002 emit_farith(0xDE, 0xF8, i);
2003 }
2004
fprem()2005 void Assembler::fprem() {
2006 EnsureSpace ensure_space(this);
2007 EMIT(0xD9);
2008 EMIT(0xF8);
2009 }
2010
fprem1()2011 void Assembler::fprem1() {
2012 EnsureSpace ensure_space(this);
2013 EMIT(0xD9);
2014 EMIT(0xF5);
2015 }
2016
fxch(int i)2017 void Assembler::fxch(int i) {
2018 EnsureSpace ensure_space(this);
2019 emit_farith(0xD9, 0xC8, i);
2020 }
2021
fincstp()2022 void Assembler::fincstp() {
2023 EnsureSpace ensure_space(this);
2024 EMIT(0xD9);
2025 EMIT(0xF7);
2026 }
2027
ffree(int i)2028 void Assembler::ffree(int i) {
2029 EnsureSpace ensure_space(this);
2030 emit_farith(0xDD, 0xC0, i);
2031 }
2032
ftst()2033 void Assembler::ftst() {
2034 EnsureSpace ensure_space(this);
2035 EMIT(0xD9);
2036 EMIT(0xE4);
2037 }
2038
fucomp(int i)2039 void Assembler::fucomp(int i) {
2040 EnsureSpace ensure_space(this);
2041 emit_farith(0xDD, 0xE8, i);
2042 }
2043
fucompp()2044 void Assembler::fucompp() {
2045 EnsureSpace ensure_space(this);
2046 EMIT(0xDA);
2047 EMIT(0xE9);
2048 }
2049
fucomi(int i)2050 void Assembler::fucomi(int i) {
2051 EnsureSpace ensure_space(this);
2052 EMIT(0xDB);
2053 EMIT(0xE8 + i);
2054 }
2055
fucomip()2056 void Assembler::fucomip() {
2057 EnsureSpace ensure_space(this);
2058 EMIT(0xDF);
2059 EMIT(0xE9);
2060 }
2061
fcompp()2062 void Assembler::fcompp() {
2063 EnsureSpace ensure_space(this);
2064 EMIT(0xDE);
2065 EMIT(0xD9);
2066 }
2067
fnstsw_ax()2068 void Assembler::fnstsw_ax() {
2069 EnsureSpace ensure_space(this);
2070 EMIT(0xDF);
2071 EMIT(0xE0);
2072 }
2073
fwait()2074 void Assembler::fwait() {
2075 EnsureSpace ensure_space(this);
2076 EMIT(0x9B);
2077 }
2078
frndint()2079 void Assembler::frndint() {
2080 EnsureSpace ensure_space(this);
2081 EMIT(0xD9);
2082 EMIT(0xFC);
2083 }
2084
fnclex()2085 void Assembler::fnclex() {
2086 EnsureSpace ensure_space(this);
2087 EMIT(0xDB);
2088 EMIT(0xE2);
2089 }
2090
sahf()2091 void Assembler::sahf() {
2092 EnsureSpace ensure_space(this);
2093 EMIT(0x9E);
2094 }
2095
setcc(Condition cc,Register reg)2096 void Assembler::setcc(Condition cc, Register reg) {
2097 DCHECK(reg.is_byte_register());
2098 EnsureSpace ensure_space(this);
2099 EMIT(0x0F);
2100 EMIT(0x90 | cc);
2101 EMIT(0xC0 | reg.code());
2102 }
2103
cvttss2si(Register dst,Operand src)2104 void Assembler::cvttss2si(Register dst, Operand src) {
2105 EnsureSpace ensure_space(this);
2106 // The [src] might contain ebx's register code, but in
2107 // this case, it refers to xmm3, so it is OK to emit.
2108 EMIT(0xF3);
2109 EMIT(0x0F);
2110 EMIT(0x2C);
2111 emit_operand(dst, src);
2112 }
2113
cvttsd2si(Register dst,Operand src)2114 void Assembler::cvttsd2si(Register dst, Operand src) {
2115 EnsureSpace ensure_space(this);
2116 // The [src] might contain ebx's register code, but in
2117 // this case, it refers to xmm3, so it is OK to emit.
2118 EMIT(0xF2);
2119 EMIT(0x0F);
2120 EMIT(0x2C);
2121 emit_operand(dst, src);
2122 }
2123
cvtsd2si(Register dst,XMMRegister src)2124 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
2125 EnsureSpace ensure_space(this);
2126 EMIT(0xF2);
2127 EMIT(0x0F);
2128 EMIT(0x2D);
2129 emit_sse_operand(dst, src);
2130 }
2131
cvtsi2ss(XMMRegister dst,Operand src)2132 void Assembler::cvtsi2ss(XMMRegister dst, Operand src) {
2133 EnsureSpace ensure_space(this);
2134 EMIT(0xF3);
2135 EMIT(0x0F);
2136 EMIT(0x2A);
2137 emit_sse_operand(dst, src);
2138 }
2139
cvtsi2sd(XMMRegister dst,Operand src)2140 void Assembler::cvtsi2sd(XMMRegister dst, Operand src) {
2141 EnsureSpace ensure_space(this);
2142 EMIT(0xF2);
2143 EMIT(0x0F);
2144 EMIT(0x2A);
2145 emit_sse_operand(dst, src);
2146 }
2147
cvtss2sd(XMMRegister dst,Operand src)2148 void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
2149 EnsureSpace ensure_space(this);
2150 EMIT(0xF3);
2151 EMIT(0x0F);
2152 EMIT(0x5A);
2153 emit_sse_operand(dst, src);
2154 }
2155
cvtsd2ss(XMMRegister dst,Operand src)2156 void Assembler::cvtsd2ss(XMMRegister dst, Operand src) {
2157 EnsureSpace ensure_space(this);
2158 EMIT(0xF2);
2159 EMIT(0x0F);
2160 EMIT(0x5A);
2161 emit_sse_operand(dst, src);
2162 }
2163
cvtdq2ps(XMMRegister dst,Operand src)2164 void Assembler::cvtdq2ps(XMMRegister dst, Operand src) {
2165 EnsureSpace ensure_space(this);
2166 EMIT(0x0F);
2167 EMIT(0x5B);
2168 emit_sse_operand(dst, src);
2169 }
2170
cvttps2dq(XMMRegister dst,Operand src)2171 void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
2172 EnsureSpace ensure_space(this);
2173 EMIT(0xF3);
2174 EMIT(0x0F);
2175 EMIT(0x5B);
2176 emit_sse_operand(dst, src);
2177 }
2178
addsd(XMMRegister dst,Operand src)2179 void Assembler::addsd(XMMRegister dst, Operand src) {
2180 EnsureSpace ensure_space(this);
2181 EMIT(0xF2);
2182 EMIT(0x0F);
2183 EMIT(0x58);
2184 emit_sse_operand(dst, src);
2185 }
2186
mulsd(XMMRegister dst,Operand src)2187 void Assembler::mulsd(XMMRegister dst, Operand src) {
2188 EnsureSpace ensure_space(this);
2189 EMIT(0xF2);
2190 EMIT(0x0F);
2191 EMIT(0x59);
2192 emit_sse_operand(dst, src);
2193 }
2194
subsd(XMMRegister dst,Operand src)2195 void Assembler::subsd(XMMRegister dst, Operand src) {
2196 EnsureSpace ensure_space(this);
2197 EMIT(0xF2);
2198 EMIT(0x0F);
2199 EMIT(0x5C);
2200 emit_sse_operand(dst, src);
2201 }
2202
divsd(XMMRegister dst,Operand src)2203 void Assembler::divsd(XMMRegister dst, Operand src) {
2204 EnsureSpace ensure_space(this);
2205 EMIT(0xF2);
2206 EMIT(0x0F);
2207 EMIT(0x5E);
2208 emit_sse_operand(dst, src);
2209 }
2210
rcpps(XMMRegister dst,Operand src)2211 void Assembler::rcpps(XMMRegister dst, Operand src) {
2212 EnsureSpace ensure_space(this);
2213 EMIT(0x0F);
2214 EMIT(0x53);
2215 emit_sse_operand(dst, src);
2216 }
2217
sqrtps(XMMRegister dst,Operand src)2218 void Assembler::sqrtps(XMMRegister dst, Operand src) {
2219 EnsureSpace ensure_space(this);
2220 EMIT(0x0F);
2221 EMIT(0x51);
2222 emit_sse_operand(dst, src);
2223 }
2224
rsqrtps(XMMRegister dst,Operand src)2225 void Assembler::rsqrtps(XMMRegister dst, Operand src) {
2226 EnsureSpace ensure_space(this);
2227 EMIT(0x0F);
2228 EMIT(0x52);
2229 emit_sse_operand(dst, src);
2230 }
2231
cmpps(XMMRegister dst,Operand src,uint8_t cmp)2232 void Assembler::cmpps(XMMRegister dst, Operand src, uint8_t cmp) {
2233 EnsureSpace ensure_space(this);
2234 EMIT(0x0F);
2235 EMIT(0xC2);
2236 emit_sse_operand(dst, src);
2237 EMIT(cmp);
2238 }
2239
cmppd(XMMRegister dst,Operand src,uint8_t cmp)2240 void Assembler::cmppd(XMMRegister dst, Operand src, uint8_t cmp) {
2241 EnsureSpace ensure_space(this);
2242 EMIT(0x66);
2243 EMIT(0x0F);
2244 EMIT(0xC2);
2245 emit_sse_operand(dst, src);
2246 EMIT(cmp);
2247 }
2248
sqrtsd(XMMRegister dst,Operand src)2249 void Assembler::sqrtsd(XMMRegister dst, Operand src) {
2250 EnsureSpace ensure_space(this);
2251 EMIT(0xF2);
2252 EMIT(0x0F);
2253 EMIT(0x51);
2254 emit_sse_operand(dst, src);
2255 }
2256
haddps(XMMRegister dst,Operand src)2257 void Assembler::haddps(XMMRegister dst, Operand src) {
2258 DCHECK(IsEnabled(SSE3));
2259 EnsureSpace ensure_space(this);
2260 EMIT(0xF2);
2261 EMIT(0x0F);
2262 EMIT(0x7C);
2263 emit_sse_operand(dst, src);
2264 }
2265
ucomisd(XMMRegister dst,Operand src)2266 void Assembler::ucomisd(XMMRegister dst, Operand src) {
2267 EnsureSpace ensure_space(this);
2268 EMIT(0x66);
2269 EMIT(0x0F);
2270 EMIT(0x2E);
2271 emit_sse_operand(dst, src);
2272 }
2273
roundps(XMMRegister dst,XMMRegister src,RoundingMode mode)2274 void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2275 DCHECK(IsEnabled(SSE4_1));
2276 EnsureSpace ensure_space(this);
2277 EMIT(0x66);
2278 EMIT(0x0F);
2279 EMIT(0x3A);
2280 EMIT(0x08);
2281 emit_sse_operand(dst, src);
2282 // Mask precision exeption.
2283 EMIT(static_cast<byte>(mode) | 0x8);
2284 }
2285
roundpd(XMMRegister dst,XMMRegister src,RoundingMode mode)2286 void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2287 DCHECK(IsEnabled(SSE4_1));
2288 EnsureSpace ensure_space(this);
2289 EMIT(0x66);
2290 EMIT(0x0F);
2291 EMIT(0x3A);
2292 EMIT(0x09);
2293 emit_sse_operand(dst, src);
2294 // Mask precision exeption.
2295 EMIT(static_cast<byte>(mode) | 0x8);
2296 }
2297
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)2298 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2299 DCHECK(IsEnabled(SSE4_1));
2300 EnsureSpace ensure_space(this);
2301 EMIT(0x66);
2302 EMIT(0x0F);
2303 EMIT(0x3A);
2304 EMIT(0x0A);
2305 emit_sse_operand(dst, src);
2306 // Mask precision exeption.
2307 EMIT(static_cast<byte>(mode) | 0x8);
2308 }
2309
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)2310 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2311 DCHECK(IsEnabled(SSE4_1));
2312 EnsureSpace ensure_space(this);
2313 EMIT(0x66);
2314 EMIT(0x0F);
2315 EMIT(0x3A);
2316 EMIT(0x0B);
2317 emit_sse_operand(dst, src);
2318 // Mask precision exeption.
2319 EMIT(static_cast<byte>(mode) | 0x8);
2320 }
2321
movmskpd(Register dst,XMMRegister src)2322 void Assembler::movmskpd(Register dst, XMMRegister src) {
2323 EnsureSpace ensure_space(this);
2324 EMIT(0x66);
2325 EMIT(0x0F);
2326 EMIT(0x50);
2327 emit_sse_operand(dst, src);
2328 }
2329
movmskps(Register dst,XMMRegister src)2330 void Assembler::movmskps(Register dst, XMMRegister src) {
2331 EnsureSpace ensure_space(this);
2332 EMIT(0x0F);
2333 EMIT(0x50);
2334 emit_sse_operand(dst, src);
2335 }
2336
pmovmskb(Register dst,XMMRegister src)2337 void Assembler::pmovmskb(Register dst, XMMRegister src) {
2338 EnsureSpace ensure_space(this);
2339 EMIT(0x66);
2340 EMIT(0x0F);
2341 EMIT(0xD7);
2342 emit_sse_operand(dst, src);
2343 }
2344
maxsd(XMMRegister dst,Operand src)2345 void Assembler::maxsd(XMMRegister dst, Operand src) {
2346 EnsureSpace ensure_space(this);
2347 EMIT(0xF2);
2348 EMIT(0x0F);
2349 EMIT(0x5F);
2350 emit_sse_operand(dst, src);
2351 }
2352
minsd(XMMRegister dst,Operand src)2353 void Assembler::minsd(XMMRegister dst, Operand src) {
2354 EnsureSpace ensure_space(this);
2355 EMIT(0xF2);
2356 EMIT(0x0F);
2357 EMIT(0x5D);
2358 emit_sse_operand(dst, src);
2359 }
2360
cmpltsd(XMMRegister dst,XMMRegister src)2361 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
2362 EnsureSpace ensure_space(this);
2363 EMIT(0xF2);
2364 EMIT(0x0F);
2365 EMIT(0xC2);
2366 emit_sse_operand(dst, src);
2367 EMIT(1); // LT == 1
2368 }
2369
movaps(XMMRegister dst,Operand src)2370 void Assembler::movaps(XMMRegister dst, Operand src) {
2371 EnsureSpace ensure_space(this);
2372 EMIT(0x0F);
2373 EMIT(0x28);
2374 emit_sse_operand(dst, src);
2375 }
2376
movups(XMMRegister dst,Operand src)2377 void Assembler::movups(XMMRegister dst, Operand src) {
2378 EnsureSpace ensure_space(this);
2379 EMIT(0x0F);
2380 EMIT(0x10);
2381 emit_sse_operand(dst, src);
2382 }
2383
movups(Operand dst,XMMRegister src)2384 void Assembler::movups(Operand dst, XMMRegister src) {
2385 EnsureSpace ensure_space(this);
2386 EMIT(0x0F);
2387 EMIT(0x11);
2388 emit_sse_operand(src, dst);
2389 }
2390
movddup(XMMRegister dst,Operand src)2391 void Assembler::movddup(XMMRegister dst, Operand src) {
2392 DCHECK(IsEnabled(SSE3));
2393 EnsureSpace ensure_space(this);
2394 EMIT(0xF2);
2395 EMIT(0x0F);
2396 EMIT(0x12);
2397 emit_sse_operand(dst, src);
2398 }
2399
shufps(XMMRegister dst,XMMRegister src,byte imm8)2400 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2401 DCHECK(is_uint8(imm8));
2402 EnsureSpace ensure_space(this);
2403 EMIT(0x0F);
2404 EMIT(0xC6);
2405 emit_sse_operand(dst, src);
2406 EMIT(imm8);
2407 }
2408
shufpd(XMMRegister dst,XMMRegister src,byte imm8)2409 void Assembler::shufpd(XMMRegister dst, XMMRegister src, byte imm8) {
2410 DCHECK(is_uint8(imm8));
2411 EnsureSpace ensure_space(this);
2412 EMIT(0x66);
2413 EMIT(0x0F);
2414 EMIT(0xC6);
2415 emit_sse_operand(dst, src);
2416 EMIT(imm8);
2417 }
2418
movdqa(Operand dst,XMMRegister src)2419 void Assembler::movdqa(Operand dst, XMMRegister src) {
2420 EnsureSpace ensure_space(this);
2421 EMIT(0x66);
2422 EMIT(0x0F);
2423 EMIT(0x7F);
2424 emit_sse_operand(src, dst);
2425 }
2426
movdqa(XMMRegister dst,Operand src)2427 void Assembler::movdqa(XMMRegister dst, Operand src) {
2428 EnsureSpace ensure_space(this);
2429 EMIT(0x66);
2430 EMIT(0x0F);
2431 EMIT(0x6F);
2432 emit_sse_operand(dst, src);
2433 }
2434
movdqu(Operand dst,XMMRegister src)2435 void Assembler::movdqu(Operand dst, XMMRegister src) {
2436 EnsureSpace ensure_space(this);
2437 EMIT(0xF3);
2438 EMIT(0x0F);
2439 EMIT(0x7F);
2440 emit_sse_operand(src, dst);
2441 }
2442
movdqu(XMMRegister dst,Operand src)2443 void Assembler::movdqu(XMMRegister dst, Operand src) {
2444 EnsureSpace ensure_space(this);
2445 EMIT(0xF3);
2446 EMIT(0x0F);
2447 EMIT(0x6F);
2448 emit_sse_operand(dst, src);
2449 }
2450
prefetch(Operand src,int level)2451 void Assembler::prefetch(Operand src, int level) {
2452 DCHECK(is_uint2(level));
2453 EnsureSpace ensure_space(this);
2454 EMIT(0x0F);
2455 EMIT(0x18);
2456 // Emit hint number in Reg position of RegR/M.
2457 XMMRegister code = XMMRegister::from_code(level);
2458 emit_sse_operand(code, src);
2459 }
2460
movsd(Operand dst,XMMRegister src)2461 void Assembler::movsd(Operand dst, XMMRegister src) {
2462 EnsureSpace ensure_space(this);
2463 EMIT(0xF2); // double
2464 EMIT(0x0F);
2465 EMIT(0x11); // store
2466 emit_sse_operand(src, dst);
2467 }
2468
movsd(XMMRegister dst,Operand src)2469 void Assembler::movsd(XMMRegister dst, Operand src) {
2470 EnsureSpace ensure_space(this);
2471 EMIT(0xF2); // double
2472 EMIT(0x0F);
2473 EMIT(0x10); // load
2474 emit_sse_operand(dst, src);
2475 }
2476
movss(Operand dst,XMMRegister src)2477 void Assembler::movss(Operand dst, XMMRegister src) {
2478 EnsureSpace ensure_space(this);
2479 EMIT(0xF3); // float
2480 EMIT(0x0F);
2481 EMIT(0x11); // store
2482 emit_sse_operand(src, dst);
2483 }
2484
movss(XMMRegister dst,Operand src)2485 void Assembler::movss(XMMRegister dst, Operand src) {
2486 EnsureSpace ensure_space(this);
2487 EMIT(0xF3); // float
2488 EMIT(0x0F);
2489 EMIT(0x10); // load
2490 emit_sse_operand(dst, src);
2491 }
2492
movd(XMMRegister dst,Operand src)2493 void Assembler::movd(XMMRegister dst, Operand src) {
2494 EnsureSpace ensure_space(this);
2495 EMIT(0x66);
2496 EMIT(0x0F);
2497 EMIT(0x6E);
2498 emit_sse_operand(dst, src);
2499 }
2500
movd(Operand dst,XMMRegister src)2501 void Assembler::movd(Operand dst, XMMRegister src) {
2502 EnsureSpace ensure_space(this);
2503 EMIT(0x66);
2504 EMIT(0x0F);
2505 EMIT(0x7E);
2506 emit_sse_operand(src, dst);
2507 }
2508
extractps(Register dst,XMMRegister src,byte imm8)2509 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2510 DCHECK(IsEnabled(SSE4_1));
2511 DCHECK(is_uint8(imm8));
2512 EnsureSpace ensure_space(this);
2513 EMIT(0x66);
2514 EMIT(0x0F);
2515 EMIT(0x3A);
2516 EMIT(0x17);
2517 emit_sse_operand(src, dst);
2518 EMIT(imm8);
2519 }
2520
psllw(XMMRegister reg,uint8_t shift)2521 void Assembler::psllw(XMMRegister reg, uint8_t shift) {
2522 EnsureSpace ensure_space(this);
2523 EMIT(0x66);
2524 EMIT(0x0F);
2525 EMIT(0x71);
2526 emit_sse_operand(esi, reg); // esi == 6
2527 EMIT(shift);
2528 }
2529
pslld(XMMRegister reg,uint8_t shift)2530 void Assembler::pslld(XMMRegister reg, uint8_t shift) {
2531 EnsureSpace ensure_space(this);
2532 EMIT(0x66);
2533 EMIT(0x0F);
2534 EMIT(0x72);
2535 emit_sse_operand(esi, reg); // esi == 6
2536 EMIT(shift);
2537 }
2538
psrlw(XMMRegister reg,uint8_t shift)2539 void Assembler::psrlw(XMMRegister reg, uint8_t shift) {
2540 EnsureSpace ensure_space(this);
2541 EMIT(0x66);
2542 EMIT(0x0F);
2543 EMIT(0x71);
2544 emit_sse_operand(edx, reg); // edx == 2
2545 EMIT(shift);
2546 }
2547
psrld(XMMRegister reg,uint8_t shift)2548 void Assembler::psrld(XMMRegister reg, uint8_t shift) {
2549 EnsureSpace ensure_space(this);
2550 EMIT(0x66);
2551 EMIT(0x0F);
2552 EMIT(0x72);
2553 emit_sse_operand(edx, reg); // edx == 2
2554 EMIT(shift);
2555 }
2556
psraw(XMMRegister reg,uint8_t shift)2557 void Assembler::psraw(XMMRegister reg, uint8_t shift) {
2558 EnsureSpace ensure_space(this);
2559 EMIT(0x66);
2560 EMIT(0x0F);
2561 EMIT(0x71);
2562 emit_sse_operand(esp, reg); // esp == 4
2563 EMIT(shift);
2564 }
2565
psrad(XMMRegister reg,uint8_t shift)2566 void Assembler::psrad(XMMRegister reg, uint8_t shift) {
2567 EnsureSpace ensure_space(this);
2568 EMIT(0x66);
2569 EMIT(0x0F);
2570 EMIT(0x72);
2571 emit_sse_operand(esp, reg); // esp == 4
2572 EMIT(shift);
2573 }
2574
psllq(XMMRegister reg,uint8_t shift)2575 void Assembler::psllq(XMMRegister reg, uint8_t shift) {
2576 EnsureSpace ensure_space(this);
2577 EMIT(0x66);
2578 EMIT(0x0F);
2579 EMIT(0x73);
2580 emit_sse_operand(esi, reg); // esi == 6
2581 EMIT(shift);
2582 }
2583
psrlq(XMMRegister reg,uint8_t shift)2584 void Assembler::psrlq(XMMRegister reg, uint8_t shift) {
2585 EnsureSpace ensure_space(this);
2586 EMIT(0x66);
2587 EMIT(0x0F);
2588 EMIT(0x73);
2589 emit_sse_operand(edx, reg); // edx == 2
2590 EMIT(shift);
2591 }
2592
pshufhw(XMMRegister dst,Operand src,uint8_t shuffle)2593 void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
2594 EnsureSpace ensure_space(this);
2595 EMIT(0xF3);
2596 EMIT(0x0F);
2597 EMIT(0x70);
2598 emit_sse_operand(dst, src);
2599 EMIT(shuffle);
2600 }
2601
pshuflw(XMMRegister dst,Operand src,uint8_t shuffle)2602 void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
2603 EnsureSpace ensure_space(this);
2604 EMIT(0xF2);
2605 EMIT(0x0F);
2606 EMIT(0x70);
2607 emit_sse_operand(dst, src);
2608 EMIT(shuffle);
2609 }
2610
pshufd(XMMRegister dst,Operand src,uint8_t shuffle)2611 void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
2612 EnsureSpace ensure_space(this);
2613 EMIT(0x66);
2614 EMIT(0x0F);
2615 EMIT(0x70);
2616 emit_sse_operand(dst, src);
2617 EMIT(shuffle);
2618 }
2619
pblendw(XMMRegister dst,Operand src,uint8_t mask)2620 void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
2621 DCHECK(IsEnabled(SSE4_1));
2622 EnsureSpace ensure_space(this);
2623 EMIT(0x66);
2624 EMIT(0x0F);
2625 EMIT(0x3A);
2626 EMIT(0x0E);
2627 emit_sse_operand(dst, src);
2628 EMIT(mask);
2629 }
2630
palignr(XMMRegister dst,Operand src,uint8_t mask)2631 void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
2632 DCHECK(IsEnabled(SSSE3));
2633 EnsureSpace ensure_space(this);
2634 EMIT(0x66);
2635 EMIT(0x0F);
2636 EMIT(0x3A);
2637 EMIT(0x0F);
2638 emit_sse_operand(dst, src);
2639 EMIT(mask);
2640 }
2641
pextrb(Operand dst,XMMRegister src,uint8_t offset)2642 void Assembler::pextrb(Operand dst, XMMRegister src, uint8_t offset) {
2643 DCHECK(IsEnabled(SSE4_1));
2644 EnsureSpace ensure_space(this);
2645 EMIT(0x66);
2646 EMIT(0x0F);
2647 EMIT(0x3A);
2648 EMIT(0x14);
2649 emit_sse_operand(src, dst);
2650 EMIT(offset);
2651 }
2652
pextrw(Operand dst,XMMRegister src,uint8_t offset)2653 void Assembler::pextrw(Operand dst, XMMRegister src, uint8_t offset) {
2654 DCHECK(IsEnabled(SSE4_1));
2655 EnsureSpace ensure_space(this);
2656 EMIT(0x66);
2657 EMIT(0x0F);
2658 EMIT(0x3A);
2659 EMIT(0x15);
2660 emit_sse_operand(src, dst);
2661 EMIT(offset);
2662 }
2663
pextrd(Operand dst,XMMRegister src,uint8_t offset)2664 void Assembler::pextrd(Operand dst, XMMRegister src, uint8_t offset) {
2665 DCHECK(IsEnabled(SSE4_1));
2666 EnsureSpace ensure_space(this);
2667 EMIT(0x66);
2668 EMIT(0x0F);
2669 EMIT(0x3A);
2670 EMIT(0x16);
2671 emit_sse_operand(src, dst);
2672 EMIT(offset);
2673 }
2674
insertps(XMMRegister dst,Operand src,uint8_t offset)2675 void Assembler::insertps(XMMRegister dst, Operand src, uint8_t offset) {
2676 DCHECK(IsEnabled(SSE4_1));
2677 EnsureSpace ensure_space(this);
2678 EMIT(0x66);
2679 EMIT(0x0F);
2680 EMIT(0x3A);
2681 EMIT(0x21);
2682 emit_sse_operand(dst, src);
2683 EMIT(offset);
2684 }
2685
pinsrb(XMMRegister dst,Operand src,uint8_t offset)2686 void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t offset) {
2687 DCHECK(IsEnabled(SSE4_1));
2688 EnsureSpace ensure_space(this);
2689 EMIT(0x66);
2690 EMIT(0x0F);
2691 EMIT(0x3A);
2692 EMIT(0x20);
2693 emit_sse_operand(dst, src);
2694 EMIT(offset);
2695 }
2696
pinsrw(XMMRegister dst,Operand src,uint8_t offset)2697 void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t offset) {
2698 DCHECK(is_uint8(offset));
2699 EnsureSpace ensure_space(this);
2700 EMIT(0x66);
2701 EMIT(0x0F);
2702 EMIT(0xC4);
2703 emit_sse_operand(dst, src);
2704 EMIT(offset);
2705 }
2706
pinsrd(XMMRegister dst,Operand src,uint8_t offset)2707 void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t offset) {
2708 DCHECK(IsEnabled(SSE4_1));
2709 EnsureSpace ensure_space(this);
2710 EMIT(0x66);
2711 EMIT(0x0F);
2712 EMIT(0x3A);
2713 EMIT(0x22);
2714 emit_sse_operand(dst, src);
2715 EMIT(offset);
2716 }
2717
addss(XMMRegister dst,Operand src)2718 void Assembler::addss(XMMRegister dst, Operand src) {
2719 EnsureSpace ensure_space(this);
2720 EMIT(0xF3);
2721 EMIT(0x0F);
2722 EMIT(0x58);
2723 emit_sse_operand(dst, src);
2724 }
2725
subss(XMMRegister dst,Operand src)2726 void Assembler::subss(XMMRegister dst, Operand src) {
2727 EnsureSpace ensure_space(this);
2728 EMIT(0xF3);
2729 EMIT(0x0F);
2730 EMIT(0x5C);
2731 emit_sse_operand(dst, src);
2732 }
2733
mulss(XMMRegister dst,Operand src)2734 void Assembler::mulss(XMMRegister dst, Operand src) {
2735 EnsureSpace ensure_space(this);
2736 EMIT(0xF3);
2737 EMIT(0x0F);
2738 EMIT(0x59);
2739 emit_sse_operand(dst, src);
2740 }
2741
divss(XMMRegister dst,Operand src)2742 void Assembler::divss(XMMRegister dst, Operand src) {
2743 EnsureSpace ensure_space(this);
2744 EMIT(0xF3);
2745 EMIT(0x0F);
2746 EMIT(0x5E);
2747 emit_sse_operand(dst, src);
2748 }
2749
sqrtss(XMMRegister dst,Operand src)2750 void Assembler::sqrtss(XMMRegister dst, Operand src) {
2751 EnsureSpace ensure_space(this);
2752 EMIT(0xF3);
2753 EMIT(0x0F);
2754 EMIT(0x51);
2755 emit_sse_operand(dst, src);
2756 }
2757
ucomiss(XMMRegister dst,Operand src)2758 void Assembler::ucomiss(XMMRegister dst, Operand src) {
2759 EnsureSpace ensure_space(this);
2760 EMIT(0x0F);
2761 EMIT(0x2E);
2762 emit_sse_operand(dst, src);
2763 }
2764
maxss(XMMRegister dst,Operand src)2765 void Assembler::maxss(XMMRegister dst, Operand src) {
2766 EnsureSpace ensure_space(this);
2767 EMIT(0xF3);
2768 EMIT(0x0F);
2769 EMIT(0x5F);
2770 emit_sse_operand(dst, src);
2771 }
2772
minss(XMMRegister dst,Operand src)2773 void Assembler::minss(XMMRegister dst, Operand src) {
2774 EnsureSpace ensure_space(this);
2775 EMIT(0xF3);
2776 EMIT(0x0F);
2777 EMIT(0x5D);
2778 emit_sse_operand(dst, src);
2779 }
2780
2781 // Packed single-precision floating-point SSE instructions.
ps(byte opcode,XMMRegister dst,Operand src)2782 void Assembler::ps(byte opcode, XMMRegister dst, Operand src) {
2783 EnsureSpace ensure_space(this);
2784 EMIT(0x0F);
2785 EMIT(opcode);
2786 emit_sse_operand(dst, src);
2787 }
2788
2789 // Packed double-precision floating-point SSE instructions.
pd(byte opcode,XMMRegister dst,Operand src)2790 void Assembler::pd(byte opcode, XMMRegister dst, Operand src) {
2791 EnsureSpace ensure_space(this);
2792 EMIT(0x66);
2793 EMIT(0x0F);
2794 EMIT(opcode);
2795 emit_sse_operand(dst, src);
2796 }
2797
2798 // AVX instructions
vfmasd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2799 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
2800 Operand src2) {
2801 DCHECK(IsEnabled(FMA3));
2802 EnsureSpace ensure_space(this);
2803 emit_vex_prefix(src1, kLIG, k66, k0F38, kW1);
2804 EMIT(op);
2805 emit_sse_operand(dst, src2);
2806 }
2807
vfmass(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2808 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
2809 Operand src2) {
2810 DCHECK(IsEnabled(FMA3));
2811 EnsureSpace ensure_space(this);
2812 emit_vex_prefix(src1, kLIG, k66, k0F38, kW0);
2813 EMIT(op);
2814 emit_sse_operand(dst, src2);
2815 }
2816
vsd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2817 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
2818 vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
2819 }
2820
vss(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2821 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
2822 vinstr(op, dst, src1, src2, kF3, k0F, kWIG);
2823 }
2824
vps(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2825 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
2826 vinstr(op, dst, src1, src2, kNone, k0F, kWIG);
2827 }
2828
vpd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)2829 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
2830 vinstr(op, dst, src1, src2, k66, k0F, kWIG);
2831 }
2832
vshufpd(XMMRegister dst,XMMRegister src1,Operand src2,byte imm8)2833 void Assembler::vshufpd(XMMRegister dst, XMMRegister src1, Operand src2,
2834 byte imm8) {
2835 DCHECK(is_uint8(imm8));
2836 vpd(0xC6, dst, src1, src2);
2837 EMIT(imm8);
2838 }
2839
vcmpps(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t cmp)2840 void Assembler::vcmpps(XMMRegister dst, XMMRegister src1, Operand src2,
2841 uint8_t cmp) {
2842 vps(0xC2, dst, src1, src2);
2843 EMIT(cmp);
2844 }
2845
vcmppd(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t cmp)2846 void Assembler::vcmppd(XMMRegister dst, XMMRegister src1, Operand src2,
2847 uint8_t cmp) {
2848 vpd(0xC2, dst, src1, src2);
2849 EMIT(cmp);
2850 }
2851
vshufps(XMMRegister dst,XMMRegister src1,Operand src2,byte imm8)2852 void Assembler::vshufps(XMMRegister dst, XMMRegister src1, Operand src2,
2853 byte imm8) {
2854 DCHECK(is_uint8(imm8));
2855 vps(0xC6, dst, src1, src2);
2856 EMIT(imm8);
2857 }
2858
vpsllw(XMMRegister dst,XMMRegister src,uint8_t imm8)2859 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2860 XMMRegister iop = XMMRegister::from_code(6);
2861 vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
2862 EMIT(imm8);
2863 }
2864
vpslld(XMMRegister dst,XMMRegister src,uint8_t imm8)2865 void Assembler::vpslld(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2866 XMMRegister iop = XMMRegister::from_code(6);
2867 vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
2868 EMIT(imm8);
2869 }
2870
vpsllq(XMMRegister dst,XMMRegister src,uint8_t imm8)2871 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2872 XMMRegister iop = XMMRegister::from_code(6);
2873 vinstr(0x73, iop, dst, Operand(src), k66, k0F, kWIG);
2874 EMIT(imm8);
2875 }
2876
vpsrlw(XMMRegister dst,XMMRegister src,uint8_t imm8)2877 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2878 XMMRegister iop = XMMRegister::from_code(2);
2879 vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
2880 EMIT(imm8);
2881 }
2882
vpsrld(XMMRegister dst,XMMRegister src,uint8_t imm8)2883 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2884 XMMRegister iop = XMMRegister::from_code(2);
2885 vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
2886 EMIT(imm8);
2887 }
2888
vpsrlq(XMMRegister dst,XMMRegister src,uint8_t imm8)2889 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2890 XMMRegister iop = XMMRegister::from_code(2);
2891 vinstr(0x73, iop, dst, Operand(src), k66, k0F, kWIG);
2892 EMIT(imm8);
2893 }
2894
vpsraw(XMMRegister dst,XMMRegister src,uint8_t imm8)2895 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2896 XMMRegister iop = XMMRegister::from_code(4);
2897 vinstr(0x71, iop, dst, Operand(src), k66, k0F, kWIG);
2898 EMIT(imm8);
2899 }
2900
vpsrad(XMMRegister dst,XMMRegister src,uint8_t imm8)2901 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, uint8_t imm8) {
2902 XMMRegister iop = XMMRegister::from_code(4);
2903 vinstr(0x72, iop, dst, Operand(src), k66, k0F, kWIG);
2904 EMIT(imm8);
2905 }
2906
vpshufhw(XMMRegister dst,Operand src,uint8_t shuffle)2907 void Assembler::vpshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
2908 vinstr(0x70, dst, xmm0, src, kF3, k0F, kWIG);
2909 EMIT(shuffle);
2910 }
2911
vpshuflw(XMMRegister dst,Operand src,uint8_t shuffle)2912 void Assembler::vpshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
2913 vinstr(0x70, dst, xmm0, src, kF2, k0F, kWIG);
2914 EMIT(shuffle);
2915 }
2916
vpshufd(XMMRegister dst,Operand src,uint8_t shuffle)2917 void Assembler::vpshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
2918 vinstr(0x70, dst, xmm0, src, k66, k0F, kWIG);
2919 EMIT(shuffle);
2920 }
2921
vpblendw(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t mask)2922 void Assembler::vpblendw(XMMRegister dst, XMMRegister src1, Operand src2,
2923 uint8_t mask) {
2924 vinstr(0x0E, dst, src1, src2, k66, k0F3A, kWIG);
2925 EMIT(mask);
2926 }
2927
vpalignr(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t mask)2928 void Assembler::vpalignr(XMMRegister dst, XMMRegister src1, Operand src2,
2929 uint8_t mask) {
2930 vinstr(0x0F, dst, src1, src2, k66, k0F3A, kWIG);
2931 EMIT(mask);
2932 }
2933
vpextrb(Operand dst,XMMRegister src,uint8_t offset)2934 void Assembler::vpextrb(Operand dst, XMMRegister src, uint8_t offset) {
2935 vinstr(0x14, src, xmm0, dst, k66, k0F3A, kWIG);
2936 EMIT(offset);
2937 }
2938
vpextrw(Operand dst,XMMRegister src,uint8_t offset)2939 void Assembler::vpextrw(Operand dst, XMMRegister src, uint8_t offset) {
2940 vinstr(0x15, src, xmm0, dst, k66, k0F3A, kWIG);
2941 EMIT(offset);
2942 }
2943
vpextrd(Operand dst,XMMRegister src,uint8_t offset)2944 void Assembler::vpextrd(Operand dst, XMMRegister src, uint8_t offset) {
2945 vinstr(0x16, src, xmm0, dst, k66, k0F3A, kWIG);
2946 EMIT(offset);
2947 }
2948
vinsertps(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t offset)2949 void Assembler::vinsertps(XMMRegister dst, XMMRegister src1, Operand src2,
2950 uint8_t offset) {
2951 vinstr(0x21, dst, src1, src2, k66, k0F3A, kWIG);
2952 EMIT(offset);
2953 }
2954
vpinsrb(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t offset)2955 void Assembler::vpinsrb(XMMRegister dst, XMMRegister src1, Operand src2,
2956 uint8_t offset) {
2957 vinstr(0x20, dst, src1, src2, k66, k0F3A, kWIG);
2958 EMIT(offset);
2959 }
2960
vpinsrw(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t offset)2961 void Assembler::vpinsrw(XMMRegister dst, XMMRegister src1, Operand src2,
2962 uint8_t offset) {
2963 vinstr(0xC4, dst, src1, src2, k66, k0F, kWIG);
2964 EMIT(offset);
2965 }
2966
vpinsrd(XMMRegister dst,XMMRegister src1,Operand src2,uint8_t offset)2967 void Assembler::vpinsrd(XMMRegister dst, XMMRegister src1, Operand src2,
2968 uint8_t offset) {
2969 vinstr(0x22, dst, src1, src2, k66, k0F3A, kWIG);
2970 EMIT(offset);
2971 }
2972
vroundps(XMMRegister dst,XMMRegister src,RoundingMode mode)2973 void Assembler::vroundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2974 vinstr(0x08, dst, xmm0, Operand(src), k66, k0F3A, kWIG);
2975 EMIT(static_cast<byte>(mode) | 0x8); // Mask precision exception.
2976 }
vroundpd(XMMRegister dst,XMMRegister src,RoundingMode mode)2977 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
2978 vinstr(0x09, dst, xmm0, Operand(src), k66, k0F3A, kWIG);
2979 EMIT(static_cast<byte>(mode) | 0x8); // Mask precision exception.
2980 }
2981
vmovmskps(Register dst,XMMRegister src)2982 void Assembler::vmovmskps(Register dst, XMMRegister src) {
2983 DCHECK(IsEnabled(AVX));
2984 EnsureSpace ensure_space(this);
2985 emit_vex_prefix(xmm0, kL128, kNone, k0F, kWIG);
2986 EMIT(0x50);
2987 emit_sse_operand(dst, src);
2988 }
2989
vpmovmskb(Register dst,XMMRegister src)2990 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
2991 DCHECK(IsEnabled(AVX));
2992 EnsureSpace ensure_space(this);
2993 emit_vex_prefix(xmm0, kL128, k66, k0F, kWIG);
2994 EMIT(0xD7);
2995 emit_sse_operand(dst, src);
2996 }
2997
bmi1(byte op,Register reg,Register vreg,Operand rm)2998 void Assembler::bmi1(byte op, Register reg, Register vreg, Operand rm) {
2999 DCHECK(IsEnabled(BMI1));
3000 EnsureSpace ensure_space(this);
3001 emit_vex_prefix(vreg, kLZ, kNone, k0F38, kW0);
3002 EMIT(op);
3003 emit_operand(reg, rm);
3004 }
3005
tzcnt(Register dst,Operand src)3006 void Assembler::tzcnt(Register dst, Operand src) {
3007 DCHECK(IsEnabled(BMI1));
3008 EnsureSpace ensure_space(this);
3009 EMIT(0xF3);
3010 EMIT(0x0F);
3011 EMIT(0xBC);
3012 emit_operand(dst, src);
3013 }
3014
lzcnt(Register dst,Operand src)3015 void Assembler::lzcnt(Register dst, Operand src) {
3016 DCHECK(IsEnabled(LZCNT));
3017 EnsureSpace ensure_space(this);
3018 EMIT(0xF3);
3019 EMIT(0x0F);
3020 EMIT(0xBD);
3021 emit_operand(dst, src);
3022 }
3023
popcnt(Register dst,Operand src)3024 void Assembler::popcnt(Register dst, Operand src) {
3025 DCHECK(IsEnabled(POPCNT));
3026 EnsureSpace ensure_space(this);
3027 EMIT(0xF3);
3028 EMIT(0x0F);
3029 EMIT(0xB8);
3030 emit_operand(dst, src);
3031 }
3032
bmi2(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)3033 void Assembler::bmi2(SIMDPrefix pp, byte op, Register reg, Register vreg,
3034 Operand rm) {
3035 DCHECK(IsEnabled(BMI2));
3036 EnsureSpace ensure_space(this);
3037 emit_vex_prefix(vreg, kLZ, pp, k0F38, kW0);
3038 EMIT(op);
3039 emit_operand(reg, rm);
3040 }
3041
rorx(Register dst,Operand src,byte imm8)3042 void Assembler::rorx(Register dst, Operand src, byte imm8) {
3043 DCHECK(IsEnabled(BMI2));
3044 DCHECK(is_uint8(imm8));
3045 Register vreg = Register::from_code(0); // VEX.vvvv unused
3046 EnsureSpace ensure_space(this);
3047 emit_vex_prefix(vreg, kLZ, kF2, k0F3A, kW0);
3048 EMIT(0xF0);
3049 emit_operand(dst, src);
3050 EMIT(imm8);
3051 }
3052
sse2_instr(XMMRegister dst,Operand src,byte prefix,byte escape,byte opcode)3053 void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
3054 byte escape, byte opcode) {
3055 EnsureSpace ensure_space(this);
3056 EMIT(prefix);
3057 EMIT(escape);
3058 EMIT(opcode);
3059 emit_sse_operand(dst, src);
3060 }
3061
ssse3_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)3062 void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
3063 byte escape1, byte escape2, byte opcode) {
3064 DCHECK(IsEnabled(SSSE3));
3065 EnsureSpace ensure_space(this);
3066 EMIT(prefix);
3067 EMIT(escape1);
3068 EMIT(escape2);
3069 EMIT(opcode);
3070 emit_sse_operand(dst, src);
3071 }
3072
sse4_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)3073 void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
3074 byte escape1, byte escape2, byte opcode) {
3075 DCHECK(IsEnabled(SSE4_1));
3076 EnsureSpace ensure_space(this);
3077 EMIT(prefix);
3078 EMIT(escape1);
3079 EMIT(escape2);
3080 EMIT(opcode);
3081 emit_sse_operand(dst, src);
3082 }
3083
vinstr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,SIMDPrefix pp,LeadingOpcode m,VexW w)3084 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
3085 SIMDPrefix pp, LeadingOpcode m, VexW w) {
3086 DCHECK(IsEnabled(AVX));
3087 EnsureSpace ensure_space(this);
3088 emit_vex_prefix(src1, kL128, pp, m, w);
3089 EMIT(op);
3090 emit_sse_operand(dst, src2);
3091 }
3092
emit_sse_operand(XMMRegister reg,Operand adr)3093 void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
3094 Register ireg = Register::from_code(reg.code());
3095 emit_operand(ireg, adr);
3096 }
3097
emit_sse_operand(XMMRegister dst,XMMRegister src)3098 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
3099 EMIT(0xC0 | dst.code() << 3 | src.code());
3100 }
3101
emit_sse_operand(Register dst,XMMRegister src)3102 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
3103 EMIT(0xC0 | dst.code() << 3 | src.code());
3104 }
3105
emit_sse_operand(XMMRegister dst,Register src)3106 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
3107 EMIT(0xC0 | (dst.code() << 3) | src.code());
3108 }
3109
emit_vex_prefix(XMMRegister vreg,VectorLength l,SIMDPrefix pp,LeadingOpcode mm,VexW w)3110 void Assembler::emit_vex_prefix(XMMRegister vreg, VectorLength l, SIMDPrefix pp,
3111 LeadingOpcode mm, VexW w) {
3112 if (mm != k0F || w != kW0) {
3113 EMIT(0xC4);
3114 // Change RXB from "110" to "111" to align with gdb disassembler.
3115 EMIT(0xE0 | mm);
3116 EMIT(w | ((~vreg.code() & 0xF) << 3) | l | pp);
3117 } else {
3118 EMIT(0xC5);
3119 EMIT(((~vreg.code()) << 3) | l | pp);
3120 }
3121 }
3122
emit_vex_prefix(Register vreg,VectorLength l,SIMDPrefix pp,LeadingOpcode mm,VexW w)3123 void Assembler::emit_vex_prefix(Register vreg, VectorLength l, SIMDPrefix pp,
3124 LeadingOpcode mm, VexW w) {
3125 XMMRegister ivreg = XMMRegister::from_code(vreg.code());
3126 emit_vex_prefix(ivreg, l, pp, mm, w);
3127 }
3128
GrowBuffer()3129 void Assembler::GrowBuffer() {
3130 DCHECK(buffer_overflow());
3131 DCHECK_EQ(buffer_start_, buffer_->start());
3132
3133 // Compute new buffer size.
3134 int old_size = buffer_->size();
3135 int new_size = 2 * old_size;
3136
3137 // Some internal data structures overflow for very large buffers,
3138 // they must ensure that kMaximalBufferSize is not too large.
3139 if (new_size > kMaximalBufferSize) {
3140 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
3141 }
3142
3143 // Set up new buffer.
3144 std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
3145 DCHECK_EQ(new_size, new_buffer->size());
3146 byte* new_start = new_buffer->start();
3147
3148 // Copy the data.
3149 intptr_t pc_delta = new_start - buffer_start_;
3150 intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
3151 size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
3152 MemMove(new_start, buffer_start_, pc_offset());
3153 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
3154 reloc_size);
3155
3156 // Switch buffers.
3157 buffer_ = std::move(new_buffer);
3158 buffer_start_ = new_start;
3159 pc_ += pc_delta;
3160 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
3161 reloc_info_writer.last_pc() + pc_delta);
3162
3163 // Relocate internal references.
3164 for (auto pos : internal_reference_positions_) {
3165 Address p = reinterpret_cast<Address>(buffer_start_ + pos);
3166 WriteUnalignedValue(p, ReadUnalignedValue<int>(p) + pc_delta);
3167 }
3168
3169 // Relocate pc-relative references.
3170 int mode_mask = RelocInfo::ModeMask(RelocInfo::OFF_HEAP_TARGET);
3171 DCHECK_EQ(mode_mask, RelocInfo::kApplyMask & mode_mask);
3172 Vector<byte> instructions{buffer_start_, static_cast<size_t>(pc_offset())};
3173 Vector<const byte> reloc_info{reloc_info_writer.pos(), reloc_size};
3174 for (RelocIterator it(instructions, reloc_info, 0, mode_mask); !it.done();
3175 it.next()) {
3176 it.rinfo()->apply(pc_delta);
3177 }
3178
3179 DCHECK(!buffer_overflow());
3180 }
3181
emit_arith_b(int op1,int op2,Register dst,int imm8)3182 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
3183 DCHECK(is_uint8(op1) && is_uint8(op2)); // wrong opcode
3184 DCHECK(is_uint8(imm8));
3185 DCHECK_EQ(op1 & 0x01, 0); // should be 8bit operation
3186 EMIT(op1);
3187 EMIT(op2 | dst.code());
3188 EMIT(imm8);
3189 }
3190
emit_arith(int sel,Operand dst,const Immediate & x)3191 void Assembler::emit_arith(int sel, Operand dst, const Immediate& x) {
3192 DCHECK((0 <= sel) && (sel <= 7));
3193 Register ireg = Register::from_code(sel);
3194 if (x.is_int8()) {
3195 EMIT(0x83); // using a sign-extended 8-bit immediate.
3196 emit_operand(ireg, dst);
3197 EMIT(x.immediate() & 0xFF);
3198 } else if (dst.is_reg(eax)) {
3199 EMIT((sel << 3) | 0x05); // short form if the destination is eax.
3200 emit(x);
3201 } else {
3202 EMIT(0x81); // using a literal 32-bit immediate.
3203 emit_operand(ireg, dst);
3204 emit(x);
3205 }
3206 }
3207
emit_operand(Register reg,Operand adr)3208 void Assembler::emit_operand(Register reg, Operand adr) {
3209 emit_operand(reg.code(), adr);
3210 }
3211
emit_operand(XMMRegister reg,Operand adr)3212 void Assembler::emit_operand(XMMRegister reg, Operand adr) {
3213 Register ireg = Register::from_code(reg.code());
3214 emit_operand(ireg, adr);
3215 }
3216
emit_operand(int code,Operand adr)3217 void Assembler::emit_operand(int code, Operand adr) {
3218 // Isolate-independent code may not embed relocatable addresses.
3219 DCHECK(!options().isolate_independent_code ||
3220 adr.rmode_ != RelocInfo::CODE_TARGET);
3221 DCHECK(!options().isolate_independent_code ||
3222 adr.rmode_ != RelocInfo::FULL_EMBEDDED_OBJECT);
3223 DCHECK(!options().isolate_independent_code ||
3224 adr.rmode_ != RelocInfo::EXTERNAL_REFERENCE);
3225
3226 const unsigned length = adr.len_;
3227 DCHECK_GT(length, 0);
3228
3229 // Emit updated ModRM byte containing the given register.
3230 EMIT((adr.buf_[0] & ~0x38) | (code << 3));
3231
3232 // Emit the rest of the encoded operand.
3233 for (unsigned i = 1; i < length; i++) EMIT(adr.buf_[i]);
3234
3235 // Emit relocation information if necessary.
3236 if (length >= sizeof(int32_t) && !RelocInfo::IsNone(adr.rmode_)) {
3237 pc_ -= sizeof(int32_t); // pc_ must be *at* disp32
3238 RecordRelocInfo(adr.rmode_);
3239 if (adr.rmode_ == RelocInfo::INTERNAL_REFERENCE) { // Fixup for labels
3240 emit_label(ReadUnalignedValue<Label*>(reinterpret_cast<Address>(pc_)));
3241 } else {
3242 pc_ += sizeof(int32_t);
3243 }
3244 }
3245 }
3246
emit_label(Label * label)3247 void Assembler::emit_label(Label* label) {
3248 if (label->is_bound()) {
3249 internal_reference_positions_.push_back(pc_offset());
3250 emit(reinterpret_cast<uint32_t>(buffer_start_ + label->pos()));
3251 } else {
3252 emit_disp(label, Displacement::CODE_ABSOLUTE);
3253 }
3254 }
3255
emit_farith(int b1,int b2,int i)3256 void Assembler::emit_farith(int b1, int b2, int i) {
3257 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
3258 DCHECK(0 <= i && i < 8); // illegal stack offset
3259 EMIT(b1);
3260 EMIT(b2 + i);
3261 }
3262
db(uint8_t data)3263 void Assembler::db(uint8_t data) {
3264 EnsureSpace ensure_space(this);
3265 EMIT(data);
3266 }
3267
dd(uint32_t data)3268 void Assembler::dd(uint32_t data) {
3269 EnsureSpace ensure_space(this);
3270 emit(data);
3271 }
3272
dq(uint64_t data)3273 void Assembler::dq(uint64_t data) {
3274 EnsureSpace ensure_space(this);
3275 emit_q(data);
3276 }
3277
dd(Label * label)3278 void Assembler::dd(Label* label) {
3279 EnsureSpace ensure_space(this);
3280 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
3281 emit_label(label);
3282 }
3283
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)3284 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
3285 if (!ShouldRecordRelocInfo(rmode)) return;
3286 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
3287 reloc_info_writer.Write(&rinfo);
3288 }
3289
3290 #undef EMIT
3291
3292 } // namespace internal
3293 } // namespace v8
3294
3295 #endif // V8_TARGET_ARCH_IA32
3296