1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/base/bits.h"
19 #include "src/base/cpu.h"
20 #include "src/codegen/assembler-inl.h"
21 #include "src/codegen/macro-assembler.h"
22 #include "src/codegen/string-constants.h"
23 #include "src/deoptimizer/deoptimizer.h"
24 #include "src/init/v8.h"
25
26 namespace v8 {
27 namespace internal {
28
29 // -----------------------------------------------------------------------------
30 // Implementation of CpuFeatures
31
32 namespace {
33
xgetbv(unsigned int xcr)34 V8_INLINE uint64_t xgetbv(unsigned int xcr) {
35 #if V8_LIBC_MSVCRT
36 return _xgetbv(xcr);
37 #else
38 unsigned eax, edx;
39 // Check xgetbv; this uses a .byte sequence instead of the instruction
40 // directly because older assemblers do not include support for xgetbv and
41 // there is no easy way to conditionally compile based on the assembler
42 // used.
43 __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
44 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
45 #endif
46 }
47
OSHasAVXSupport()48 bool OSHasAVXSupport() {
49 #if V8_OS_MACOSX
50 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
51 // caused by ISRs, so we detect that here and disable AVX in that case.
52 char buffer[128];
53 size_t buffer_size = arraysize(buffer);
54 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
55 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
56 FATAL("V8 failed to get kernel version");
57 }
58 // The buffer now contains a string of the form XX.YY.ZZ, where
59 // XX is the major kernel version component.
60 char* period_pos = strchr(buffer, '.');
61 DCHECK_NOT_NULL(period_pos);
62 *period_pos = '\0';
63 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
64 if (kernel_version_major <= 13) return false;
65 #endif // V8_OS_MACOSX
66 // Check whether OS claims to support AVX.
67 uint64_t feature_mask = xgetbv(0); // XCR_XFEATURE_ENABLED_MASK
68 return (feature_mask & 0x6) == 0x6;
69 }
70
71 } // namespace
72
ProbeImpl(bool cross_compile)73 void CpuFeatures::ProbeImpl(bool cross_compile) {
74 base::CPU cpu;
75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
76 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
77
78 // Only use statically determined features for cross compile (snapshot).
79 if (cross_compile) return;
80
81 if (cpu.has_sse42() && FLAG_enable_sse4_2) supported_ |= 1u << SSE4_2;
82 if (cpu.has_sse41() && FLAG_enable_sse4_1) {
83 supported_ |= 1u << SSE4_1;
84 supported_ |= 1u << SSSE3;
85 }
86 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
87 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
88 // SAHF is not generally available in long mode.
89 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
90 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
91 OSHasAVXSupport()) {
92 supported_ |= 1u << AVX;
93 }
94 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
95 OSHasAVXSupport()) {
96 supported_ |= 1u << FMA3;
97 }
98 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
99 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
100 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
101 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
102 if (strcmp(FLAG_mcpu, "auto") == 0) {
103 if (cpu.is_atom()) supported_ |= 1u << ATOM;
104 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
105 supported_ |= 1u << ATOM;
106 }
107 }
108
PrintTarget()109 void CpuFeatures::PrintTarget() {}
PrintFeatures()110 void CpuFeatures::PrintFeatures() {
111 printf(
112 "SSE3=%d SSSE3=%d SSE4_1=%d SSE4_2=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d "
113 "BMI2=%d "
114 "LZCNT=%d "
115 "POPCNT=%d ATOM=%d\n",
116 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
117 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SSE4_2),
118 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
119 CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
120 CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
121 CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
122 }
123
124 // -----------------------------------------------------------------------------
125 // Implementation of RelocInfo
126
wasm_call_tag() const127 uint32_t RelocInfo::wasm_call_tag() const {
128 DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
129 return ReadUnalignedValue<uint32_t>(pc_);
130 }
131
132 // -----------------------------------------------------------------------------
133 // Implementation of Operand
134
Operand(Operand operand,int32_t offset)135 Operand::Operand(Operand operand, int32_t offset) {
136 DCHECK_GE(operand.data().len, 1);
137 // Operand encodes REX ModR/M [SIB] [Disp].
138 byte modrm = operand.data().buf[0];
139 DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
140 bool has_sib = ((modrm & 0x07) == 0x04);
141 byte mode = modrm & 0xC0;
142 int disp_offset = has_sib ? 2 : 1;
143 int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
144 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
145 // displacement.
146 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
147 int32_t disp_value = 0;
148 if (mode == 0x80 || is_baseless) {
149 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
150 disp_value = ReadUnalignedValue<int32_t>(
151 reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
152 } else if (mode == 0x40) {
153 // Mode 1: Byte displacement.
154 disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
155 }
156
157 // Write new operand with same registers, but with modified displacement.
158 DCHECK(offset >= 0 ? disp_value + offset > disp_value
159 : disp_value + offset < disp_value); // No overflow.
160 disp_value += offset;
161 data_.rex = operand.data().rex;
162 if (!is_int8(disp_value) || is_baseless) {
163 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
164 data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
165 data_.len = disp_offset + 4;
166 WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
167 disp_value);
168 } else if (disp_value != 0 || (base_reg == 0x05)) {
169 // Need 8 bits of displacement.
170 data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
171 data_.len = disp_offset + 1;
172 data_.buf[disp_offset] = static_cast<byte>(disp_value);
173 } else {
174 // Need no displacement.
175 data_.buf[0] = (modrm & 0x3F); // Mode 0.
176 data_.len = disp_offset;
177 }
178 if (has_sib) {
179 data_.buf[1] = operand.data().buf[1];
180 }
181 }
182
AddressUsesRegister(Register reg) const183 bool Operand::AddressUsesRegister(Register reg) const {
184 int code = reg.code();
185 DCHECK_NE(data_.buf[0] & 0xC0, 0xC0); // Always a memory operand.
186 // Start with only low three bits of base register. Initial decoding
187 // doesn't distinguish on the REX.B bit.
188 int base_code = data_.buf[0] & 0x07;
189 if (base_code == rsp.code()) {
190 // SIB byte present in buf_[1].
191 // Check the index register from the SIB byte + REX.X prefix.
192 int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
193 // Index code (including REX.X) of 0x04 (rsp) means no index register.
194 if (index_code != rsp.code() && index_code == code) return true;
195 // Add REX.B to get the full base register code.
196 base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
197 // A base register of 0x05 (rbp) with mod = 0 means no base register.
198 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
199 return code == base_code;
200 } else {
201 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
202 // no base register.
203 if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
204 base_code |= ((data_.rex & 0x01) << 3);
205 return code == base_code;
206 }
207 }
208
AllocateAndInstallRequestedHeapObjects(Isolate * isolate)209 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
210 DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
211 for (auto& request : heap_object_requests_) {
212 Address pc = reinterpret_cast<Address>(buffer_start_) + request.offset();
213 switch (request.kind()) {
214 case HeapObjectRequest::kHeapNumber: {
215 Handle<HeapNumber> object =
216 isolate->factory()->NewHeapNumber<AllocationType::kOld>(
217 request.heap_number());
218 WriteUnalignedValue(pc, object);
219 break;
220 }
221 case HeapObjectRequest::kStringConstant: {
222 const StringConstantBase* str = request.string();
223 CHECK_NOT_NULL(str);
224 Handle<String> allocated = str->AllocateStringConstant(isolate);
225 WriteUnalignedValue(pc, allocated);
226 break;
227 }
228 }
229 }
230 }
231
232 // Partial Constant Pool.
AddSharedEntry(uint64_t data,int offset)233 bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
234 auto existing = entries_.find(data);
235 if (existing == entries_.end()) {
236 entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
237 return false;
238 }
239
240 // Make sure this is called with strictly ascending offsets.
241 DCHECK_GT(offset + kMoveImm64Offset, existing->second);
242
243 entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
244 return true;
245 }
246
TryRecordEntry(intptr_t data,RelocInfo::Mode mode)247 bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
248 if (!FLAG_partial_constant_pool) return false;
249 DCHECK_WITH_MSG(
250 FLAG_text_is_readable,
251 "The partial constant pool requires a readable .text section");
252 if (!RelocInfo::IsShareableRelocMode(mode)) return false;
253
254 // Currently, partial constant pool only handles the following kinds of
255 // RelocInfo.
256 if (mode != RelocInfo::NONE && mode != RelocInfo::EXTERNAL_REFERENCE &&
257 mode != RelocInfo::OFF_HEAP_TARGET)
258 return false;
259
260 uint64_t raw_data = static_cast<uint64_t>(data);
261 int offset = assm_->pc_offset();
262 return AddSharedEntry(raw_data, offset);
263 }
264
IsMoveRipRelative(Address instr)265 bool ConstPool::IsMoveRipRelative(Address instr) {
266 return (ReadUnalignedValue<uint32_t>(instr) & kMoveRipRelativeMask) ==
267 kMoveRipRelativeInstr;
268 }
269
Clear()270 void ConstPool::Clear() { entries_.clear(); }
271
PatchEntries()272 void ConstPool::PatchEntries() {
273 for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
274 iter = entries_.upper_bound(iter->first)) {
275 std::pair<EntryMap::iterator, EntryMap::iterator> range =
276 entries_.equal_range(iter->first);
277 int constant_entry_offset = 0;
278 for (EntryMap::iterator it = range.first; it != range.second; it++) {
279 if (it == range.first) {
280 constant_entry_offset = it->second;
281 continue;
282 }
283
284 DCHECK_GT(constant_entry_offset, 0);
285 DCHECK_LT(constant_entry_offset, it->second);
286 int32_t disp32 =
287 constant_entry_offset - (it->second + kRipRelativeDispSize);
288 Address disp_addr = assm_->addr_at(it->second);
289
290 // Check if the instruction is actually a rip-relative move.
291 DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
292 // The displacement of the rip-relative move should be 0 before patching.
293 DCHECK(ReadUnalignedValue<uint32_t>(disp_addr) == 0);
294 WriteUnalignedValue(disp_addr, disp32);
295 }
296 }
297 Clear();
298 }
299
PatchConstPool()300 void Assembler::PatchConstPool() {
301 // There is nothing to do if there are no pending entries.
302 if (constpool_.IsEmpty()) {
303 return;
304 }
305 constpool_.PatchEntries();
306 }
307
UseConstPoolFor(RelocInfo::Mode rmode)308 bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
309 if (!FLAG_partial_constant_pool) return false;
310 return (rmode == RelocInfo::NONE || rmode == RelocInfo::EXTERNAL_REFERENCE ||
311 rmode == RelocInfo::OFF_HEAP_TARGET);
312 }
313
314 // -----------------------------------------------------------------------------
315 // Implementation of Assembler.
316
Assembler(const AssemblerOptions & options,std::unique_ptr<AssemblerBuffer> buffer)317 Assembler::Assembler(const AssemblerOptions& options,
318 std::unique_ptr<AssemblerBuffer> buffer)
319 : AssemblerBase(options, std::move(buffer)), constpool_(this) {
320 reloc_info_writer.Reposition(buffer_start_ + buffer_->size(), pc_);
321 if (CpuFeatures::IsSupported(SSE4_2)) {
322 EnableCpuFeature(SSE4_1);
323 }
324 if (CpuFeatures::IsSupported(SSE4_1)) {
325 EnableCpuFeature(SSSE3);
326 }
327
328 #if defined(V8_OS_WIN_X64)
329 if (options.collect_win64_unwind_info) {
330 xdata_encoder_ = std::make_unique<win64_unwindinfo::XdataEncoder>(*this);
331 }
332 #endif
333 }
334
GetCode(Isolate * isolate,CodeDesc * desc,SafepointTableBuilder * safepoint_table_builder,int handler_table_offset)335 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc,
336 SafepointTableBuilder* safepoint_table_builder,
337 int handler_table_offset) {
338 // As a crutch to avoid having to add manual Align calls wherever we use a
339 // raw workflow to create Code objects (mostly in tests), add another Align
340 // call here. It does no harm - the end of the Code object is aligned to the
341 // (larger) kCodeAlignment anyways.
342 // TODO(jgruber): Consider moving responsibility for proper alignment to
343 // metadata table builders (safepoint, handler, constant pool, code
344 // comments).
345 DataAlign(Code::kMetadataAlignment);
346
347 PatchConstPool();
348 DCHECK(constpool_.IsEmpty());
349
350 const int code_comments_size = WriteCodeComments();
351
352 // At this point overflow() may be true, but the gap ensures
353 // that we are still not overlapping instructions and relocation info.
354 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
355
356 AllocateAndInstallRequestedHeapObjects(isolate);
357
358 // Set up code descriptor.
359 // TODO(jgruber): Reconsider how these offsets and sizes are maintained up to
360 // this point to make CodeDesc initialization less fiddly.
361
362 static constexpr int kConstantPoolSize = 0;
363 const int instruction_size = pc_offset();
364 const int code_comments_offset = instruction_size - code_comments_size;
365 const int constant_pool_offset = code_comments_offset - kConstantPoolSize;
366 const int handler_table_offset2 = (handler_table_offset == kNoHandlerTable)
367 ? constant_pool_offset
368 : handler_table_offset;
369 const int safepoint_table_offset =
370 (safepoint_table_builder == kNoSafepointTable)
371 ? handler_table_offset2
372 : safepoint_table_builder->GetCodeOffset();
373 const int reloc_info_offset =
374 static_cast<int>(reloc_info_writer.pos() - buffer_->start());
375 CodeDesc::Initialize(desc, this, safepoint_table_offset,
376 handler_table_offset2, constant_pool_offset,
377 code_comments_offset, reloc_info_offset);
378 }
379
FinalizeJumpOptimizationInfo()380 void Assembler::FinalizeJumpOptimizationInfo() {
381 // Collection stage
382 auto jump_opt = jump_optimization_info();
383 if (jump_opt && jump_opt->is_collecting()) {
384 auto& bitmap = jump_opt->farjmp_bitmap();
385 int num = static_cast<int>(farjmp_positions_.size());
386 if (num && bitmap.empty()) {
387 bool can_opt = false;
388
389 bitmap.resize((num + 31) / 32, 0);
390 for (int i = 0; i < num; i++) {
391 int disp_pos = farjmp_positions_[i];
392 int disp = long_at(disp_pos);
393 if (is_int8(disp)) {
394 bitmap[i / 32] |= 1 << (i & 31);
395 can_opt = true;
396 }
397 }
398 if (can_opt) {
399 jump_opt->set_optimizable();
400 }
401 }
402 }
403 }
404
405 #if defined(V8_OS_WIN_X64)
GetUnwindInfo() const406 win64_unwindinfo::BuiltinUnwindInfo Assembler::GetUnwindInfo() const {
407 DCHECK(options().collect_win64_unwind_info);
408 DCHECK_NOT_NULL(xdata_encoder_);
409 return xdata_encoder_->unwinding_info();
410 }
411 #endif
412
Align(int m)413 void Assembler::Align(int m) {
414 DCHECK(base::bits::IsPowerOfTwo(m));
415 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
416 Nop(delta);
417 }
418
CodeTargetAlign()419 void Assembler::CodeTargetAlign() {
420 Align(16); // Preferred alignment of jump targets on x64.
421 }
422
IsNop(Address addr)423 bool Assembler::IsNop(Address addr) {
424 byte* a = reinterpret_cast<byte*>(addr);
425 while (*a == 0x66) a++;
426 if (*a == 0x90) return true;
427 if (a[0] == 0xF && a[1] == 0x1F) return true;
428 return false;
429 }
430
bind_to(Label * L,int pos)431 void Assembler::bind_to(Label* L, int pos) {
432 DCHECK(!L->is_bound()); // Label may only be bound once.
433 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
434 if (L->is_linked()) {
435 int current = L->pos();
436 int next = long_at(current);
437 while (next != current) {
438 if (current >= 4 && long_at(current - 4) == 0) {
439 // Absolute address.
440 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
441 WriteUnalignedValue(addr_at(current - 4), imm64);
442 internal_reference_positions_.push_back(current - 4);
443 } else {
444 // Relative address, relative to point after address.
445 int imm32 = pos - (current + sizeof(int32_t));
446 long_at_put(current, imm32);
447 }
448 current = next;
449 next = long_at(next);
450 }
451 // Fix up last fixup on linked list.
452 if (current >= 4 && long_at(current - 4) == 0) {
453 // Absolute address.
454 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_start_ + pos);
455 WriteUnalignedValue(addr_at(current - 4), imm64);
456 internal_reference_positions_.push_back(current - 4);
457 } else {
458 // Relative address, relative to point after address.
459 int imm32 = pos - (current + sizeof(int32_t));
460 long_at_put(current, imm32);
461 }
462 }
463 while (L->is_near_linked()) {
464 int fixup_pos = L->near_link_pos();
465 int offset_to_next =
466 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
467 DCHECK_LE(offset_to_next, 0);
468 int disp = pos - (fixup_pos + sizeof(int8_t));
469 CHECK(is_int8(disp));
470 set_byte_at(fixup_pos, disp);
471 if (offset_to_next < 0) {
472 L->link_to(fixup_pos + offset_to_next, Label::kNear);
473 } else {
474 L->UnuseNear();
475 }
476 }
477
478 // Optimization stage
479 auto jump_opt = jump_optimization_info();
480 if (jump_opt && jump_opt->is_optimizing()) {
481 auto it = label_farjmp_maps_.find(L);
482 if (it != label_farjmp_maps_.end()) {
483 auto& pos_vector = it->second;
484 for (auto fixup_pos : pos_vector) {
485 int disp = pos - (fixup_pos + sizeof(int8_t));
486 CHECK(is_int8(disp));
487 set_byte_at(fixup_pos, disp);
488 }
489 label_farjmp_maps_.erase(it);
490 }
491 }
492 L->bind_to(pos);
493 }
494
bind(Label * L)495 void Assembler::bind(Label* L) { bind_to(L, pc_offset()); }
496
record_farjmp_position(Label * L,int pos)497 void Assembler::record_farjmp_position(Label* L, int pos) {
498 auto& pos_vector = label_farjmp_maps_[L];
499 pos_vector.push_back(pos);
500 }
501
is_optimizable_farjmp(int idx)502 bool Assembler::is_optimizable_farjmp(int idx) {
503 if (predictable_code_size()) return false;
504
505 auto jump_opt = jump_optimization_info();
506 CHECK(jump_opt->is_optimizing());
507
508 auto& bitmap = jump_opt->farjmp_bitmap();
509 CHECK(idx < static_cast<int>(bitmap.size() * 32));
510 return !!(bitmap[idx / 32] & (1 << (idx & 31)));
511 }
512
GrowBuffer()513 void Assembler::GrowBuffer() {
514 DCHECK(buffer_overflow());
515
516 // Compute new buffer size.
517 DCHECK_EQ(buffer_start_, buffer_->start());
518 int old_size = buffer_->size();
519 int new_size = 2 * old_size;
520
521 // Some internal data structures overflow for very large buffers,
522 // they must ensure that kMaximalBufferSize is not too large.
523 if (new_size > kMaximalBufferSize) {
524 V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
525 }
526
527 // Set up new buffer.
528 std::unique_ptr<AssemblerBuffer> new_buffer = buffer_->Grow(new_size);
529 DCHECK_EQ(new_size, new_buffer->size());
530 byte* new_start = new_buffer->start();
531
532 // Copy the data.
533 intptr_t pc_delta = new_start - buffer_start_;
534 intptr_t rc_delta = (new_start + new_size) - (buffer_start_ + old_size);
535 size_t reloc_size = (buffer_start_ + old_size) - reloc_info_writer.pos();
536 MemMove(new_start, buffer_start_, pc_offset());
537 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
538 reloc_size);
539
540 // Switch buffers.
541 buffer_ = std::move(new_buffer);
542 buffer_start_ = new_start;
543 pc_ += pc_delta;
544 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
545 reloc_info_writer.last_pc() + pc_delta);
546
547 // Relocate internal references.
548 for (auto pos : internal_reference_positions_) {
549 Address p = reinterpret_cast<Address>(buffer_start_ + pos);
550 WriteUnalignedValue(p, ReadUnalignedValue<intptr_t>(p) + pc_delta);
551 }
552
553 DCHECK(!buffer_overflow());
554 }
555
emit_operand(int code,Operand adr)556 void Assembler::emit_operand(int code, Operand adr) {
557 DCHECK(is_uint3(code));
558 const unsigned length = adr.data().len;
559 DCHECK_GT(length, 0);
560
561 // Emit updated ModR/M byte containing the given register.
562 DCHECK_EQ(adr.data().buf[0] & 0x38, 0);
563 *pc_++ = adr.data().buf[0] | code << 3;
564
565 // Recognize RIP relative addressing.
566 if (adr.data().buf[0] == 5) {
567 DCHECK_EQ(9u, length);
568 Label* label = ReadUnalignedValue<Label*>(
569 reinterpret_cast<Address>(&adr.data().buf[1]));
570 if (label->is_bound()) {
571 int offset =
572 label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
573 DCHECK_GE(0, offset);
574 emitl(offset);
575 } else if (label->is_linked()) {
576 emitl(label->pos());
577 label->link_to(pc_offset() - sizeof(int32_t));
578 } else {
579 DCHECK(label->is_unused());
580 int32_t current = pc_offset();
581 emitl(current);
582 label->link_to(current);
583 }
584 } else {
585 // Emit the rest of the encoded operand.
586 for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
587 }
588 }
589
590 // Assembler Instruction implementations.
591
arithmetic_op(byte opcode,Register reg,Operand op,int size)592 void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
593 EnsureSpace ensure_space(this);
594 emit_rex(reg, op, size);
595 emit(opcode);
596 emit_operand(reg, op);
597 }
598
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)599 void Assembler::arithmetic_op(byte opcode, Register reg, Register rm_reg,
600 int size) {
601 EnsureSpace ensure_space(this);
602 DCHECK_EQ(opcode & 0xC6, 2);
603 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
604 // Swap reg and rm_reg and change opcode operand order.
605 emit_rex(rm_reg, reg, size);
606 emit(opcode ^ 0x02);
607 emit_modrm(rm_reg, reg);
608 } else {
609 emit_rex(reg, rm_reg, size);
610 emit(opcode);
611 emit_modrm(reg, rm_reg);
612 }
613 }
614
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)615 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
616 EnsureSpace ensure_space(this);
617 DCHECK_EQ(opcode & 0xC6, 2);
618 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
619 // Swap reg and rm_reg and change opcode operand order.
620 emit(0x66);
621 emit_optional_rex_32(rm_reg, reg);
622 emit(opcode ^ 0x02);
623 emit_modrm(rm_reg, reg);
624 } else {
625 emit(0x66);
626 emit_optional_rex_32(reg, rm_reg);
627 emit(opcode);
628 emit_modrm(reg, rm_reg);
629 }
630 }
631
arithmetic_op_16(byte opcode,Register reg,Operand rm_reg)632 void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
633 EnsureSpace ensure_space(this);
634 emit(0x66);
635 emit_optional_rex_32(reg, rm_reg);
636 emit(opcode);
637 emit_operand(reg, rm_reg);
638 }
639
arithmetic_op_8(byte opcode,Register reg,Operand op)640 void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
641 EnsureSpace ensure_space(this);
642 if (!reg.is_byte_register()) {
643 emit_rex_32(reg, op);
644 } else {
645 emit_optional_rex_32(reg, op);
646 }
647 emit(opcode);
648 emit_operand(reg, op);
649 }
650
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)651 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
652 EnsureSpace ensure_space(this);
653 DCHECK_EQ(opcode & 0xC6, 2);
654 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
655 // Swap reg and rm_reg and change opcode operand order.
656 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
657 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
658 emit_rex_32(rm_reg, reg);
659 }
660 emit(opcode ^ 0x02);
661 emit_modrm(rm_reg, reg);
662 } else {
663 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
664 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
665 emit_rex_32(reg, rm_reg);
666 }
667 emit(opcode);
668 emit_modrm(reg, rm_reg);
669 }
670 }
671
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)672 void Assembler::immediate_arithmetic_op(byte subcode, Register dst,
673 Immediate src, int size) {
674 EnsureSpace ensure_space(this);
675 emit_rex(dst, size);
676 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
677 emit(0x83);
678 emit_modrm(subcode, dst);
679 emit(src.value_);
680 } else if (dst == rax) {
681 emit(0x05 | (subcode << 3));
682 emit(src);
683 } else {
684 emit(0x81);
685 emit_modrm(subcode, dst);
686 emit(src);
687 }
688 }
689
immediate_arithmetic_op(byte subcode,Operand dst,Immediate src,int size)690 void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
691 Immediate src, int size) {
692 EnsureSpace ensure_space(this);
693 emit_rex(dst, size);
694 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
695 emit(0x83);
696 emit_operand(subcode, dst);
697 emit(src.value_);
698 } else {
699 emit(0x81);
700 emit_operand(subcode, dst);
701 emit(src);
702 }
703 }
704
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)705 void Assembler::immediate_arithmetic_op_16(byte subcode, Register dst,
706 Immediate src) {
707 EnsureSpace ensure_space(this);
708 emit(0x66); // Operand size override prefix.
709 emit_optional_rex_32(dst);
710 if (is_int8(src.value_)) {
711 emit(0x83);
712 emit_modrm(subcode, dst);
713 emit(src.value_);
714 } else if (dst == rax) {
715 emit(0x05 | (subcode << 3));
716 emitw(src.value_);
717 } else {
718 emit(0x81);
719 emit_modrm(subcode, dst);
720 emitw(src.value_);
721 }
722 }
723
immediate_arithmetic_op_16(byte subcode,Operand dst,Immediate src)724 void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
725 Immediate src) {
726 EnsureSpace ensure_space(this);
727 emit(0x66); // Operand size override prefix.
728 emit_optional_rex_32(dst);
729 if (is_int8(src.value_)) {
730 emit(0x83);
731 emit_operand(subcode, dst);
732 emit(src.value_);
733 } else {
734 emit(0x81);
735 emit_operand(subcode, dst);
736 emitw(src.value_);
737 }
738 }
739
immediate_arithmetic_op_8(byte subcode,Operand dst,Immediate src)740 void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
741 Immediate src) {
742 EnsureSpace ensure_space(this);
743 emit_optional_rex_32(dst);
744 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
745 emit(0x80);
746 emit_operand(subcode, dst);
747 emit(src.value_);
748 }
749
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)750 void Assembler::immediate_arithmetic_op_8(byte subcode, Register dst,
751 Immediate src) {
752 EnsureSpace ensure_space(this);
753 if (!dst.is_byte_register()) {
754 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
755 emit_rex_32(dst);
756 }
757 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
758 emit(0x80);
759 emit_modrm(subcode, dst);
760 emit(src.value_);
761 }
762
shift(Register dst,Immediate shift_amount,int subcode,int size)763 void Assembler::shift(Register dst, Immediate shift_amount, int subcode,
764 int size) {
765 EnsureSpace ensure_space(this);
766 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
767 : is_uint5(shift_amount.value_));
768 if (shift_amount.value_ == 1) {
769 emit_rex(dst, size);
770 emit(0xD1);
771 emit_modrm(subcode, dst);
772 } else {
773 emit_rex(dst, size);
774 emit(0xC1);
775 emit_modrm(subcode, dst);
776 emit(shift_amount.value_);
777 }
778 }
779
shift(Operand dst,Immediate shift_amount,int subcode,int size)780 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
781 int size) {
782 EnsureSpace ensure_space(this);
783 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
784 : is_uint5(shift_amount.value_));
785 if (shift_amount.value_ == 1) {
786 emit_rex(dst, size);
787 emit(0xD1);
788 emit_operand(subcode, dst);
789 } else {
790 emit_rex(dst, size);
791 emit(0xC1);
792 emit_operand(subcode, dst);
793 emit(shift_amount.value_);
794 }
795 }
796
shift(Register dst,int subcode,int size)797 void Assembler::shift(Register dst, int subcode, int size) {
798 EnsureSpace ensure_space(this);
799 emit_rex(dst, size);
800 emit(0xD3);
801 emit_modrm(subcode, dst);
802 }
803
shift(Operand dst,int subcode,int size)804 void Assembler::shift(Operand dst, int subcode, int size) {
805 EnsureSpace ensure_space(this);
806 emit_rex(dst, size);
807 emit(0xD3);
808 emit_operand(subcode, dst);
809 }
810
bswapl(Register dst)811 void Assembler::bswapl(Register dst) {
812 EnsureSpace ensure_space(this);
813 emit_rex_32(dst);
814 emit(0x0F);
815 emit(0xC8 + dst.low_bits());
816 }
817
bswapq(Register dst)818 void Assembler::bswapq(Register dst) {
819 EnsureSpace ensure_space(this);
820 emit_rex_64(dst);
821 emit(0x0F);
822 emit(0xC8 + dst.low_bits());
823 }
824
btq(Operand dst,Register src)825 void Assembler::btq(Operand dst, Register src) {
826 EnsureSpace ensure_space(this);
827 emit_rex_64(src, dst);
828 emit(0x0F);
829 emit(0xA3);
830 emit_operand(src, dst);
831 }
832
btsq(Operand dst,Register src)833 void Assembler::btsq(Operand dst, Register src) {
834 EnsureSpace ensure_space(this);
835 emit_rex_64(src, dst);
836 emit(0x0F);
837 emit(0xAB);
838 emit_operand(src, dst);
839 }
840
btsq(Register dst,Immediate imm8)841 void Assembler::btsq(Register dst, Immediate imm8) {
842 EnsureSpace ensure_space(this);
843 emit_rex_64(dst);
844 emit(0x0F);
845 emit(0xBA);
846 emit_modrm(0x5, dst);
847 emit(imm8.value_);
848 }
849
btrq(Register dst,Immediate imm8)850 void Assembler::btrq(Register dst, Immediate imm8) {
851 EnsureSpace ensure_space(this);
852 emit_rex_64(dst);
853 emit(0x0F);
854 emit(0xBA);
855 emit_modrm(0x6, dst);
856 emit(imm8.value_);
857 }
858
bsrl(Register dst,Register src)859 void Assembler::bsrl(Register dst, Register src) {
860 EnsureSpace ensure_space(this);
861 emit_optional_rex_32(dst, src);
862 emit(0x0F);
863 emit(0xBD);
864 emit_modrm(dst, src);
865 }
866
bsrl(Register dst,Operand src)867 void Assembler::bsrl(Register dst, Operand src) {
868 EnsureSpace ensure_space(this);
869 emit_optional_rex_32(dst, src);
870 emit(0x0F);
871 emit(0xBD);
872 emit_operand(dst, src);
873 }
874
bsrq(Register dst,Register src)875 void Assembler::bsrq(Register dst, Register src) {
876 EnsureSpace ensure_space(this);
877 emit_rex_64(dst, src);
878 emit(0x0F);
879 emit(0xBD);
880 emit_modrm(dst, src);
881 }
882
bsrq(Register dst,Operand src)883 void Assembler::bsrq(Register dst, Operand src) {
884 EnsureSpace ensure_space(this);
885 emit_rex_64(dst, src);
886 emit(0x0F);
887 emit(0xBD);
888 emit_operand(dst, src);
889 }
890
bsfl(Register dst,Register src)891 void Assembler::bsfl(Register dst, Register src) {
892 EnsureSpace ensure_space(this);
893 emit_optional_rex_32(dst, src);
894 emit(0x0F);
895 emit(0xBC);
896 emit_modrm(dst, src);
897 }
898
bsfl(Register dst,Operand src)899 void Assembler::bsfl(Register dst, Operand src) {
900 EnsureSpace ensure_space(this);
901 emit_optional_rex_32(dst, src);
902 emit(0x0F);
903 emit(0xBC);
904 emit_operand(dst, src);
905 }
906
bsfq(Register dst,Register src)907 void Assembler::bsfq(Register dst, Register src) {
908 EnsureSpace ensure_space(this);
909 emit_rex_64(dst, src);
910 emit(0x0F);
911 emit(0xBC);
912 emit_modrm(dst, src);
913 }
914
bsfq(Register dst,Operand src)915 void Assembler::bsfq(Register dst, Operand src) {
916 EnsureSpace ensure_space(this);
917 emit_rex_64(dst, src);
918 emit(0x0F);
919 emit(0xBC);
920 emit_operand(dst, src);
921 }
922
pblendw(XMMRegister dst,Operand src,uint8_t mask)923 void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
924 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
925 emit(mask);
926 }
927
pblendw(XMMRegister dst,XMMRegister src,uint8_t mask)928 void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
929 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
930 emit(mask);
931 }
932
palignr(XMMRegister dst,Operand src,uint8_t mask)933 void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
934 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
935 emit(mask);
936 }
937
palignr(XMMRegister dst,XMMRegister src,uint8_t mask)938 void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
939 ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
940 emit(mask);
941 }
942
call(Label * L)943 void Assembler::call(Label* L) {
944 EnsureSpace ensure_space(this);
945 // 1110 1000 #32-bit disp.
946 emit(0xE8);
947 if (L->is_bound()) {
948 int offset = L->pos() - pc_offset() - sizeof(int32_t);
949 DCHECK_LE(offset, 0);
950 emitl(offset);
951 } else if (L->is_linked()) {
952 emitl(L->pos());
953 L->link_to(pc_offset() - sizeof(int32_t));
954 } else {
955 DCHECK(L->is_unused());
956 int32_t current = pc_offset();
957 emitl(current);
958 L->link_to(current);
959 }
960 }
961
call(Address entry,RelocInfo::Mode rmode)962 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
963 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
964 EnsureSpace ensure_space(this);
965 // 1110 1000 #32-bit disp.
966 emit(0xE8);
967 emit_runtime_entry(entry, rmode);
968 }
969
call(Handle<Code> target,RelocInfo::Mode rmode)970 void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
971 DCHECK(RelocInfo::IsCodeTarget(rmode));
972 DCHECK(target->IsExecutable());
973 EnsureSpace ensure_space(this);
974 // 1110 1000 #32-bit disp.
975 emit(0xE8);
976 RecordRelocInfo(rmode);
977 int code_target_index = AddCodeTarget(target);
978 emitl(code_target_index);
979 }
980
near_call(intptr_t disp,RelocInfo::Mode rmode)981 void Assembler::near_call(intptr_t disp, RelocInfo::Mode rmode) {
982 EnsureSpace ensure_space(this);
983 emit(0xE8);
984 DCHECK(is_int32(disp));
985 RecordRelocInfo(rmode);
986 emitl(static_cast<int32_t>(disp));
987 }
988
near_jmp(intptr_t disp,RelocInfo::Mode rmode)989 void Assembler::near_jmp(intptr_t disp, RelocInfo::Mode rmode) {
990 EnsureSpace ensure_space(this);
991 emit(0xE9);
992 DCHECK(is_int32(disp));
993 if (!RelocInfo::IsNone(rmode)) RecordRelocInfo(rmode);
994 emitl(static_cast<int32_t>(disp));
995 }
996
call(Register adr)997 void Assembler::call(Register adr) {
998 EnsureSpace ensure_space(this);
999 // Opcode: FF /2 r64.
1000 emit_optional_rex_32(adr);
1001 emit(0xFF);
1002 emit_modrm(0x2, adr);
1003 }
1004
call(Operand op)1005 void Assembler::call(Operand op) {
1006 EnsureSpace ensure_space(this);
1007 // Opcode: FF /2 m64.
1008 emit_optional_rex_32(op);
1009 emit(0xFF);
1010 emit_operand(0x2, op);
1011 }
1012
clc()1013 void Assembler::clc() {
1014 EnsureSpace ensure_space(this);
1015 emit(0xF8);
1016 }
1017
cld()1018 void Assembler::cld() {
1019 EnsureSpace ensure_space(this);
1020 emit(0xFC);
1021 }
1022
cdq()1023 void Assembler::cdq() {
1024 EnsureSpace ensure_space(this);
1025 emit(0x99);
1026 }
1027
cmovq(Condition cc,Register dst,Register src)1028 void Assembler::cmovq(Condition cc, Register dst, Register src) {
1029 if (cc == always) {
1030 movq(dst, src);
1031 } else if (cc == never) {
1032 return;
1033 }
1034 // No need to check CpuInfo for CMOV support, it's a required part of the
1035 // 64-bit architecture.
1036 DCHECK_GE(cc, 0); // Use mov for unconditional moves.
1037 EnsureSpace ensure_space(this);
1038 // Opcode: REX.W 0f 40 + cc /r.
1039 emit_rex_64(dst, src);
1040 emit(0x0F);
1041 emit(0x40 + cc);
1042 emit_modrm(dst, src);
1043 }
1044
cmovq(Condition cc,Register dst,Operand src)1045 void Assembler::cmovq(Condition cc, Register dst, Operand src) {
1046 if (cc == always) {
1047 movq(dst, src);
1048 } else if (cc == never) {
1049 return;
1050 }
1051 DCHECK_GE(cc, 0);
1052 EnsureSpace ensure_space(this);
1053 // Opcode: REX.W 0f 40 + cc /r.
1054 emit_rex_64(dst, src);
1055 emit(0x0F);
1056 emit(0x40 + cc);
1057 emit_operand(dst, src);
1058 }
1059
cmovl(Condition cc,Register dst,Register src)1060 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1061 if (cc == always) {
1062 movl(dst, src);
1063 } else if (cc == never) {
1064 return;
1065 }
1066 DCHECK_GE(cc, 0);
1067 EnsureSpace ensure_space(this);
1068 // Opcode: 0f 40 + cc /r.
1069 emit_optional_rex_32(dst, src);
1070 emit(0x0F);
1071 emit(0x40 + cc);
1072 emit_modrm(dst, src);
1073 }
1074
cmovl(Condition cc,Register dst,Operand src)1075 void Assembler::cmovl(Condition cc, Register dst, Operand src) {
1076 if (cc == always) {
1077 movl(dst, src);
1078 } else if (cc == never) {
1079 return;
1080 }
1081 DCHECK_GE(cc, 0);
1082 EnsureSpace ensure_space(this);
1083 // Opcode: 0f 40 + cc /r.
1084 emit_optional_rex_32(dst, src);
1085 emit(0x0F);
1086 emit(0x40 + cc);
1087 emit_operand(dst, src);
1088 }
1089
cmpb_al(Immediate imm8)1090 void Assembler::cmpb_al(Immediate imm8) {
1091 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
1092 EnsureSpace ensure_space(this);
1093 emit(0x3C);
1094 emit(imm8.value_);
1095 }
1096
lock()1097 void Assembler::lock() {
1098 EnsureSpace ensure_space(this);
1099 emit(0xF0);
1100 }
1101
xaddb(Operand dst,Register src)1102 void Assembler::xaddb(Operand dst, Register src) {
1103 EnsureSpace ensure_space(this);
1104 emit_optional_rex_8(src, dst);
1105 emit(0x0F);
1106 emit(0xC0);
1107 emit_operand(src, dst);
1108 }
1109
xaddw(Operand dst,Register src)1110 void Assembler::xaddw(Operand dst, Register src) {
1111 EnsureSpace ensure_space(this);
1112 emit(0x66);
1113 emit_optional_rex_32(src, dst);
1114 emit(0x0F);
1115 emit(0xC1);
1116 emit_operand(src, dst);
1117 }
1118
xaddl(Operand dst,Register src)1119 void Assembler::xaddl(Operand dst, Register src) {
1120 EnsureSpace ensure_space(this);
1121 emit_optional_rex_32(src, dst);
1122 emit(0x0F);
1123 emit(0xC1);
1124 emit_operand(src, dst);
1125 }
1126
xaddq(Operand dst,Register src)1127 void Assembler::xaddq(Operand dst, Register src) {
1128 EnsureSpace ensure_space(this);
1129 emit_rex(src, dst, kInt64Size);
1130 emit(0x0F);
1131 emit(0xC1);
1132 emit_operand(src, dst);
1133 }
1134
cmpxchgb(Operand dst,Register src)1135 void Assembler::cmpxchgb(Operand dst, Register src) {
1136 EnsureSpace ensure_space(this);
1137 if (!src.is_byte_register()) {
1138 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1139 emit_rex_32(src, dst);
1140 } else {
1141 emit_optional_rex_32(src, dst);
1142 }
1143 emit(0x0F);
1144 emit(0xB0);
1145 emit_operand(src, dst);
1146 }
1147
cmpxchgw(Operand dst,Register src)1148 void Assembler::cmpxchgw(Operand dst, Register src) {
1149 EnsureSpace ensure_space(this);
1150 emit(0x66);
1151 emit_optional_rex_32(src, dst);
1152 emit(0x0F);
1153 emit(0xB1);
1154 emit_operand(src, dst);
1155 }
1156
emit_cmpxchg(Operand dst,Register src,int size)1157 void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
1158 EnsureSpace ensure_space(this);
1159 emit_rex(src, dst, size);
1160 emit(0x0F);
1161 emit(0xB1);
1162 emit_operand(src, dst);
1163 }
1164
mfence()1165 void Assembler::mfence() {
1166 EnsureSpace ensure_space(this);
1167 emit(0x0F);
1168 emit(0xAE);
1169 emit(0xF0);
1170 }
1171
lfence()1172 void Assembler::lfence() {
1173 EnsureSpace ensure_space(this);
1174 emit(0x0F);
1175 emit(0xAE);
1176 emit(0xE8);
1177 }
1178
cpuid()1179 void Assembler::cpuid() {
1180 EnsureSpace ensure_space(this);
1181 emit(0x0F);
1182 emit(0xA2);
1183 }
1184
cqo()1185 void Assembler::cqo() {
1186 EnsureSpace ensure_space(this);
1187 emit_rex_64();
1188 emit(0x99);
1189 }
1190
emit_dec(Register dst,int size)1191 void Assembler::emit_dec(Register dst, int size) {
1192 EnsureSpace ensure_space(this);
1193 emit_rex(dst, size);
1194 emit(0xFF);
1195 emit_modrm(0x1, dst);
1196 }
1197
emit_dec(Operand dst,int size)1198 void Assembler::emit_dec(Operand dst, int size) {
1199 EnsureSpace ensure_space(this);
1200 emit_rex(dst, size);
1201 emit(0xFF);
1202 emit_operand(1, dst);
1203 }
1204
decb(Register dst)1205 void Assembler::decb(Register dst) {
1206 EnsureSpace ensure_space(this);
1207 if (!dst.is_byte_register()) {
1208 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1209 emit_rex_32(dst);
1210 }
1211 emit(0xFE);
1212 emit_modrm(0x1, dst);
1213 }
1214
decb(Operand dst)1215 void Assembler::decb(Operand dst) {
1216 EnsureSpace ensure_space(this);
1217 emit_optional_rex_32(dst);
1218 emit(0xFE);
1219 emit_operand(1, dst);
1220 }
1221
hlt()1222 void Assembler::hlt() {
1223 EnsureSpace ensure_space(this);
1224 emit(0xF4);
1225 }
1226
emit_idiv(Register src,int size)1227 void Assembler::emit_idiv(Register src, int size) {
1228 EnsureSpace ensure_space(this);
1229 emit_rex(src, size);
1230 emit(0xF7);
1231 emit_modrm(0x7, src);
1232 }
1233
emit_div(Register src,int size)1234 void Assembler::emit_div(Register src, int size) {
1235 EnsureSpace ensure_space(this);
1236 emit_rex(src, size);
1237 emit(0xF7);
1238 emit_modrm(0x6, src);
1239 }
1240
emit_imul(Register src,int size)1241 void Assembler::emit_imul(Register src, int size) {
1242 EnsureSpace ensure_space(this);
1243 emit_rex(src, size);
1244 emit(0xF7);
1245 emit_modrm(0x5, src);
1246 }
1247
emit_imul(Operand src,int size)1248 void Assembler::emit_imul(Operand src, int size) {
1249 EnsureSpace ensure_space(this);
1250 emit_rex(src, size);
1251 emit(0xF7);
1252 emit_operand(0x5, src);
1253 }
1254
emit_imul(Register dst,Register src,int size)1255 void Assembler::emit_imul(Register dst, Register src, int size) {
1256 EnsureSpace ensure_space(this);
1257 emit_rex(dst, src, size);
1258 emit(0x0F);
1259 emit(0xAF);
1260 emit_modrm(dst, src);
1261 }
1262
emit_imul(Register dst,Operand src,int size)1263 void Assembler::emit_imul(Register dst, Operand src, int size) {
1264 EnsureSpace ensure_space(this);
1265 emit_rex(dst, src, size);
1266 emit(0x0F);
1267 emit(0xAF);
1268 emit_operand(dst, src);
1269 }
1270
emit_imul(Register dst,Register src,Immediate imm,int size)1271 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1272 EnsureSpace ensure_space(this);
1273 emit_rex(dst, src, size);
1274 if (is_int8(imm.value_)) {
1275 emit(0x6B);
1276 emit_modrm(dst, src);
1277 emit(imm.value_);
1278 } else {
1279 emit(0x69);
1280 emit_modrm(dst, src);
1281 emitl(imm.value_);
1282 }
1283 }
1284
emit_imul(Register dst,Operand src,Immediate imm,int size)1285 void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
1286 EnsureSpace ensure_space(this);
1287 emit_rex(dst, src, size);
1288 if (is_int8(imm.value_)) {
1289 emit(0x6B);
1290 emit_operand(dst, src);
1291 emit(imm.value_);
1292 } else {
1293 emit(0x69);
1294 emit_operand(dst, src);
1295 emitl(imm.value_);
1296 }
1297 }
1298
emit_inc(Register dst,int size)1299 void Assembler::emit_inc(Register dst, int size) {
1300 EnsureSpace ensure_space(this);
1301 emit_rex(dst, size);
1302 emit(0xFF);
1303 emit_modrm(0x0, dst);
1304 }
1305
emit_inc(Operand dst,int size)1306 void Assembler::emit_inc(Operand dst, int size) {
1307 EnsureSpace ensure_space(this);
1308 emit_rex(dst, size);
1309 emit(0xFF);
1310 emit_operand(0, dst);
1311 }
1312
int3()1313 void Assembler::int3() {
1314 EnsureSpace ensure_space(this);
1315 emit(0xCC);
1316 }
1317
j(Condition cc,Label * L,Label::Distance distance)1318 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1319 if (cc == always) {
1320 jmp(L, distance);
1321 return;
1322 } else if (cc == never) {
1323 return;
1324 }
1325 EnsureSpace ensure_space(this);
1326 DCHECK(is_uint4(cc));
1327 if (L->is_bound()) {
1328 const int short_size = 2;
1329 const int long_size = 6;
1330 int offs = L->pos() - pc_offset();
1331 DCHECK_LE(offs, 0);
1332 // Determine whether we can use 1-byte offsets for backwards branches,
1333 // which have a max range of 128 bytes.
1334
1335 // We also need to check predictable_code_size() flag here, because on x64,
1336 // when the full code generator recompiles code for debugging, some places
1337 // need to be padded out to a certain size. The debugger is keeping track of
1338 // how often it did this so that it can adjust return addresses on the
1339 // stack, but if the size of jump instructions can also change, that's not
1340 // enough and the calculated offsets would be incorrect.
1341 if (is_int8(offs - short_size) && !predictable_code_size()) {
1342 // 0111 tttn #8-bit disp.
1343 emit(0x70 | cc);
1344 emit((offs - short_size) & 0xFF);
1345 } else {
1346 // 0000 1111 1000 tttn #32-bit disp.
1347 emit(0x0F);
1348 emit(0x80 | cc);
1349 emitl(offs - long_size);
1350 }
1351 } else if (distance == Label::kNear) {
1352 // 0111 tttn #8-bit disp
1353 emit(0x70 | cc);
1354 byte disp = 0x00;
1355 if (L->is_near_linked()) {
1356 int offset = L->near_link_pos() - pc_offset();
1357 DCHECK(is_int8(offset));
1358 disp = static_cast<byte>(offset & 0xFF);
1359 }
1360 L->link_to(pc_offset(), Label::kNear);
1361 emit(disp);
1362 } else {
1363 auto jump_opt = jump_optimization_info();
1364 if (V8_UNLIKELY(jump_opt)) {
1365 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1366 // 0111 tttn #8-bit disp
1367 emit(0x70 | cc);
1368 record_farjmp_position(L, pc_offset());
1369 emit(0);
1370 return;
1371 }
1372 if (jump_opt->is_collecting()) {
1373 farjmp_positions_.push_back(pc_offset() + 2);
1374 }
1375 }
1376 if (L->is_linked()) {
1377 // 0000 1111 1000 tttn #32-bit disp.
1378 emit(0x0F);
1379 emit(0x80 | cc);
1380 emitl(L->pos());
1381 L->link_to(pc_offset() - sizeof(int32_t));
1382 } else {
1383 DCHECK(L->is_unused());
1384 emit(0x0F);
1385 emit(0x80 | cc);
1386 int32_t current = pc_offset();
1387 emitl(current);
1388 L->link_to(current);
1389 }
1390 }
1391 }
1392
j(Condition cc,Address entry,RelocInfo::Mode rmode)1393 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1394 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1395 EnsureSpace ensure_space(this);
1396 DCHECK(is_uint4(cc));
1397 emit(0x0F);
1398 emit(0x80 | cc);
1399 emit_runtime_entry(entry, rmode);
1400 }
1401
j(Condition cc,Handle<Code> target,RelocInfo::Mode rmode)1402 void Assembler::j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode) {
1403 if (cc == always) {
1404 jmp(target, rmode);
1405 return;
1406 } else if (cc == never) {
1407 return;
1408 }
1409 EnsureSpace ensure_space(this);
1410 DCHECK(is_uint4(cc));
1411 // 0000 1111 1000 tttn #32-bit disp.
1412 emit(0x0F);
1413 emit(0x80 | cc);
1414 DCHECK(RelocInfo::IsCodeTarget(rmode));
1415 RecordRelocInfo(rmode);
1416 int code_target_index = AddCodeTarget(target);
1417 emitl(code_target_index);
1418 }
1419
jmp_rel(int32_t offset)1420 void Assembler::jmp_rel(int32_t offset) {
1421 EnsureSpace ensure_space(this);
1422 // The offset is encoded relative to the next instruction.
1423 constexpr int32_t kShortJmpDisplacement = 1 + sizeof(int8_t);
1424 constexpr int32_t kNearJmpDisplacement = 1 + sizeof(int32_t);
1425 DCHECK_LE(std::numeric_limits<int32_t>::min() + kNearJmpDisplacement, offset);
1426 if (is_int8(offset - kShortJmpDisplacement) && !predictable_code_size()) {
1427 // 0xEB #8-bit disp.
1428 emit(0xEB);
1429 emit(offset - kShortJmpDisplacement);
1430 } else {
1431 // 0xE9 #32-bit disp.
1432 emit(0xE9);
1433 emitl(offset - kNearJmpDisplacement);
1434 }
1435 }
1436
jmp(Label * L,Label::Distance distance)1437 void Assembler::jmp(Label* L, Label::Distance distance) {
1438 const int long_size = sizeof(int32_t);
1439
1440 if (L->is_bound()) {
1441 int offset = L->pos() - pc_offset();
1442 DCHECK_LE(offset, 0); // backward jump.
1443 jmp_rel(offset);
1444 return;
1445 }
1446
1447 EnsureSpace ensure_space(this);
1448 if (distance == Label::kNear) {
1449 emit(0xEB);
1450 byte disp = 0x00;
1451 if (L->is_near_linked()) {
1452 int offset = L->near_link_pos() - pc_offset();
1453 DCHECK(is_int8(offset));
1454 disp = static_cast<byte>(offset & 0xFF);
1455 }
1456 L->link_to(pc_offset(), Label::kNear);
1457 emit(disp);
1458 } else {
1459 auto jump_opt = jump_optimization_info();
1460 if (V8_UNLIKELY(jump_opt)) {
1461 if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1462 emit(0xEB);
1463 record_farjmp_position(L, pc_offset());
1464 emit(0);
1465 return;
1466 }
1467 if (jump_opt->is_collecting()) {
1468 farjmp_positions_.push_back(pc_offset() + 1);
1469 }
1470 }
1471 if (L->is_linked()) {
1472 // 1110 1001 #32-bit disp.
1473 emit(0xE9);
1474 emitl(L->pos());
1475 L->link_to(pc_offset() - long_size);
1476 } else {
1477 // 1110 1001 #32-bit disp.
1478 DCHECK(L->is_unused());
1479 emit(0xE9);
1480 int32_t current = pc_offset();
1481 emitl(current);
1482 L->link_to(current);
1483 }
1484 }
1485 }
1486
jmp(Handle<Code> target,RelocInfo::Mode rmode)1487 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1488 DCHECK(RelocInfo::IsCodeTarget(rmode));
1489 EnsureSpace ensure_space(this);
1490 // 1110 1001 #32-bit disp.
1491 emit(0xE9);
1492 RecordRelocInfo(rmode);
1493 int code_target_index = AddCodeTarget(target);
1494 emitl(code_target_index);
1495 }
1496
jmp(Register target)1497 void Assembler::jmp(Register target) {
1498 EnsureSpace ensure_space(this);
1499 // Opcode FF/4 r64.
1500 emit_optional_rex_32(target);
1501 emit(0xFF);
1502 emit_modrm(0x4, target);
1503 }
1504
jmp(Operand src)1505 void Assembler::jmp(Operand src) {
1506 EnsureSpace ensure_space(this);
1507 // Opcode FF/4 m64.
1508 emit_optional_rex_32(src);
1509 emit(0xFF);
1510 emit_operand(0x4, src);
1511 }
1512
emit_lea(Register dst,Operand src,int size)1513 void Assembler::emit_lea(Register dst, Operand src, int size) {
1514 EnsureSpace ensure_space(this);
1515 emit_rex(dst, src, size);
1516 emit(0x8D);
1517 emit_operand(dst, src);
1518 }
1519
load_rax(Address value,RelocInfo::Mode mode)1520 void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
1521 EnsureSpace ensure_space(this);
1522 emit(0x48); // REX.W
1523 emit(0xA1);
1524 emit(Immediate64(value, mode));
1525 }
1526
load_rax(ExternalReference ref)1527 void Assembler::load_rax(ExternalReference ref) {
1528 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1529 }
1530
leave()1531 void Assembler::leave() {
1532 EnsureSpace ensure_space(this);
1533 emit(0xC9);
1534 }
1535
movb(Register dst,Operand src)1536 void Assembler::movb(Register dst, Operand src) {
1537 EnsureSpace ensure_space(this);
1538 if (!dst.is_byte_register()) {
1539 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1540 emit_rex_32(dst, src);
1541 } else {
1542 emit_optional_rex_32(dst, src);
1543 }
1544 emit(0x8A);
1545 emit_operand(dst, src);
1546 }
1547
movb(Register dst,Immediate imm)1548 void Assembler::movb(Register dst, Immediate imm) {
1549 EnsureSpace ensure_space(this);
1550 if (!dst.is_byte_register()) {
1551 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1552 emit_rex_32(dst);
1553 }
1554 emit(0xB0 + dst.low_bits());
1555 emit(imm.value_);
1556 }
1557
movb(Operand dst,Register src)1558 void Assembler::movb(Operand dst, Register src) {
1559 EnsureSpace ensure_space(this);
1560 if (!src.is_byte_register()) {
1561 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1562 emit_rex_32(src, dst);
1563 } else {
1564 emit_optional_rex_32(src, dst);
1565 }
1566 emit(0x88);
1567 emit_operand(src, dst);
1568 }
1569
movb(Operand dst,Immediate imm)1570 void Assembler::movb(Operand dst, Immediate imm) {
1571 EnsureSpace ensure_space(this);
1572 emit_optional_rex_32(dst);
1573 emit(0xC6);
1574 emit_operand(0x0, dst);
1575 emit(static_cast<byte>(imm.value_));
1576 }
1577
movw(Register dst,Operand src)1578 void Assembler::movw(Register dst, Operand src) {
1579 EnsureSpace ensure_space(this);
1580 emit(0x66);
1581 emit_optional_rex_32(dst, src);
1582 emit(0x8B);
1583 emit_operand(dst, src);
1584 }
1585
movw(Operand dst,Register src)1586 void Assembler::movw(Operand dst, Register src) {
1587 EnsureSpace ensure_space(this);
1588 emit(0x66);
1589 emit_optional_rex_32(src, dst);
1590 emit(0x89);
1591 emit_operand(src, dst);
1592 }
1593
movw(Operand dst,Immediate imm)1594 void Assembler::movw(Operand dst, Immediate imm) {
1595 EnsureSpace ensure_space(this);
1596 emit(0x66);
1597 emit_optional_rex_32(dst);
1598 emit(0xC7);
1599 emit_operand(0x0, dst);
1600 emit(static_cast<byte>(imm.value_ & 0xFF));
1601 emit(static_cast<byte>(imm.value_ >> 8));
1602 }
1603
emit_mov(Register dst,Operand src,int size)1604 void Assembler::emit_mov(Register dst, Operand src, int size) {
1605 EnsureSpace ensure_space(this);
1606 emit_rex(dst, src, size);
1607 emit(0x8B);
1608 emit_operand(dst, src);
1609 }
1610
emit_mov(Register dst,Register src,int size)1611 void Assembler::emit_mov(Register dst, Register src, int size) {
1612 EnsureSpace ensure_space(this);
1613 if (src.low_bits() == 4) {
1614 emit_rex(src, dst, size);
1615 emit(0x89);
1616 emit_modrm(src, dst);
1617 } else {
1618 emit_rex(dst, src, size);
1619 emit(0x8B);
1620 emit_modrm(dst, src);
1621 }
1622
1623 #if defined(V8_OS_WIN_X64)
1624 if (xdata_encoder_ && dst == rbp && src == rsp) {
1625 xdata_encoder_->onMovRbpRsp();
1626 }
1627 #endif
1628 }
1629
emit_mov(Operand dst,Register src,int size)1630 void Assembler::emit_mov(Operand dst, Register src, int size) {
1631 EnsureSpace ensure_space(this);
1632 emit_rex(src, dst, size);
1633 emit(0x89);
1634 emit_operand(src, dst);
1635 }
1636
emit_mov(Register dst,Immediate value,int size)1637 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1638 EnsureSpace ensure_space(this);
1639 emit_rex(dst, size);
1640 if (size == kInt64Size) {
1641 emit(0xC7);
1642 emit_modrm(0x0, dst);
1643 } else {
1644 DCHECK_EQ(size, kInt32Size);
1645 emit(0xB8 + dst.low_bits());
1646 }
1647 emit(value);
1648 }
1649
emit_mov(Operand dst,Immediate value,int size)1650 void Assembler::emit_mov(Operand dst, Immediate value, int size) {
1651 EnsureSpace ensure_space(this);
1652 emit_rex(dst, size);
1653 emit(0xC7);
1654 emit_operand(0x0, dst);
1655 emit(value);
1656 }
1657
emit_mov(Register dst,Immediate64 value,int size)1658 void Assembler::emit_mov(Register dst, Immediate64 value, int size) {
1659 DCHECK_EQ(size, kInt64Size);
1660 if (constpool_.TryRecordEntry(value.value_, value.rmode_)) {
1661 // Emit rip-relative move with offset = 0
1662 Label label;
1663 emit_mov(dst, Operand(&label, 0), size);
1664 bind(&label);
1665 } else {
1666 EnsureSpace ensure_space(this);
1667 emit_rex(dst, size);
1668 emit(0xB8 | dst.low_bits());
1669 emit(value);
1670 }
1671 }
1672
movq_imm64(Register dst,int64_t value)1673 void Assembler::movq_imm64(Register dst, int64_t value) {
1674 EnsureSpace ensure_space(this);
1675 emit_rex(dst, kInt64Size);
1676 emit(0xB8 | dst.low_bits());
1677 emitq(static_cast<uint64_t>(value));
1678 }
1679
movq_heap_number(Register dst,double value)1680 void Assembler::movq_heap_number(Register dst, double value) {
1681 EnsureSpace ensure_space(this);
1682 emit_rex(dst, kInt64Size);
1683 emit(0xB8 | dst.low_bits());
1684 RequestHeapObject(HeapObjectRequest(value));
1685 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1686 }
1687
movq_string(Register dst,const StringConstantBase * str)1688 void Assembler::movq_string(Register dst, const StringConstantBase* str) {
1689 EnsureSpace ensure_space(this);
1690 emit_rex(dst, kInt64Size);
1691 emit(0xB8 | dst.low_bits());
1692 RequestHeapObject(HeapObjectRequest(str));
1693 emit(Immediate64(kNullAddress, RelocInfo::FULL_EMBEDDED_OBJECT));
1694 }
1695
1696 // Loads the ip-relative location of the src label into the target location
1697 // (as a 32-bit offset sign extended to 64-bit).
movl(Operand dst,Label * src)1698 void Assembler::movl(Operand dst, Label* src) {
1699 EnsureSpace ensure_space(this);
1700 emit_optional_rex_32(dst);
1701 emit(0xC7);
1702 emit_operand(0, dst);
1703 if (src->is_bound()) {
1704 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1705 DCHECK_LE(offset, 0);
1706 emitl(offset);
1707 } else if (src->is_linked()) {
1708 emitl(src->pos());
1709 src->link_to(pc_offset() - sizeof(int32_t));
1710 } else {
1711 DCHECK(src->is_unused());
1712 int32_t current = pc_offset();
1713 emitl(current);
1714 src->link_to(current);
1715 }
1716 }
1717
movsxbl(Register dst,Register src)1718 void Assembler::movsxbl(Register dst, Register src) {
1719 EnsureSpace ensure_space(this);
1720 if (!src.is_byte_register()) {
1721 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1722 emit_rex_32(dst, src);
1723 } else {
1724 emit_optional_rex_32(dst, src);
1725 }
1726 emit(0x0F);
1727 emit(0xBE);
1728 emit_modrm(dst, src);
1729 }
1730
movsxbl(Register dst,Operand src)1731 void Assembler::movsxbl(Register dst, Operand src) {
1732 EnsureSpace ensure_space(this);
1733 emit_optional_rex_32(dst, src);
1734 emit(0x0F);
1735 emit(0xBE);
1736 emit_operand(dst, src);
1737 }
1738
movsxbq(Register dst,Operand src)1739 void Assembler::movsxbq(Register dst, Operand src) {
1740 EnsureSpace ensure_space(this);
1741 emit_rex_64(dst, src);
1742 emit(0x0F);
1743 emit(0xBE);
1744 emit_operand(dst, src);
1745 }
1746
movsxbq(Register dst,Register src)1747 void Assembler::movsxbq(Register dst, Register src) {
1748 EnsureSpace ensure_space(this);
1749 emit_rex_64(dst, src);
1750 emit(0x0F);
1751 emit(0xBE);
1752 emit_modrm(dst, src);
1753 }
1754
movsxwl(Register dst,Register src)1755 void Assembler::movsxwl(Register dst, Register src) {
1756 EnsureSpace ensure_space(this);
1757 emit_optional_rex_32(dst, src);
1758 emit(0x0F);
1759 emit(0xBF);
1760 emit_modrm(dst, src);
1761 }
1762
movsxwl(Register dst,Operand src)1763 void Assembler::movsxwl(Register dst, Operand src) {
1764 EnsureSpace ensure_space(this);
1765 emit_optional_rex_32(dst, src);
1766 emit(0x0F);
1767 emit(0xBF);
1768 emit_operand(dst, src);
1769 }
1770
movsxwq(Register dst,Operand src)1771 void Assembler::movsxwq(Register dst, Operand src) {
1772 EnsureSpace ensure_space(this);
1773 emit_rex_64(dst, src);
1774 emit(0x0F);
1775 emit(0xBF);
1776 emit_operand(dst, src);
1777 }
1778
movsxwq(Register dst,Register src)1779 void Assembler::movsxwq(Register dst, Register src) {
1780 EnsureSpace ensure_space(this);
1781 emit_rex_64(dst, src);
1782 emit(0x0F);
1783 emit(0xBF);
1784 emit_modrm(dst, src);
1785 }
1786
movsxlq(Register dst,Register src)1787 void Assembler::movsxlq(Register dst, Register src) {
1788 EnsureSpace ensure_space(this);
1789 emit_rex_64(dst, src);
1790 emit(0x63);
1791 emit_modrm(dst, src);
1792 }
1793
movsxlq(Register dst,Operand src)1794 void Assembler::movsxlq(Register dst, Operand src) {
1795 EnsureSpace ensure_space(this);
1796 emit_rex_64(dst, src);
1797 emit(0x63);
1798 emit_operand(dst, src);
1799 }
1800
emit_movzxb(Register dst,Operand src,int size)1801 void Assembler::emit_movzxb(Register dst, Operand src, int size) {
1802 EnsureSpace ensure_space(this);
1803 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1804 // there is no need to make this a 64 bit operation.
1805 emit_optional_rex_32(dst, src);
1806 emit(0x0F);
1807 emit(0xB6);
1808 emit_operand(dst, src);
1809 }
1810
emit_movzxb(Register dst,Register src,int size)1811 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1812 EnsureSpace ensure_space(this);
1813 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1814 // there is no need to make this a 64 bit operation.
1815 if (!src.is_byte_register()) {
1816 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1817 emit_rex_32(dst, src);
1818 } else {
1819 emit_optional_rex_32(dst, src);
1820 }
1821 emit(0x0F);
1822 emit(0xB6);
1823 emit_modrm(dst, src);
1824 }
1825
emit_movzxw(Register dst,Operand src,int size)1826 void Assembler::emit_movzxw(Register dst, Operand src, int size) {
1827 EnsureSpace ensure_space(this);
1828 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1829 // there is no need to make this a 64 bit operation.
1830 emit_optional_rex_32(dst, src);
1831 emit(0x0F);
1832 emit(0xB7);
1833 emit_operand(dst, src);
1834 }
1835
emit_movzxw(Register dst,Register src,int size)1836 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1837 EnsureSpace ensure_space(this);
1838 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1839 // there is no need to make this a 64 bit operation.
1840 emit_optional_rex_32(dst, src);
1841 emit(0x0F);
1842 emit(0xB7);
1843 emit_modrm(dst, src);
1844 }
1845
repmovsb()1846 void Assembler::repmovsb() {
1847 EnsureSpace ensure_space(this);
1848 emit(0xF3);
1849 emit(0xA4);
1850 }
1851
repmovsw()1852 void Assembler::repmovsw() {
1853 EnsureSpace ensure_space(this);
1854 emit(0x66); // Operand size override.
1855 emit(0xF3);
1856 emit(0xA4);
1857 }
1858
emit_repmovs(int size)1859 void Assembler::emit_repmovs(int size) {
1860 EnsureSpace ensure_space(this);
1861 emit(0xF3);
1862 emit_rex(size);
1863 emit(0xA5);
1864 }
1865
repstosl()1866 void Assembler::repstosl() {
1867 EnsureSpace ensure_space(this);
1868 emit(0xF3);
1869 emit(0xAB);
1870 }
1871
repstosq()1872 void Assembler::repstosq() {
1873 EnsureSpace ensure_space(this);
1874 emit(0xF3);
1875 emit_rex_64();
1876 emit(0xAB);
1877 }
1878
mull(Register src)1879 void Assembler::mull(Register src) {
1880 EnsureSpace ensure_space(this);
1881 emit_optional_rex_32(src);
1882 emit(0xF7);
1883 emit_modrm(0x4, src);
1884 }
1885
mull(Operand src)1886 void Assembler::mull(Operand src) {
1887 EnsureSpace ensure_space(this);
1888 emit_optional_rex_32(src);
1889 emit(0xF7);
1890 emit_operand(0x4, src);
1891 }
1892
mulq(Register src)1893 void Assembler::mulq(Register src) {
1894 EnsureSpace ensure_space(this);
1895 emit_rex_64(src);
1896 emit(0xF7);
1897 emit_modrm(0x4, src);
1898 }
1899
negb(Register reg)1900 void Assembler::negb(Register reg) {
1901 EnsureSpace ensure_space(this);
1902 emit_optional_rex_8(reg);
1903 emit(0xF6);
1904 emit_modrm(0x3, reg);
1905 }
1906
negw(Register reg)1907 void Assembler::negw(Register reg) {
1908 EnsureSpace ensure_space(this);
1909 emit(0x66);
1910 emit_optional_rex_32(reg);
1911 emit(0xF7);
1912 emit_modrm(0x3, reg);
1913 }
1914
negl(Register reg)1915 void Assembler::negl(Register reg) {
1916 EnsureSpace ensure_space(this);
1917 emit_optional_rex_32(reg);
1918 emit(0xF7);
1919 emit_modrm(0x3, reg);
1920 }
1921
negq(Register reg)1922 void Assembler::negq(Register reg) {
1923 EnsureSpace ensure_space(this);
1924 emit_rex_64(reg);
1925 emit(0xF7);
1926 emit_modrm(0x3, reg);
1927 }
1928
negb(Operand op)1929 void Assembler::negb(Operand op) {
1930 EnsureSpace ensure_space(this);
1931 emit_optional_rex_32(op);
1932 emit(0xF6);
1933 emit_operand(0x3, op);
1934 }
1935
negw(Operand op)1936 void Assembler::negw(Operand op) {
1937 EnsureSpace ensure_space(this);
1938 emit(0x66);
1939 emit_optional_rex_32(op);
1940 emit(0xF7);
1941 emit_operand(0x3, op);
1942 }
1943
negl(Operand op)1944 void Assembler::negl(Operand op) {
1945 EnsureSpace ensure_space(this);
1946 emit_optional_rex_32(op);
1947 emit(0xF7);
1948 emit_operand(0x3, op);
1949 }
1950
negq(Operand op)1951 void Assembler::negq(Operand op) {
1952 EnsureSpace ensure_space(this);
1953 emit_rex_64(op);
1954 emit(0xF7);
1955 emit_operand(0x3, op);
1956 }
1957
nop()1958 void Assembler::nop() {
1959 EnsureSpace ensure_space(this);
1960 emit(0x90);
1961 }
1962
emit_not(Register dst,int size)1963 void Assembler::emit_not(Register dst, int size) {
1964 EnsureSpace ensure_space(this);
1965 emit_rex(dst, size);
1966 emit(0xF7);
1967 emit_modrm(0x2, dst);
1968 }
1969
emit_not(Operand dst,int size)1970 void Assembler::emit_not(Operand dst, int size) {
1971 EnsureSpace ensure_space(this);
1972 emit_rex(dst, size);
1973 emit(0xF7);
1974 emit_operand(2, dst);
1975 }
1976
Nop(int n)1977 void Assembler::Nop(int n) {
1978 DCHECK_LE(0, n);
1979 // The recommended muti-byte sequences of NOP instructions from the Intel 64
1980 // and IA-32 Architectures Software Developer's Manual.
1981 //
1982 // Len Assembly Byte Sequence
1983 // 2 66 NOP 66 90H
1984 // 3 NOP DWORD ptr [EAX] 0F 1F 00H
1985 // 4 NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
1986 // 5 NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
1987 // 6 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
1988 // 7 NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
1989 // 8 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
1990 // 9 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] 66 0F 1F 84 00 00 00 00 00H
1991
1992 constexpr const char* kNopSequences =
1993 "\x66\x90" // length 1 (@1) / 2 (@0)
1994 "\x0F\x1F\x00" // length 3 (@2)
1995 "\x0F\x1F\x40\x00" // length 4 (@5)
1996 "\x66\x0F\x1F\x44\x00\x00" // length 5 (@10) / 6 (@9)
1997 "\x0F\x1F\x80\x00\x00\x00\x00" // length 7 (@15)
1998 "\x66\x0F\x1F\x84\x00\x00\x00\x00\x00"; // length 8 (@23) / 9 (@22)
1999 constexpr int8_t kNopOffsets[10] = {0, 1, 0, 2, 5, 10, 9, 15, 23, 22};
2000
2001 do {
2002 EnsureSpace ensure_space(this);
2003 int nop_bytes = std::min(n, 9);
2004 const char* sequence = kNopSequences + kNopOffsets[nop_bytes];
2005 memcpy(pc_, sequence, nop_bytes);
2006 pc_ += nop_bytes;
2007 n -= nop_bytes;
2008 } while (n);
2009 }
2010
popq(Register dst)2011 void Assembler::popq(Register dst) {
2012 EnsureSpace ensure_space(this);
2013 emit_optional_rex_32(dst);
2014 emit(0x58 | dst.low_bits());
2015 }
2016
popq(Operand dst)2017 void Assembler::popq(Operand dst) {
2018 EnsureSpace ensure_space(this);
2019 emit_optional_rex_32(dst);
2020 emit(0x8F);
2021 emit_operand(0, dst);
2022 }
2023
popfq()2024 void Assembler::popfq() {
2025 EnsureSpace ensure_space(this);
2026 emit(0x9D);
2027 }
2028
pushq(Register src)2029 void Assembler::pushq(Register src) {
2030 EnsureSpace ensure_space(this);
2031 emit_optional_rex_32(src);
2032 emit(0x50 | src.low_bits());
2033
2034 #if defined(V8_OS_WIN_X64)
2035 if (xdata_encoder_ && src == rbp) {
2036 xdata_encoder_->onPushRbp();
2037 }
2038 #endif
2039 }
2040
pushq(Operand src)2041 void Assembler::pushq(Operand src) {
2042 EnsureSpace ensure_space(this);
2043 emit_optional_rex_32(src);
2044 emit(0xFF);
2045 emit_operand(6, src);
2046 }
2047
pushq(Immediate value)2048 void Assembler::pushq(Immediate value) {
2049 EnsureSpace ensure_space(this);
2050 if (is_int8(value.value_)) {
2051 emit(0x6A);
2052 emit(value.value_); // Emit low byte of value.
2053 } else {
2054 emit(0x68);
2055 emitl(value.value_);
2056 }
2057 }
2058
pushq_imm32(int32_t imm32)2059 void Assembler::pushq_imm32(int32_t imm32) {
2060 EnsureSpace ensure_space(this);
2061 emit(0x68);
2062 emitl(imm32);
2063 }
2064
pushfq()2065 void Assembler::pushfq() {
2066 EnsureSpace ensure_space(this);
2067 emit(0x9C);
2068 }
2069
ret(int imm16)2070 void Assembler::ret(int imm16) {
2071 EnsureSpace ensure_space(this);
2072 DCHECK(is_uint16(imm16));
2073 if (imm16 == 0) {
2074 emit(0xC3);
2075 } else {
2076 emit(0xC2);
2077 emit(imm16 & 0xFF);
2078 emit((imm16 >> 8) & 0xFF);
2079 }
2080 }
2081
ud2()2082 void Assembler::ud2() {
2083 EnsureSpace ensure_space(this);
2084 emit(0x0F);
2085 emit(0x0B);
2086 }
2087
setcc(Condition cc,Register reg)2088 void Assembler::setcc(Condition cc, Register reg) {
2089 if (cc > last_condition) {
2090 movb(reg, Immediate(cc == always ? 1 : 0));
2091 return;
2092 }
2093 EnsureSpace ensure_space(this);
2094 DCHECK(is_uint4(cc));
2095 if (!reg.is_byte_register()) {
2096 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2097 emit_rex_32(reg);
2098 }
2099 emit(0x0F);
2100 emit(0x90 | cc);
2101 emit_modrm(0x0, reg);
2102 }
2103
shld(Register dst,Register src)2104 void Assembler::shld(Register dst, Register src) {
2105 EnsureSpace ensure_space(this);
2106 emit_rex_64(src, dst);
2107 emit(0x0F);
2108 emit(0xA5);
2109 emit_modrm(src, dst);
2110 }
2111
shrd(Register dst,Register src)2112 void Assembler::shrd(Register dst, Register src) {
2113 EnsureSpace ensure_space(this);
2114 emit_rex_64(src, dst);
2115 emit(0x0F);
2116 emit(0xAD);
2117 emit_modrm(src, dst);
2118 }
2119
xchgb(Register reg,Operand op)2120 void Assembler::xchgb(Register reg, Operand op) {
2121 EnsureSpace ensure_space(this);
2122 if (!reg.is_byte_register()) {
2123 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2124 emit_rex_32(reg, op);
2125 } else {
2126 emit_optional_rex_32(reg, op);
2127 }
2128 emit(0x86);
2129 emit_operand(reg, op);
2130 }
2131
xchgw(Register reg,Operand op)2132 void Assembler::xchgw(Register reg, Operand op) {
2133 EnsureSpace ensure_space(this);
2134 emit(0x66);
2135 emit_optional_rex_32(reg, op);
2136 emit(0x87);
2137 emit_operand(reg, op);
2138 }
2139
emit_xchg(Register dst,Register src,int size)2140 void Assembler::emit_xchg(Register dst, Register src, int size) {
2141 EnsureSpace ensure_space(this);
2142 if (src == rax || dst == rax) { // Single-byte encoding
2143 Register other = src == rax ? dst : src;
2144 emit_rex(other, size);
2145 emit(0x90 | other.low_bits());
2146 } else if (dst.low_bits() == 4) {
2147 emit_rex(dst, src, size);
2148 emit(0x87);
2149 emit_modrm(dst, src);
2150 } else {
2151 emit_rex(src, dst, size);
2152 emit(0x87);
2153 emit_modrm(src, dst);
2154 }
2155 }
2156
emit_xchg(Register dst,Operand src,int size)2157 void Assembler::emit_xchg(Register dst, Operand src, int size) {
2158 EnsureSpace ensure_space(this);
2159 emit_rex(dst, src, size);
2160 emit(0x87);
2161 emit_operand(dst, src);
2162 }
2163
store_rax(Address dst,RelocInfo::Mode mode)2164 void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
2165 EnsureSpace ensure_space(this);
2166 emit(0x48); // REX.W
2167 emit(0xA3);
2168 emit(Immediate64(dst, mode));
2169 }
2170
store_rax(ExternalReference ref)2171 void Assembler::store_rax(ExternalReference ref) {
2172 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2173 }
2174
sub_sp_32(uint32_t imm)2175 void Assembler::sub_sp_32(uint32_t imm) {
2176 emit_rex_64();
2177 emit(0x81); // using a literal 32-bit immediate.
2178 emit_modrm(0x5, rsp);
2179 emitl(imm);
2180 }
2181
testb(Register dst,Register src)2182 void Assembler::testb(Register dst, Register src) {
2183 EnsureSpace ensure_space(this);
2184 emit_test(dst, src, sizeof(int8_t));
2185 }
2186
testb(Register reg,Immediate mask)2187 void Assembler::testb(Register reg, Immediate mask) {
2188 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2189 emit_test(reg, mask, sizeof(int8_t));
2190 }
2191
testb(Operand op,Immediate mask)2192 void Assembler::testb(Operand op, Immediate mask) {
2193 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2194 emit_test(op, mask, sizeof(int8_t));
2195 }
2196
testb(Operand op,Register reg)2197 void Assembler::testb(Operand op, Register reg) {
2198 emit_test(op, reg, sizeof(int8_t));
2199 }
2200
testw(Register dst,Register src)2201 void Assembler::testw(Register dst, Register src) {
2202 emit_test(dst, src, sizeof(uint16_t));
2203 }
2204
testw(Register reg,Immediate mask)2205 void Assembler::testw(Register reg, Immediate mask) {
2206 emit_test(reg, mask, sizeof(int16_t));
2207 }
2208
testw(Operand op,Immediate mask)2209 void Assembler::testw(Operand op, Immediate mask) {
2210 emit_test(op, mask, sizeof(int16_t));
2211 }
2212
testw(Operand op,Register reg)2213 void Assembler::testw(Operand op, Register reg) {
2214 emit_test(op, reg, sizeof(int16_t));
2215 }
2216
emit_test(Register dst,Register src,int size)2217 void Assembler::emit_test(Register dst, Register src, int size) {
2218 EnsureSpace ensure_space(this);
2219 if (src.low_bits() == 4) std::swap(dst, src);
2220 if (size == sizeof(int16_t)) {
2221 emit(0x66);
2222 size = sizeof(int32_t);
2223 }
2224 bool byte_operand = size == sizeof(int8_t);
2225 if (byte_operand) {
2226 size = sizeof(int32_t);
2227 if (!src.is_byte_register() || !dst.is_byte_register()) {
2228 emit_rex_32(dst, src);
2229 }
2230 } else {
2231 emit_rex(dst, src, size);
2232 }
2233 emit(byte_operand ? 0x84 : 0x85);
2234 emit_modrm(dst, src);
2235 }
2236
emit_test(Register reg,Immediate mask,int size)2237 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2238 if (is_uint8(mask.value_)) {
2239 size = sizeof(int8_t);
2240 } else if (is_uint16(mask.value_)) {
2241 size = sizeof(int16_t);
2242 }
2243 EnsureSpace ensure_space(this);
2244 bool half_word = size == sizeof(int16_t);
2245 if (half_word) {
2246 emit(0x66);
2247 size = sizeof(int32_t);
2248 }
2249 bool byte_operand = size == sizeof(int8_t);
2250 if (byte_operand) {
2251 size = sizeof(int32_t);
2252 if (!reg.is_byte_register()) emit_rex_32(reg);
2253 } else {
2254 emit_rex(reg, size);
2255 }
2256 if (reg == rax) {
2257 emit(byte_operand ? 0xA8 : 0xA9);
2258 } else {
2259 emit(byte_operand ? 0xF6 : 0xF7);
2260 emit_modrm(0x0, reg);
2261 }
2262 if (byte_operand) {
2263 emit(mask.value_);
2264 } else if (half_word) {
2265 emitw(mask.value_);
2266 } else {
2267 emit(mask);
2268 }
2269 }
2270
emit_test(Operand op,Immediate mask,int size)2271 void Assembler::emit_test(Operand op, Immediate mask, int size) {
2272 if (is_uint8(mask.value_)) {
2273 size = sizeof(int8_t);
2274 } else if (is_uint16(mask.value_)) {
2275 size = sizeof(int16_t);
2276 }
2277 EnsureSpace ensure_space(this);
2278 bool half_word = size == sizeof(int16_t);
2279 if (half_word) {
2280 emit(0x66);
2281 size = sizeof(int32_t);
2282 }
2283 bool byte_operand = size == sizeof(int8_t);
2284 if (byte_operand) {
2285 size = sizeof(int32_t);
2286 }
2287 emit_rex(rax, op, size);
2288 emit(byte_operand ? 0xF6 : 0xF7);
2289 emit_operand(rax, op); // Operation code 0
2290 if (byte_operand) {
2291 emit(mask.value_);
2292 } else if (half_word) {
2293 emitw(mask.value_);
2294 } else {
2295 emit(mask);
2296 }
2297 }
2298
emit_test(Operand op,Register reg,int size)2299 void Assembler::emit_test(Operand op, Register reg, int size) {
2300 EnsureSpace ensure_space(this);
2301 if (size == sizeof(int16_t)) {
2302 emit(0x66);
2303 size = sizeof(int32_t);
2304 }
2305 bool byte_operand = size == sizeof(int8_t);
2306 if (byte_operand) {
2307 size = sizeof(int32_t);
2308 if (!reg.is_byte_register()) {
2309 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2310 emit_rex_32(reg, op);
2311 } else {
2312 emit_optional_rex_32(reg, op);
2313 }
2314 } else {
2315 emit_rex(reg, op, size);
2316 }
2317 emit(byte_operand ? 0x84 : 0x85);
2318 emit_operand(reg, op);
2319 }
2320
2321 // FPU instructions.
2322
fld(int i)2323 void Assembler::fld(int i) {
2324 EnsureSpace ensure_space(this);
2325 emit_farith(0xD9, 0xC0, i);
2326 }
2327
fld1()2328 void Assembler::fld1() {
2329 EnsureSpace ensure_space(this);
2330 emit(0xD9);
2331 emit(0xE8);
2332 }
2333
fldz()2334 void Assembler::fldz() {
2335 EnsureSpace ensure_space(this);
2336 emit(0xD9);
2337 emit(0xEE);
2338 }
2339
fldpi()2340 void Assembler::fldpi() {
2341 EnsureSpace ensure_space(this);
2342 emit(0xD9);
2343 emit(0xEB);
2344 }
2345
fldln2()2346 void Assembler::fldln2() {
2347 EnsureSpace ensure_space(this);
2348 emit(0xD9);
2349 emit(0xED);
2350 }
2351
fld_s(Operand adr)2352 void Assembler::fld_s(Operand adr) {
2353 EnsureSpace ensure_space(this);
2354 emit_optional_rex_32(adr);
2355 emit(0xD9);
2356 emit_operand(0, adr);
2357 }
2358
fld_d(Operand adr)2359 void Assembler::fld_d(Operand adr) {
2360 EnsureSpace ensure_space(this);
2361 emit_optional_rex_32(adr);
2362 emit(0xDD);
2363 emit_operand(0, adr);
2364 }
2365
fstp_s(Operand adr)2366 void Assembler::fstp_s(Operand adr) {
2367 EnsureSpace ensure_space(this);
2368 emit_optional_rex_32(adr);
2369 emit(0xD9);
2370 emit_operand(3, adr);
2371 }
2372
fstp_d(Operand adr)2373 void Assembler::fstp_d(Operand adr) {
2374 EnsureSpace ensure_space(this);
2375 emit_optional_rex_32(adr);
2376 emit(0xDD);
2377 emit_operand(3, adr);
2378 }
2379
fstp(int index)2380 void Assembler::fstp(int index) {
2381 DCHECK(is_uint3(index));
2382 EnsureSpace ensure_space(this);
2383 emit_farith(0xDD, 0xD8, index);
2384 }
2385
fild_s(Operand adr)2386 void Assembler::fild_s(Operand adr) {
2387 EnsureSpace ensure_space(this);
2388 emit_optional_rex_32(adr);
2389 emit(0xDB);
2390 emit_operand(0, adr);
2391 }
2392
fild_d(Operand adr)2393 void Assembler::fild_d(Operand adr) {
2394 EnsureSpace ensure_space(this);
2395 emit_optional_rex_32(adr);
2396 emit(0xDF);
2397 emit_operand(5, adr);
2398 }
2399
fistp_s(Operand adr)2400 void Assembler::fistp_s(Operand adr) {
2401 EnsureSpace ensure_space(this);
2402 emit_optional_rex_32(adr);
2403 emit(0xDB);
2404 emit_operand(3, adr);
2405 }
2406
fisttp_s(Operand adr)2407 void Assembler::fisttp_s(Operand adr) {
2408 DCHECK(IsEnabled(SSE3));
2409 EnsureSpace ensure_space(this);
2410 emit_optional_rex_32(adr);
2411 emit(0xDB);
2412 emit_operand(1, adr);
2413 }
2414
fisttp_d(Operand adr)2415 void Assembler::fisttp_d(Operand adr) {
2416 DCHECK(IsEnabled(SSE3));
2417 EnsureSpace ensure_space(this);
2418 emit_optional_rex_32(adr);
2419 emit(0xDD);
2420 emit_operand(1, adr);
2421 }
2422
fist_s(Operand adr)2423 void Assembler::fist_s(Operand adr) {
2424 EnsureSpace ensure_space(this);
2425 emit_optional_rex_32(adr);
2426 emit(0xDB);
2427 emit_operand(2, adr);
2428 }
2429
fistp_d(Operand adr)2430 void Assembler::fistp_d(Operand adr) {
2431 EnsureSpace ensure_space(this);
2432 emit_optional_rex_32(adr);
2433 emit(0xDF);
2434 emit_operand(7, adr);
2435 }
2436
fabs()2437 void Assembler::fabs() {
2438 EnsureSpace ensure_space(this);
2439 emit(0xD9);
2440 emit(0xE1);
2441 }
2442
fchs()2443 void Assembler::fchs() {
2444 EnsureSpace ensure_space(this);
2445 emit(0xD9);
2446 emit(0xE0);
2447 }
2448
fcos()2449 void Assembler::fcos() {
2450 EnsureSpace ensure_space(this);
2451 emit(0xD9);
2452 emit(0xFF);
2453 }
2454
fsin()2455 void Assembler::fsin() {
2456 EnsureSpace ensure_space(this);
2457 emit(0xD9);
2458 emit(0xFE);
2459 }
2460
fptan()2461 void Assembler::fptan() {
2462 EnsureSpace ensure_space(this);
2463 emit(0xD9);
2464 emit(0xF2);
2465 }
2466
fyl2x()2467 void Assembler::fyl2x() {
2468 EnsureSpace ensure_space(this);
2469 emit(0xD9);
2470 emit(0xF1);
2471 }
2472
f2xm1()2473 void Assembler::f2xm1() {
2474 EnsureSpace ensure_space(this);
2475 emit(0xD9);
2476 emit(0xF0);
2477 }
2478
fscale()2479 void Assembler::fscale() {
2480 EnsureSpace ensure_space(this);
2481 emit(0xD9);
2482 emit(0xFD);
2483 }
2484
fninit()2485 void Assembler::fninit() {
2486 EnsureSpace ensure_space(this);
2487 emit(0xDB);
2488 emit(0xE3);
2489 }
2490
fadd(int i)2491 void Assembler::fadd(int i) {
2492 EnsureSpace ensure_space(this);
2493 emit_farith(0xDC, 0xC0, i);
2494 }
2495
fsub(int i)2496 void Assembler::fsub(int i) {
2497 EnsureSpace ensure_space(this);
2498 emit_farith(0xDC, 0xE8, i);
2499 }
2500
fisub_s(Operand adr)2501 void Assembler::fisub_s(Operand adr) {
2502 EnsureSpace ensure_space(this);
2503 emit_optional_rex_32(adr);
2504 emit(0xDA);
2505 emit_operand(4, adr);
2506 }
2507
fmul(int i)2508 void Assembler::fmul(int i) {
2509 EnsureSpace ensure_space(this);
2510 emit_farith(0xDC, 0xC8, i);
2511 }
2512
fdiv(int i)2513 void Assembler::fdiv(int i) {
2514 EnsureSpace ensure_space(this);
2515 emit_farith(0xDC, 0xF8, i);
2516 }
2517
faddp(int i)2518 void Assembler::faddp(int i) {
2519 EnsureSpace ensure_space(this);
2520 emit_farith(0xDE, 0xC0, i);
2521 }
2522
fsubp(int i)2523 void Assembler::fsubp(int i) {
2524 EnsureSpace ensure_space(this);
2525 emit_farith(0xDE, 0xE8, i);
2526 }
2527
fsubrp(int i)2528 void Assembler::fsubrp(int i) {
2529 EnsureSpace ensure_space(this);
2530 emit_farith(0xDE, 0xE0, i);
2531 }
2532
fmulp(int i)2533 void Assembler::fmulp(int i) {
2534 EnsureSpace ensure_space(this);
2535 emit_farith(0xDE, 0xC8, i);
2536 }
2537
fdivp(int i)2538 void Assembler::fdivp(int i) {
2539 EnsureSpace ensure_space(this);
2540 emit_farith(0xDE, 0xF8, i);
2541 }
2542
fprem()2543 void Assembler::fprem() {
2544 EnsureSpace ensure_space(this);
2545 emit(0xD9);
2546 emit(0xF8);
2547 }
2548
fprem1()2549 void Assembler::fprem1() {
2550 EnsureSpace ensure_space(this);
2551 emit(0xD9);
2552 emit(0xF5);
2553 }
2554
fxch(int i)2555 void Assembler::fxch(int i) {
2556 EnsureSpace ensure_space(this);
2557 emit_farith(0xD9, 0xC8, i);
2558 }
2559
fincstp()2560 void Assembler::fincstp() {
2561 EnsureSpace ensure_space(this);
2562 emit(0xD9);
2563 emit(0xF7);
2564 }
2565
ffree(int i)2566 void Assembler::ffree(int i) {
2567 EnsureSpace ensure_space(this);
2568 emit_farith(0xDD, 0xC0, i);
2569 }
2570
ftst()2571 void Assembler::ftst() {
2572 EnsureSpace ensure_space(this);
2573 emit(0xD9);
2574 emit(0xE4);
2575 }
2576
fucomp(int i)2577 void Assembler::fucomp(int i) {
2578 EnsureSpace ensure_space(this);
2579 emit_farith(0xDD, 0xE8, i);
2580 }
2581
fucompp()2582 void Assembler::fucompp() {
2583 EnsureSpace ensure_space(this);
2584 emit(0xDA);
2585 emit(0xE9);
2586 }
2587
fucomi(int i)2588 void Assembler::fucomi(int i) {
2589 EnsureSpace ensure_space(this);
2590 emit(0xDB);
2591 emit(0xE8 + i);
2592 }
2593
fucomip()2594 void Assembler::fucomip() {
2595 EnsureSpace ensure_space(this);
2596 emit(0xDF);
2597 emit(0xE9);
2598 }
2599
fcompp()2600 void Assembler::fcompp() {
2601 EnsureSpace ensure_space(this);
2602 emit(0xDE);
2603 emit(0xD9);
2604 }
2605
fnstsw_ax()2606 void Assembler::fnstsw_ax() {
2607 EnsureSpace ensure_space(this);
2608 emit(0xDF);
2609 emit(0xE0);
2610 }
2611
fwait()2612 void Assembler::fwait() {
2613 EnsureSpace ensure_space(this);
2614 emit(0x9B);
2615 }
2616
frndint()2617 void Assembler::frndint() {
2618 EnsureSpace ensure_space(this);
2619 emit(0xD9);
2620 emit(0xFC);
2621 }
2622
fnclex()2623 void Assembler::fnclex() {
2624 EnsureSpace ensure_space(this);
2625 emit(0xDB);
2626 emit(0xE2);
2627 }
2628
sahf()2629 void Assembler::sahf() {
2630 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2631 // in 64-bit mode. Test CpuID.
2632 DCHECK(IsEnabled(SAHF));
2633 EnsureSpace ensure_space(this);
2634 emit(0x9E);
2635 }
2636
emit_farith(int b1,int b2,int i)2637 void Assembler::emit_farith(int b1, int b2, int i) {
2638 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2639 DCHECK(is_uint3(i)); // illegal stack offset
2640 emit(b1);
2641 emit(b2 + i);
2642 }
2643
2644 // SSE 2 operations.
2645
movd(XMMRegister dst,Register src)2646 void Assembler::movd(XMMRegister dst, Register src) {
2647 DCHECK(!IsEnabled(AVX));
2648 EnsureSpace ensure_space(this);
2649 emit(0x66);
2650 emit_optional_rex_32(dst, src);
2651 emit(0x0F);
2652 emit(0x6E);
2653 emit_sse_operand(dst, src);
2654 }
2655
movd(XMMRegister dst,Operand src)2656 void Assembler::movd(XMMRegister dst, Operand src) {
2657 DCHECK(!IsEnabled(AVX));
2658 EnsureSpace ensure_space(this);
2659 emit(0x66);
2660 emit_optional_rex_32(dst, src);
2661 emit(0x0F);
2662 emit(0x6E);
2663 emit_sse_operand(dst, src);
2664 }
2665
movd(Register dst,XMMRegister src)2666 void Assembler::movd(Register dst, XMMRegister src) {
2667 DCHECK(!IsEnabled(AVX));
2668 EnsureSpace ensure_space(this);
2669 emit(0x66);
2670 emit_optional_rex_32(src, dst);
2671 emit(0x0F);
2672 emit(0x7E);
2673 emit_sse_operand(src, dst);
2674 }
2675
movq(XMMRegister dst,Register src)2676 void Assembler::movq(XMMRegister dst, Register src) {
2677 // Mixing AVX and non-AVX is expensive, catch those cases
2678 DCHECK(!IsEnabled(AVX));
2679 EnsureSpace ensure_space(this);
2680 emit(0x66);
2681 emit_rex_64(dst, src);
2682 emit(0x0F);
2683 emit(0x6E);
2684 emit_sse_operand(dst, src);
2685 }
2686
movq(XMMRegister dst,Operand src)2687 void Assembler::movq(XMMRegister dst, Operand src) {
2688 // Mixing AVX and non-AVX is expensive, catch those cases
2689 DCHECK(!IsEnabled(AVX));
2690 EnsureSpace ensure_space(this);
2691 emit(0x66);
2692 emit_rex_64(dst, src);
2693 emit(0x0F);
2694 emit(0x6E);
2695 emit_sse_operand(dst, src);
2696 }
2697
movq(Register dst,XMMRegister src)2698 void Assembler::movq(Register dst, XMMRegister src) {
2699 // Mixing AVX and non-AVX is expensive, catch those cases
2700 DCHECK(!IsEnabled(AVX));
2701 EnsureSpace ensure_space(this);
2702 emit(0x66);
2703 emit_rex_64(src, dst);
2704 emit(0x0F);
2705 emit(0x7E);
2706 emit_sse_operand(src, dst);
2707 }
2708
movq(XMMRegister dst,XMMRegister src)2709 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2710 // Mixing AVX and non-AVX is expensive, catch those cases
2711 DCHECK(!IsEnabled(AVX));
2712 EnsureSpace ensure_space(this);
2713 if (dst.low_bits() == 4) {
2714 // Avoid unnecessary SIB byte.
2715 emit(0xF3);
2716 emit_optional_rex_32(dst, src);
2717 emit(0x0F);
2718 emit(0x7E);
2719 emit_sse_operand(dst, src);
2720 } else {
2721 emit(0x66);
2722 emit_optional_rex_32(src, dst);
2723 emit(0x0F);
2724 emit(0xD6);
2725 emit_sse_operand(src, dst);
2726 }
2727 }
2728
movdqa(Operand dst,XMMRegister src)2729 void Assembler::movdqa(Operand dst, XMMRegister src) {
2730 EnsureSpace ensure_space(this);
2731 emit(0x66);
2732 emit_rex_64(src, dst);
2733 emit(0x0F);
2734 emit(0x7F);
2735 emit_sse_operand(src, dst);
2736 }
2737
movdqa(XMMRegister dst,Operand src)2738 void Assembler::movdqa(XMMRegister dst, Operand src) {
2739 EnsureSpace ensure_space(this);
2740 emit(0x66);
2741 emit_rex_64(dst, src);
2742 emit(0x0F);
2743 emit(0x6F);
2744 emit_sse_operand(dst, src);
2745 }
2746
movdqu(Operand dst,XMMRegister src)2747 void Assembler::movdqu(Operand dst, XMMRegister src) {
2748 EnsureSpace ensure_space(this);
2749 emit(0xF3);
2750 emit_rex_64(src, dst);
2751 emit(0x0F);
2752 emit(0x7F);
2753 emit_sse_operand(src, dst);
2754 }
2755
movdqu(XMMRegister dst,Operand src)2756 void Assembler::movdqu(XMMRegister dst, Operand src) {
2757 EnsureSpace ensure_space(this);
2758 emit(0xF3);
2759 emit_rex_64(dst, src);
2760 emit(0x0F);
2761 emit(0x6F);
2762 emit_sse_operand(dst, src);
2763 }
2764
movdqu(XMMRegister dst,XMMRegister src)2765 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2766 EnsureSpace ensure_space(this);
2767 emit(0xF3);
2768 emit_rex_64(dst, src);
2769 emit(0x0F);
2770 emit(0x6F);
2771 emit_sse_operand(dst, src);
2772 }
2773
pinsrw(XMMRegister dst,Register src,uint8_t imm8)2774 void Assembler::pinsrw(XMMRegister dst, Register src, uint8_t imm8) {
2775 EnsureSpace ensure_space(this);
2776 emit(0x66);
2777 emit_optional_rex_32(dst, src);
2778 emit(0x0F);
2779 emit(0xC4);
2780 emit_sse_operand(dst, src);
2781 emit(imm8);
2782 }
2783
pinsrw(XMMRegister dst,Operand src,uint8_t imm8)2784 void Assembler::pinsrw(XMMRegister dst, Operand src, uint8_t imm8) {
2785 EnsureSpace ensure_space(this);
2786 emit(0x66);
2787 emit_optional_rex_32(dst, src);
2788 emit(0x0F);
2789 emit(0xC4);
2790 emit_sse_operand(dst, src);
2791 emit(imm8);
2792 }
2793
pextrq(Register dst,XMMRegister src,int8_t imm8)2794 void Assembler::pextrq(Register dst, XMMRegister src, int8_t imm8) {
2795 DCHECK(IsEnabled(SSE4_1));
2796 EnsureSpace ensure_space(this);
2797 emit(0x66);
2798 emit_rex_64(src, dst);
2799 emit(0x0F);
2800 emit(0x3A);
2801 emit(0x16);
2802 emit_sse_operand(src, dst);
2803 emit(imm8);
2804 }
2805
pinsrq(XMMRegister dst,Register src,uint8_t imm8)2806 void Assembler::pinsrq(XMMRegister dst, Register src, uint8_t imm8) {
2807 DCHECK(IsEnabled(SSE4_1));
2808 EnsureSpace ensure_space(this);
2809 emit(0x66);
2810 emit_rex_64(dst, src);
2811 emit(0x0F);
2812 emit(0x3A);
2813 emit(0x22);
2814 emit_sse_operand(dst, src);
2815 emit(imm8);
2816 }
2817
pinsrq(XMMRegister dst,Operand src,uint8_t imm8)2818 void Assembler::pinsrq(XMMRegister dst, Operand src, uint8_t imm8) {
2819 DCHECK(IsEnabled(SSE4_1));
2820 EnsureSpace ensure_space(this);
2821 emit(0x66);
2822 emit_rex_64(dst, src);
2823 emit(0x0F);
2824 emit(0x3A);
2825 emit(0x22);
2826 emit_sse_operand(dst, src);
2827 emit(imm8);
2828 }
2829
pinsrd(XMMRegister dst,Register src,uint8_t imm8)2830 void Assembler::pinsrd(XMMRegister dst, Register src, uint8_t imm8) {
2831 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22, imm8);
2832 }
2833
pinsrd(XMMRegister dst,Operand src,uint8_t imm8)2834 void Assembler::pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
2835 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x22);
2836 emit(imm8);
2837 }
2838
pinsrb(XMMRegister dst,Register src,uint8_t imm8)2839 void Assembler::pinsrb(XMMRegister dst, Register src, uint8_t imm8) {
2840 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20, imm8);
2841 }
2842
pinsrb(XMMRegister dst,Operand src,uint8_t imm8)2843 void Assembler::pinsrb(XMMRegister dst, Operand src, uint8_t imm8) {
2844 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x20);
2845 emit(imm8);
2846 }
2847
insertps(XMMRegister dst,XMMRegister src,byte imm8)2848 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2849 DCHECK(is_uint8(imm8));
2850 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2851 emit(imm8);
2852 }
2853
insertps(XMMRegister dst,Operand src,byte imm8)2854 void Assembler::insertps(XMMRegister dst, Operand src, byte imm8) {
2855 DCHECK(is_uint8(imm8));
2856 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x21);
2857 emit(imm8);
2858 }
2859
movsd(Operand dst,XMMRegister src)2860 void Assembler::movsd(Operand dst, XMMRegister src) {
2861 DCHECK(!IsEnabled(AVX));
2862 EnsureSpace ensure_space(this);
2863 emit(0xF2); // double
2864 emit_optional_rex_32(src, dst);
2865 emit(0x0F);
2866 emit(0x11); // store
2867 emit_sse_operand(src, dst);
2868 }
2869
movsd(XMMRegister dst,XMMRegister src)2870 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2871 DCHECK(!IsEnabled(AVX));
2872 EnsureSpace ensure_space(this);
2873 emit(0xF2); // double
2874 emit_optional_rex_32(dst, src);
2875 emit(0x0F);
2876 emit(0x10); // load
2877 emit_sse_operand(dst, src);
2878 }
2879
movsd(XMMRegister dst,Operand src)2880 void Assembler::movsd(XMMRegister dst, Operand src) {
2881 DCHECK(!IsEnabled(AVX));
2882 EnsureSpace ensure_space(this);
2883 emit(0xF2); // double
2884 emit_optional_rex_32(dst, src);
2885 emit(0x0F);
2886 emit(0x10); // load
2887 emit_sse_operand(dst, src);
2888 }
2889
movaps(XMMRegister dst,XMMRegister src)2890 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2891 DCHECK(!IsEnabled(AVX));
2892 EnsureSpace ensure_space(this);
2893 if (src.low_bits() == 4) {
2894 // Try to avoid an unnecessary SIB byte.
2895 emit_optional_rex_32(src, dst);
2896 emit(0x0F);
2897 emit(0x29);
2898 emit_sse_operand(src, dst);
2899 } else {
2900 emit_optional_rex_32(dst, src);
2901 emit(0x0F);
2902 emit(0x28);
2903 emit_sse_operand(dst, src);
2904 }
2905 }
2906
shufps(XMMRegister dst,XMMRegister src,byte imm8)2907 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2908 DCHECK(is_uint8(imm8));
2909 EnsureSpace ensure_space(this);
2910 emit_optional_rex_32(dst, src);
2911 emit(0x0F);
2912 emit(0xC6);
2913 emit_sse_operand(dst, src);
2914 emit(imm8);
2915 }
2916
movapd(XMMRegister dst,XMMRegister src)2917 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2918 DCHECK(!IsEnabled(AVX));
2919 EnsureSpace ensure_space(this);
2920 if (src.low_bits() == 4) {
2921 // Try to avoid an unnecessary SIB byte.
2922 emit(0x66);
2923 emit_optional_rex_32(src, dst);
2924 emit(0x0F);
2925 emit(0x29);
2926 emit_sse_operand(src, dst);
2927 } else {
2928 emit(0x66);
2929 emit_optional_rex_32(dst, src);
2930 emit(0x0F);
2931 emit(0x28);
2932 emit_sse_operand(dst, src);
2933 }
2934 }
2935
movupd(XMMRegister dst,Operand src)2936 void Assembler::movupd(XMMRegister dst, Operand src) {
2937 EnsureSpace ensure_space(this);
2938 emit(0x66);
2939 emit_optional_rex_32(dst, src);
2940 emit(0x0F);
2941 emit(0x10);
2942 emit_sse_operand(dst, src);
2943 }
2944
movupd(Operand dst,XMMRegister src)2945 void Assembler::movupd(Operand dst, XMMRegister src) {
2946 EnsureSpace ensure_space(this);
2947 emit(0x66);
2948 emit_optional_rex_32(src, dst);
2949 emit(0x0F);
2950 emit(0x11);
2951 emit_sse_operand(src, dst);
2952 }
2953
ucomiss(XMMRegister dst,XMMRegister src)2954 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2955 DCHECK(!IsEnabled(AVX));
2956 EnsureSpace ensure_space(this);
2957 emit_optional_rex_32(dst, src);
2958 emit(0x0F);
2959 emit(0x2E);
2960 emit_sse_operand(dst, src);
2961 }
2962
ucomiss(XMMRegister dst,Operand src)2963 void Assembler::ucomiss(XMMRegister dst, Operand src) {
2964 DCHECK(!IsEnabled(AVX));
2965 EnsureSpace ensure_space(this);
2966 emit_optional_rex_32(dst, src);
2967 emit(0x0F);
2968 emit(0x2E);
2969 emit_sse_operand(dst, src);
2970 }
2971
movss(XMMRegister dst,XMMRegister src)2972 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2973 DCHECK(!IsEnabled(AVX));
2974 EnsureSpace ensure_space(this);
2975 emit(0xF3); // single
2976 emit_optional_rex_32(dst, src);
2977 emit(0x0F);
2978 emit(0x10); // load
2979 emit_sse_operand(dst, src);
2980 }
2981
movss(XMMRegister dst,Operand src)2982 void Assembler::movss(XMMRegister dst, Operand src) {
2983 DCHECK(!IsEnabled(AVX));
2984 EnsureSpace ensure_space(this);
2985 emit(0xF3); // single
2986 emit_optional_rex_32(dst, src);
2987 emit(0x0F);
2988 emit(0x10); // load
2989 emit_sse_operand(dst, src);
2990 }
2991
movss(Operand src,XMMRegister dst)2992 void Assembler::movss(Operand src, XMMRegister dst) {
2993 DCHECK(!IsEnabled(AVX));
2994 EnsureSpace ensure_space(this);
2995 emit(0xF3); // single
2996 emit_optional_rex_32(dst, src);
2997 emit(0x0F);
2998 emit(0x11); // store
2999 emit_sse_operand(dst, src);
3000 }
3001
movlps(XMMRegister dst,Operand src)3002 void Assembler::movlps(XMMRegister dst, Operand src) {
3003 DCHECK(!IsEnabled(AVX));
3004 EnsureSpace ensure_space(this);
3005 emit_optional_rex_32(dst, src);
3006 emit(0x0F);
3007 emit(0x12);
3008 emit_sse_operand(dst, src);
3009 }
3010
movlps(Operand src,XMMRegister dst)3011 void Assembler::movlps(Operand src, XMMRegister dst) {
3012 DCHECK(!IsEnabled(AVX));
3013 EnsureSpace ensure_space(this);
3014 emit_optional_rex_32(dst, src);
3015 emit(0x0F);
3016 emit(0x13);
3017 emit_sse_operand(dst, src);
3018 }
3019
movhps(XMMRegister dst,Operand src)3020 void Assembler::movhps(XMMRegister dst, Operand src) {
3021 DCHECK(!IsEnabled(AVX));
3022 EnsureSpace ensure_space(this);
3023 emit_optional_rex_32(dst, src);
3024 emit(0x0F);
3025 emit(0x16);
3026 emit_sse_operand(dst, src);
3027 }
3028
movhps(Operand src,XMMRegister dst)3029 void Assembler::movhps(Operand src, XMMRegister dst) {
3030 DCHECK(!IsEnabled(AVX));
3031 EnsureSpace ensure_space(this);
3032 emit_optional_rex_32(dst, src);
3033 emit(0x0F);
3034 emit(0x17);
3035 emit_sse_operand(dst, src);
3036 }
3037
cmpps(XMMRegister dst,XMMRegister src,int8_t cmp)3038 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3039 EnsureSpace ensure_space(this);
3040 emit_optional_rex_32(dst, src);
3041 emit(0x0F);
3042 emit(0xC2);
3043 emit_sse_operand(dst, src);
3044 emit(cmp);
3045 }
3046
cmpps(XMMRegister dst,Operand src,int8_t cmp)3047 void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
3048 EnsureSpace ensure_space(this);
3049 emit_optional_rex_32(dst, src);
3050 emit(0x0F);
3051 emit(0xC2);
3052 emit_sse_operand(dst, src);
3053 emit(cmp);
3054 }
3055
cmppd(XMMRegister dst,XMMRegister src,int8_t cmp)3056 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3057 EnsureSpace ensure_space(this);
3058 emit(0x66);
3059 emit_optional_rex_32(dst, src);
3060 emit(0x0F);
3061 emit(0xC2);
3062 emit_sse_operand(dst, src);
3063 emit(cmp);
3064 }
3065
cmppd(XMMRegister dst,Operand src,int8_t cmp)3066 void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
3067 EnsureSpace ensure_space(this);
3068 emit(0x66);
3069 emit_optional_rex_32(dst, src);
3070 emit(0x0F);
3071 emit(0xC2);
3072 emit_sse_operand(dst, src);
3073 emit(cmp);
3074 }
3075
cvttss2si(Register dst,Operand src)3076 void Assembler::cvttss2si(Register dst, Operand src) {
3077 DCHECK(!IsEnabled(AVX));
3078 EnsureSpace ensure_space(this);
3079 emit(0xF3);
3080 emit_optional_rex_32(dst, src);
3081 emit(0x0F);
3082 emit(0x2C);
3083 emit_operand(dst, src);
3084 }
3085
cvttss2si(Register dst,XMMRegister src)3086 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3087 DCHECK(!IsEnabled(AVX));
3088 EnsureSpace ensure_space(this);
3089 emit(0xF3);
3090 emit_optional_rex_32(dst, src);
3091 emit(0x0F);
3092 emit(0x2C);
3093 emit_sse_operand(dst, src);
3094 }
3095
cvttsd2si(Register dst,Operand src)3096 void Assembler::cvttsd2si(Register dst, Operand src) {
3097 DCHECK(!IsEnabled(AVX));
3098 EnsureSpace ensure_space(this);
3099 emit(0xF2);
3100 emit_optional_rex_32(dst, src);
3101 emit(0x0F);
3102 emit(0x2C);
3103 emit_operand(dst, src);
3104 }
3105
cvttsd2si(Register dst,XMMRegister src)3106 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3107 DCHECK(!IsEnabled(AVX));
3108 EnsureSpace ensure_space(this);
3109 emit(0xF2);
3110 emit_optional_rex_32(dst, src);
3111 emit(0x0F);
3112 emit(0x2C);
3113 emit_sse_operand(dst, src);
3114 }
3115
cvttss2siq(Register dst,XMMRegister src)3116 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3117 DCHECK(!IsEnabled(AVX));
3118 EnsureSpace ensure_space(this);
3119 emit(0xF3);
3120 emit_rex_64(dst, src);
3121 emit(0x0F);
3122 emit(0x2C);
3123 emit_sse_operand(dst, src);
3124 }
3125
cvttss2siq(Register dst,Operand src)3126 void Assembler::cvttss2siq(Register dst, Operand src) {
3127 DCHECK(!IsEnabled(AVX));
3128 EnsureSpace ensure_space(this);
3129 emit(0xF3);
3130 emit_rex_64(dst, src);
3131 emit(0x0F);
3132 emit(0x2C);
3133 emit_sse_operand(dst, src);
3134 }
3135
cvttsd2siq(Register dst,XMMRegister src)3136 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3137 DCHECK(!IsEnabled(AVX));
3138 EnsureSpace ensure_space(this);
3139 emit(0xF2);
3140 emit_rex_64(dst, src);
3141 emit(0x0F);
3142 emit(0x2C);
3143 emit_sse_operand(dst, src);
3144 }
3145
cvttsd2siq(Register dst,Operand src)3146 void Assembler::cvttsd2siq(Register dst, Operand src) {
3147 DCHECK(!IsEnabled(AVX));
3148 EnsureSpace ensure_space(this);
3149 emit(0xF2);
3150 emit_rex_64(dst, src);
3151 emit(0x0F);
3152 emit(0x2C);
3153 emit_sse_operand(dst, src);
3154 }
3155
cvttps2dq(XMMRegister dst,Operand src)3156 void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
3157 EnsureSpace ensure_space(this);
3158 emit(0xF3);
3159 emit_rex_64(dst, src);
3160 emit(0x0F);
3161 emit(0x5B);
3162 emit_sse_operand(dst, src);
3163 }
3164
cvttps2dq(XMMRegister dst,XMMRegister src)3165 void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
3166 EnsureSpace ensure_space(this);
3167 emit(0xF3);
3168 emit_rex_64(dst, src);
3169 emit(0x0F);
3170 emit(0x5B);
3171 emit_sse_operand(dst, src);
3172 }
3173
cvtlsi2sd(XMMRegister dst,Operand src)3174 void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
3175 DCHECK(!IsEnabled(AVX));
3176 EnsureSpace ensure_space(this);
3177 emit(0xF2);
3178 emit_optional_rex_32(dst, src);
3179 emit(0x0F);
3180 emit(0x2A);
3181 emit_sse_operand(dst, src);
3182 }
3183
cvtlsi2sd(XMMRegister dst,Register src)3184 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3185 DCHECK(!IsEnabled(AVX));
3186 EnsureSpace ensure_space(this);
3187 emit(0xF2);
3188 emit_optional_rex_32(dst, src);
3189 emit(0x0F);
3190 emit(0x2A);
3191 emit_sse_operand(dst, src);
3192 }
3193
cvtlsi2ss(XMMRegister dst,Operand src)3194 void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
3195 DCHECK(!IsEnabled(AVX));
3196 EnsureSpace ensure_space(this);
3197 emit(0xF3);
3198 emit_optional_rex_32(dst, src);
3199 emit(0x0F);
3200 emit(0x2A);
3201 emit_sse_operand(dst, src);
3202 }
3203
cvtlsi2ss(XMMRegister dst,Register src)3204 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3205 EnsureSpace ensure_space(this);
3206 emit(0xF3);
3207 emit_optional_rex_32(dst, src);
3208 emit(0x0F);
3209 emit(0x2A);
3210 emit_sse_operand(dst, src);
3211 }
3212
cvtqsi2ss(XMMRegister dst,Operand src)3213 void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
3214 DCHECK(!IsEnabled(AVX));
3215 EnsureSpace ensure_space(this);
3216 emit(0xF3);
3217 emit_rex_64(dst, src);
3218 emit(0x0F);
3219 emit(0x2A);
3220 emit_sse_operand(dst, src);
3221 }
3222
cvtqsi2ss(XMMRegister dst,Register src)3223 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3224 DCHECK(!IsEnabled(AVX));
3225 EnsureSpace ensure_space(this);
3226 emit(0xF3);
3227 emit_rex_64(dst, src);
3228 emit(0x0F);
3229 emit(0x2A);
3230 emit_sse_operand(dst, src);
3231 }
3232
cvtqsi2sd(XMMRegister dst,Operand src)3233 void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
3234 DCHECK(!IsEnabled(AVX));
3235 EnsureSpace ensure_space(this);
3236 emit(0xF2);
3237 emit_rex_64(dst, src);
3238 emit(0x0F);
3239 emit(0x2A);
3240 emit_sse_operand(dst, src);
3241 }
3242
cvtqsi2sd(XMMRegister dst,Register src)3243 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3244 DCHECK(!IsEnabled(AVX));
3245 EnsureSpace ensure_space(this);
3246 emit(0xF2);
3247 emit_rex_64(dst, src);
3248 emit(0x0F);
3249 emit(0x2A);
3250 emit_sse_operand(dst, src);
3251 }
3252
cvtss2sd(XMMRegister dst,XMMRegister src)3253 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3254 DCHECK(!IsEnabled(AVX));
3255 EnsureSpace ensure_space(this);
3256 emit(0xF3);
3257 emit_optional_rex_32(dst, src);
3258 emit(0x0F);
3259 emit(0x5A);
3260 emit_sse_operand(dst, src);
3261 }
3262
cvtss2sd(XMMRegister dst,Operand src)3263 void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
3264 DCHECK(!IsEnabled(AVX));
3265 EnsureSpace ensure_space(this);
3266 emit(0xF3);
3267 emit_optional_rex_32(dst, src);
3268 emit(0x0F);
3269 emit(0x5A);
3270 emit_sse_operand(dst, src);
3271 }
3272
cvtsd2si(Register dst,XMMRegister src)3273 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3274 DCHECK(!IsEnabled(AVX));
3275 EnsureSpace ensure_space(this);
3276 emit(0xF2);
3277 emit_optional_rex_32(dst, src);
3278 emit(0x0F);
3279 emit(0x2D);
3280 emit_sse_operand(dst, src);
3281 }
3282
cvtsd2siq(Register dst,XMMRegister src)3283 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3284 DCHECK(!IsEnabled(AVX));
3285 EnsureSpace ensure_space(this);
3286 emit(0xF2);
3287 emit_rex_64(dst, src);
3288 emit(0x0F);
3289 emit(0x2D);
3290 emit_sse_operand(dst, src);
3291 }
3292
haddps(XMMRegister dst,XMMRegister src)3293 void Assembler::haddps(XMMRegister dst, XMMRegister src) {
3294 DCHECK(IsEnabled(SSE3));
3295 EnsureSpace ensure_space(this);
3296 emit(0xF2);
3297 emit_optional_rex_32(dst, src);
3298 emit(0x0F);
3299 emit(0x7C);
3300 emit_sse_operand(dst, src);
3301 }
3302
haddps(XMMRegister dst,Operand src)3303 void Assembler::haddps(XMMRegister dst, Operand src) {
3304 DCHECK(IsEnabled(SSE3));
3305 EnsureSpace ensure_space(this);
3306 emit(0xF2);
3307 emit_optional_rex_32(dst, src);
3308 emit(0x0F);
3309 emit(0x7C);
3310 emit_sse_operand(dst, src);
3311 }
3312
ucomisd(XMMRegister dst,XMMRegister src)3313 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3314 DCHECK(!IsEnabled(AVX));
3315 EnsureSpace ensure_space(this);
3316 emit(0x66);
3317 emit_optional_rex_32(dst, src);
3318 emit(0x0F);
3319 emit(0x2E);
3320 emit_sse_operand(dst, src);
3321 }
3322
ucomisd(XMMRegister dst,Operand src)3323 void Assembler::ucomisd(XMMRegister dst, Operand src) {
3324 DCHECK(!IsEnabled(AVX));
3325 EnsureSpace ensure_space(this);
3326 emit(0x66);
3327 emit_optional_rex_32(dst, src);
3328 emit(0x0F);
3329 emit(0x2E);
3330 emit_sse_operand(dst, src);
3331 }
3332
cmpltsd(XMMRegister dst,XMMRegister src)3333 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3334 EnsureSpace ensure_space(this);
3335 emit(0xF2);
3336 emit_optional_rex_32(dst, src);
3337 emit(0x0F);
3338 emit(0xC2);
3339 emit_sse_operand(dst, src);
3340 emit(0x01); // LT == 1
3341 }
3342
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)3343 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3344 DCHECK(!IsEnabled(AVX));
3345 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0A);
3346 // Mask precision exception.
3347 emit(static_cast<byte>(mode) | 0x8);
3348 }
3349
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)3350 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3351 DCHECK(!IsEnabled(AVX));
3352 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0B);
3353 // Mask precision exception.
3354 emit(static_cast<byte>(mode) | 0x8);
3355 }
3356
roundps(XMMRegister dst,XMMRegister src,RoundingMode mode)3357 void Assembler::roundps(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3358 DCHECK(!IsEnabled(AVX));
3359 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x08);
3360 // Mask precision exception.
3361 emit(static_cast<byte>(mode) | 0x8);
3362 }
3363
roundpd(XMMRegister dst,XMMRegister src,RoundingMode mode)3364 void Assembler::roundpd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3365 DCHECK(!IsEnabled(AVX));
3366 sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x09);
3367 // Mask precision exception.
3368 emit(static_cast<byte>(mode) | 0x8);
3369 }
3370
movmskpd(Register dst,XMMRegister src)3371 void Assembler::movmskpd(Register dst, XMMRegister src) {
3372 EnsureSpace ensure_space(this);
3373 emit(0x66);
3374 emit_optional_rex_32(dst, src);
3375 emit(0x0F);
3376 emit(0x50);
3377 emit_sse_operand(dst, src);
3378 }
3379
movmskps(Register dst,XMMRegister src)3380 void Assembler::movmskps(Register dst, XMMRegister src) {
3381 EnsureSpace ensure_space(this);
3382 emit_optional_rex_32(dst, src);
3383 emit(0x0F);
3384 emit(0x50);
3385 emit_sse_operand(dst, src);
3386 }
3387
pmovmskb(Register dst,XMMRegister src)3388 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3389 EnsureSpace ensure_space(this);
3390 emit(0x66);
3391 emit_optional_rex_32(dst, src);
3392 emit(0x0F);
3393 emit(0xD7);
3394 emit_sse_operand(dst, src);
3395 }
3396
3397 // AVX instructions
3398
vmovddup(XMMRegister dst,XMMRegister src)3399 void Assembler::vmovddup(XMMRegister dst, XMMRegister src) {
3400 DCHECK(IsEnabled(AVX));
3401 EnsureSpace ensure_space(this);
3402 emit_vex_prefix(dst, xmm0, src, kL128, kF2, k0F, kWIG);
3403 emit(0x12);
3404 emit_sse_operand(dst, src);
3405 }
3406
vmovddup(XMMRegister dst,Operand src)3407 void Assembler::vmovddup(XMMRegister dst, Operand src) {
3408 DCHECK(IsEnabled(AVX));
3409 EnsureSpace ensure_space(this);
3410 emit_vex_prefix(dst, xmm0, src, kL128, kF2, k0F, kWIG);
3411 emit(0x12);
3412 emit_sse_operand(dst, src);
3413 }
3414
vbroadcastss(XMMRegister dst,Operand src)3415 void Assembler::vbroadcastss(XMMRegister dst, Operand src) {
3416 DCHECK(IsEnabled(AVX));
3417 EnsureSpace ensure_space(this);
3418 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F38, kW0);
3419 emit(0x18);
3420 emit_sse_operand(dst, src);
3421 }
3422
fma_instr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,VectorLength l,SIMDPrefix pp,LeadingOpcode m,VexW w)3423 void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3424 XMMRegister src2, VectorLength l, SIMDPrefix pp,
3425 LeadingOpcode m, VexW w) {
3426 DCHECK(IsEnabled(FMA3));
3427 EnsureSpace ensure_space(this);
3428 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3429 emit(op);
3430 emit_sse_operand(dst, src2);
3431 }
3432
fma_instr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,VectorLength l,SIMDPrefix pp,LeadingOpcode m,VexW w)3433 void Assembler::fma_instr(byte op, XMMRegister dst, XMMRegister src1,
3434 Operand src2, VectorLength l, SIMDPrefix pp,
3435 LeadingOpcode m, VexW w) {
3436 DCHECK(IsEnabled(FMA3));
3437 EnsureSpace ensure_space(this);
3438 emit_vex_prefix(dst, src1, src2, l, pp, m, w);
3439 emit(op);
3440 emit_sse_operand(dst, src2);
3441 }
3442
vmovd(XMMRegister dst,Register src)3443 void Assembler::vmovd(XMMRegister dst, Register src) {
3444 DCHECK(IsEnabled(AVX));
3445 EnsureSpace ensure_space(this);
3446 XMMRegister isrc = XMMRegister::from_code(src.code());
3447 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
3448 emit(0x6E);
3449 emit_sse_operand(dst, src);
3450 }
3451
vmovd(XMMRegister dst,Operand src)3452 void Assembler::vmovd(XMMRegister dst, Operand src) {
3453 DCHECK(IsEnabled(AVX));
3454 EnsureSpace ensure_space(this);
3455 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
3456 emit(0x6E);
3457 emit_sse_operand(dst, src);
3458 }
3459
vmovd(Register dst,XMMRegister src)3460 void Assembler::vmovd(Register dst, XMMRegister src) {
3461 DCHECK(IsEnabled(AVX));
3462 EnsureSpace ensure_space(this);
3463 XMMRegister idst = XMMRegister::from_code(dst.code());
3464 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
3465 emit(0x7E);
3466 emit_sse_operand(src, dst);
3467 }
3468
vmovq(XMMRegister dst,Register src)3469 void Assembler::vmovq(XMMRegister dst, Register src) {
3470 DCHECK(IsEnabled(AVX));
3471 EnsureSpace ensure_space(this);
3472 XMMRegister isrc = XMMRegister::from_code(src.code());
3473 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
3474 emit(0x6E);
3475 emit_sse_operand(dst, src);
3476 }
3477
vmovq(XMMRegister dst,Operand src)3478 void Assembler::vmovq(XMMRegister dst, Operand src) {
3479 DCHECK(IsEnabled(AVX));
3480 EnsureSpace ensure_space(this);
3481 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
3482 emit(0x6E);
3483 emit_sse_operand(dst, src);
3484 }
3485
vmovq(Register dst,XMMRegister src)3486 void Assembler::vmovq(Register dst, XMMRegister src) {
3487 DCHECK(IsEnabled(AVX));
3488 EnsureSpace ensure_space(this);
3489 XMMRegister idst = XMMRegister::from_code(dst.code());
3490 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
3491 emit(0x7E);
3492 emit_sse_operand(src, dst);
3493 }
3494
vmovdqu(XMMRegister dst,Operand src)3495 void Assembler::vmovdqu(XMMRegister dst, Operand src) {
3496 DCHECK(IsEnabled(AVX));
3497 EnsureSpace ensure_space(this);
3498 emit_vex_prefix(dst, xmm0, src, kL128, kF3, k0F, kWIG);
3499 emit(0x6F);
3500 emit_sse_operand(dst, src);
3501 }
3502
vmovdqu(Operand dst,XMMRegister src)3503 void Assembler::vmovdqu(Operand dst, XMMRegister src) {
3504 DCHECK(IsEnabled(AVX));
3505 EnsureSpace ensure_space(this);
3506 emit_vex_prefix(src, xmm0, dst, kL128, kF3, k0F, kWIG);
3507 emit(0x7F);
3508 emit_sse_operand(src, dst);
3509 }
3510
vmovlps(XMMRegister dst,XMMRegister src1,Operand src2)3511 void Assembler::vmovlps(XMMRegister dst, XMMRegister src1, Operand src2) {
3512 DCHECK(IsEnabled(AVX));
3513 EnsureSpace ensure_space(this);
3514 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3515 emit(0x12);
3516 emit_sse_operand(dst, src2);
3517 }
3518
vmovlps(Operand dst,XMMRegister src)3519 void Assembler::vmovlps(Operand dst, XMMRegister src) {
3520 DCHECK(IsEnabled(AVX));
3521 EnsureSpace ensure_space(this);
3522 emit_vex_prefix(src, xmm0, dst, kL128, kNone, k0F, kWIG);
3523 emit(0x13);
3524 emit_sse_operand(src, dst);
3525 }
3526
vmovhps(XMMRegister dst,XMMRegister src1,Operand src2)3527 void Assembler::vmovhps(XMMRegister dst, XMMRegister src1, Operand src2) {
3528 DCHECK(IsEnabled(AVX));
3529 EnsureSpace ensure_space(this);
3530 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3531 emit(0x16);
3532 emit_sse_operand(dst, src2);
3533 }
3534
vmovhps(Operand dst,XMMRegister src)3535 void Assembler::vmovhps(Operand dst, XMMRegister src) {
3536 DCHECK(IsEnabled(AVX));
3537 EnsureSpace ensure_space(this);
3538 emit_vex_prefix(src, xmm0, dst, kL128, kNone, k0F, kWIG);
3539 emit(0x17);
3540 emit_sse_operand(src, dst);
3541 }
3542
vinstr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w)3543 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
3544 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
3545 VexW w) {
3546 DCHECK(IsEnabled(AVX));
3547 EnsureSpace ensure_space(this);
3548 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3549 emit(op);
3550 emit_sse_operand(dst, src2);
3551 }
3552
vinstr(byte op,XMMRegister dst,XMMRegister src1,Operand src2,SIMDPrefix pp,LeadingOpcode m,VexW w)3553 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
3554 SIMDPrefix pp, LeadingOpcode m, VexW w) {
3555 DCHECK(IsEnabled(AVX));
3556 EnsureSpace ensure_space(this);
3557 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3558 emit(op);
3559 emit_sse_operand(dst, src2);
3560 }
3561
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3562 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3563 XMMRegister src2) {
3564 DCHECK(IsEnabled(AVX));
3565 EnsureSpace ensure_space(this);
3566 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3567 emit(op);
3568 emit_sse_operand(dst, src2);
3569 }
3570
vps(byte op,XMMRegister dst,XMMRegister src1,Operand src2)3571 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3572 DCHECK(IsEnabled(AVX));
3573 EnsureSpace ensure_space(this);
3574 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3575 emit(op);
3576 emit_sse_operand(dst, src2);
3577 }
3578
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,byte imm8)3579 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3580 XMMRegister src2, byte imm8) {
3581 DCHECK(IsEnabled(AVX));
3582 EnsureSpace ensure_space(this);
3583 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3584 emit(op);
3585 emit_sse_operand(dst, src2);
3586 emit(imm8);
3587 }
3588
vpd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3589 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
3590 XMMRegister src2) {
3591 DCHECK(IsEnabled(AVX));
3592 EnsureSpace ensure_space(this);
3593 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
3594 emit(op);
3595 emit_sse_operand(dst, src2);
3596 }
3597
vpd(byte op,XMMRegister dst,XMMRegister src1,Operand src2)3598 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3599 DCHECK(IsEnabled(AVX));
3600 EnsureSpace ensure_space(this);
3601 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
3602 emit(op);
3603 emit_sse_operand(dst, src2);
3604 }
3605
vucomiss(XMMRegister dst,XMMRegister src)3606 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
3607 DCHECK(IsEnabled(AVX));
3608 EnsureSpace ensure_space(this);
3609 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
3610 emit(0x2E);
3611 emit_sse_operand(dst, src);
3612 }
3613
vucomiss(XMMRegister dst,Operand src)3614 void Assembler::vucomiss(XMMRegister dst, Operand src) {
3615 DCHECK(IsEnabled(AVX));
3616 EnsureSpace ensure_space(this);
3617 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
3618 emit(0x2E);
3619 emit_sse_operand(dst, src);
3620 }
3621
vpmovmskb(Register dst,XMMRegister src)3622 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3623 XMMRegister idst = XMMRegister::from_code(dst.code());
3624 DCHECK(IsEnabled(AVX));
3625 EnsureSpace ensure_space(this);
3626 emit_vex_prefix(idst, xmm0, src, kL128, k66, k0F, kWIG);
3627 emit(0xD7);
3628 emit_sse_operand(idst, src);
3629 }
3630
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3631 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
3632 XMMRegister src2) {
3633 DCHECK(IsEnabled(AVX));
3634 EnsureSpace ensure_space(this);
3635 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3636 emit(op);
3637 emit_sse_operand(dst, src2);
3638 }
3639
vss(byte op,XMMRegister dst,XMMRegister src1,Operand src2)3640 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
3641 DCHECK(IsEnabled(AVX));
3642 EnsureSpace ensure_space(this);
3643 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3644 emit(op);
3645 emit_sse_operand(dst, src2);
3646 }
3647
bmi1q(byte op,Register reg,Register vreg,Register rm)3648 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
3649 DCHECK(IsEnabled(BMI1));
3650 EnsureSpace ensure_space(this);
3651 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
3652 emit(op);
3653 emit_modrm(reg, rm);
3654 }
3655
bmi1q(byte op,Register reg,Register vreg,Operand rm)3656 void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
3657 DCHECK(IsEnabled(BMI1));
3658 EnsureSpace ensure_space(this);
3659 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
3660 emit(op);
3661 emit_operand(reg, rm);
3662 }
3663
bmi1l(byte op,Register reg,Register vreg,Register rm)3664 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
3665 DCHECK(IsEnabled(BMI1));
3666 EnsureSpace ensure_space(this);
3667 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
3668 emit(op);
3669 emit_modrm(reg, rm);
3670 }
3671
bmi1l(byte op,Register reg,Register vreg,Operand rm)3672 void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
3673 DCHECK(IsEnabled(BMI1));
3674 EnsureSpace ensure_space(this);
3675 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
3676 emit(op);
3677 emit_operand(reg, rm);
3678 }
3679
tzcntq(Register dst,Register src)3680 void Assembler::tzcntq(Register dst, Register src) {
3681 DCHECK(IsEnabled(BMI1));
3682 EnsureSpace ensure_space(this);
3683 emit(0xF3);
3684 emit_rex_64(dst, src);
3685 emit(0x0F);
3686 emit(0xBC);
3687 emit_modrm(dst, src);
3688 }
3689
tzcntq(Register dst,Operand src)3690 void Assembler::tzcntq(Register dst, Operand src) {
3691 DCHECK(IsEnabled(BMI1));
3692 EnsureSpace ensure_space(this);
3693 emit(0xF3);
3694 emit_rex_64(dst, src);
3695 emit(0x0F);
3696 emit(0xBC);
3697 emit_operand(dst, src);
3698 }
3699
tzcntl(Register dst,Register src)3700 void Assembler::tzcntl(Register dst, Register src) {
3701 DCHECK(IsEnabled(BMI1));
3702 EnsureSpace ensure_space(this);
3703 emit(0xF3);
3704 emit_optional_rex_32(dst, src);
3705 emit(0x0F);
3706 emit(0xBC);
3707 emit_modrm(dst, src);
3708 }
3709
tzcntl(Register dst,Operand src)3710 void Assembler::tzcntl(Register dst, Operand src) {
3711 DCHECK(IsEnabled(BMI1));
3712 EnsureSpace ensure_space(this);
3713 emit(0xF3);
3714 emit_optional_rex_32(dst, src);
3715 emit(0x0F);
3716 emit(0xBC);
3717 emit_operand(dst, src);
3718 }
3719
lzcntq(Register dst,Register src)3720 void Assembler::lzcntq(Register dst, Register src) {
3721 DCHECK(IsEnabled(LZCNT));
3722 EnsureSpace ensure_space(this);
3723 emit(0xF3);
3724 emit_rex_64(dst, src);
3725 emit(0x0F);
3726 emit(0xBD);
3727 emit_modrm(dst, src);
3728 }
3729
lzcntq(Register dst,Operand src)3730 void Assembler::lzcntq(Register dst, Operand src) {
3731 DCHECK(IsEnabled(LZCNT));
3732 EnsureSpace ensure_space(this);
3733 emit(0xF3);
3734 emit_rex_64(dst, src);
3735 emit(0x0F);
3736 emit(0xBD);
3737 emit_operand(dst, src);
3738 }
3739
lzcntl(Register dst,Register src)3740 void Assembler::lzcntl(Register dst, Register src) {
3741 DCHECK(IsEnabled(LZCNT));
3742 EnsureSpace ensure_space(this);
3743 emit(0xF3);
3744 emit_optional_rex_32(dst, src);
3745 emit(0x0F);
3746 emit(0xBD);
3747 emit_modrm(dst, src);
3748 }
3749
lzcntl(Register dst,Operand src)3750 void Assembler::lzcntl(Register dst, Operand src) {
3751 DCHECK(IsEnabled(LZCNT));
3752 EnsureSpace ensure_space(this);
3753 emit(0xF3);
3754 emit_optional_rex_32(dst, src);
3755 emit(0x0F);
3756 emit(0xBD);
3757 emit_operand(dst, src);
3758 }
3759
popcntq(Register dst,Register src)3760 void Assembler::popcntq(Register dst, Register src) {
3761 DCHECK(IsEnabled(POPCNT));
3762 EnsureSpace ensure_space(this);
3763 emit(0xF3);
3764 emit_rex_64(dst, src);
3765 emit(0x0F);
3766 emit(0xB8);
3767 emit_modrm(dst, src);
3768 }
3769
popcntq(Register dst,Operand src)3770 void Assembler::popcntq(Register dst, Operand src) {
3771 DCHECK(IsEnabled(POPCNT));
3772 EnsureSpace ensure_space(this);
3773 emit(0xF3);
3774 emit_rex_64(dst, src);
3775 emit(0x0F);
3776 emit(0xB8);
3777 emit_operand(dst, src);
3778 }
3779
popcntl(Register dst,Register src)3780 void Assembler::popcntl(Register dst, Register src) {
3781 DCHECK(IsEnabled(POPCNT));
3782 EnsureSpace ensure_space(this);
3783 emit(0xF3);
3784 emit_optional_rex_32(dst, src);
3785 emit(0x0F);
3786 emit(0xB8);
3787 emit_modrm(dst, src);
3788 }
3789
popcntl(Register dst,Operand src)3790 void Assembler::popcntl(Register dst, Operand src) {
3791 DCHECK(IsEnabled(POPCNT));
3792 EnsureSpace ensure_space(this);
3793 emit(0xF3);
3794 emit_optional_rex_32(dst, src);
3795 emit(0x0F);
3796 emit(0xB8);
3797 emit_operand(dst, src);
3798 }
3799
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)3800 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
3801 Register rm) {
3802 DCHECK(IsEnabled(BMI2));
3803 EnsureSpace ensure_space(this);
3804 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
3805 emit(op);
3806 emit_modrm(reg, rm);
3807 }
3808
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)3809 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
3810 Operand rm) {
3811 DCHECK(IsEnabled(BMI2));
3812 EnsureSpace ensure_space(this);
3813 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
3814 emit(op);
3815 emit_operand(reg, rm);
3816 }
3817
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)3818 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
3819 Register rm) {
3820 DCHECK(IsEnabled(BMI2));
3821 EnsureSpace ensure_space(this);
3822 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
3823 emit(op);
3824 emit_modrm(reg, rm);
3825 }
3826
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Operand rm)3827 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
3828 Operand rm) {
3829 DCHECK(IsEnabled(BMI2));
3830 EnsureSpace ensure_space(this);
3831 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
3832 emit(op);
3833 emit_operand(reg, rm);
3834 }
3835
rorxq(Register dst,Register src,byte imm8)3836 void Assembler::rorxq(Register dst, Register src, byte imm8) {
3837 DCHECK(IsEnabled(BMI2));
3838 DCHECK(is_uint8(imm8));
3839 Register vreg = Register::from_code(0); // VEX.vvvv unused
3840 EnsureSpace ensure_space(this);
3841 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
3842 emit(0xF0);
3843 emit_modrm(dst, src);
3844 emit(imm8);
3845 }
3846
rorxq(Register dst,Operand src,byte imm8)3847 void Assembler::rorxq(Register dst, Operand src, byte imm8) {
3848 DCHECK(IsEnabled(BMI2));
3849 DCHECK(is_uint8(imm8));
3850 Register vreg = Register::from_code(0); // VEX.vvvv unused
3851 EnsureSpace ensure_space(this);
3852 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
3853 emit(0xF0);
3854 emit_operand(dst, src);
3855 emit(imm8);
3856 }
3857
rorxl(Register dst,Register src,byte imm8)3858 void Assembler::rorxl(Register dst, Register src, byte imm8) {
3859 DCHECK(IsEnabled(BMI2));
3860 DCHECK(is_uint8(imm8));
3861 Register vreg = Register::from_code(0); // VEX.vvvv unused
3862 EnsureSpace ensure_space(this);
3863 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
3864 emit(0xF0);
3865 emit_modrm(dst, src);
3866 emit(imm8);
3867 }
3868
rorxl(Register dst,Operand src,byte imm8)3869 void Assembler::rorxl(Register dst, Operand src, byte imm8) {
3870 DCHECK(IsEnabled(BMI2));
3871 DCHECK(is_uint8(imm8));
3872 Register vreg = Register::from_code(0); // VEX.vvvv unused
3873 EnsureSpace ensure_space(this);
3874 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
3875 emit(0xF0);
3876 emit_operand(dst, src);
3877 emit(imm8);
3878 }
3879
pause()3880 void Assembler::pause() {
3881 emit(0xF3);
3882 emit(0x90);
3883 }
3884
movups(XMMRegister dst,XMMRegister src)3885 void Assembler::movups(XMMRegister dst, XMMRegister src) {
3886 EnsureSpace ensure_space(this);
3887 if (src.low_bits() == 4) {
3888 // Try to avoid an unnecessary SIB byte.
3889 emit_optional_rex_32(src, dst);
3890 emit(0x0F);
3891 emit(0x11);
3892 emit_sse_operand(src, dst);
3893 } else {
3894 emit_optional_rex_32(dst, src);
3895 emit(0x0F);
3896 emit(0x10);
3897 emit_sse_operand(dst, src);
3898 }
3899 }
3900
movups(XMMRegister dst,Operand src)3901 void Assembler::movups(XMMRegister dst, Operand src) {
3902 EnsureSpace ensure_space(this);
3903 emit_optional_rex_32(dst, src);
3904 emit(0x0F);
3905 emit(0x10);
3906 emit_sse_operand(dst, src);
3907 }
3908
movups(Operand dst,XMMRegister src)3909 void Assembler::movups(Operand dst, XMMRegister src) {
3910 EnsureSpace ensure_space(this);
3911 emit_optional_rex_32(src, dst);
3912 emit(0x0F);
3913 emit(0x11);
3914 emit_sse_operand(src, dst);
3915 }
3916
sse_instr(XMMRegister dst,XMMRegister src,byte escape,byte opcode)3917 void Assembler::sse_instr(XMMRegister dst, XMMRegister src, byte escape,
3918 byte opcode) {
3919 EnsureSpace ensure_space(this);
3920 emit_optional_rex_32(dst, src);
3921 emit(escape);
3922 emit(opcode);
3923 emit_sse_operand(dst, src);
3924 }
3925
sse_instr(XMMRegister dst,Operand src,byte escape,byte opcode)3926 void Assembler::sse_instr(XMMRegister dst, Operand src, byte escape,
3927 byte opcode) {
3928 EnsureSpace ensure_space(this);
3929 emit_optional_rex_32(dst, src);
3930 emit(escape);
3931 emit(opcode);
3932 emit_sse_operand(dst, src);
3933 }
3934
sse2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape,byte opcode)3935 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
3936 byte escape, byte opcode) {
3937 EnsureSpace ensure_space(this);
3938 emit(prefix);
3939 emit_optional_rex_32(dst, src);
3940 emit(escape);
3941 emit(opcode);
3942 emit_sse_operand(dst, src);
3943 }
3944
sse2_instr(XMMRegister dst,Operand src,byte prefix,byte escape,byte opcode)3945 void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
3946 byte escape, byte opcode) {
3947 EnsureSpace ensure_space(this);
3948 emit(prefix);
3949 emit_optional_rex_32(dst, src);
3950 emit(escape);
3951 emit(opcode);
3952 emit_sse_operand(dst, src);
3953 }
3954
ssse3_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)3955 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
3956 byte escape1, byte escape2, byte opcode) {
3957 DCHECK(IsEnabled(SSSE3));
3958 EnsureSpace ensure_space(this);
3959 emit(prefix);
3960 emit_optional_rex_32(dst, src);
3961 emit(escape1);
3962 emit(escape2);
3963 emit(opcode);
3964 emit_sse_operand(dst, src);
3965 }
3966
ssse3_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)3967 void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
3968 byte escape1, byte escape2, byte opcode) {
3969 DCHECK(IsEnabled(SSSE3));
3970 EnsureSpace ensure_space(this);
3971 emit(prefix);
3972 emit_optional_rex_32(dst, src);
3973 emit(escape1);
3974 emit(escape2);
3975 emit(opcode);
3976 emit_sse_operand(dst, src);
3977 }
3978
sse4_instr(XMMRegister dst,Register src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)3979 void Assembler::sse4_instr(XMMRegister dst, Register src, byte prefix,
3980 byte escape1, byte escape2, byte opcode,
3981 int8_t imm8) {
3982 DCHECK(is_uint8(imm8));
3983 DCHECK(IsEnabled(SSE4_1));
3984 EnsureSpace ensure_space(this);
3985 emit(prefix);
3986 emit_optional_rex_32(dst, src);
3987 emit(escape1);
3988 emit(escape2);
3989 emit(opcode);
3990 emit_sse_operand(dst, src);
3991 emit(imm8);
3992 }
3993
sse4_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)3994 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
3995 byte escape1, byte escape2, byte opcode) {
3996 DCHECK(IsEnabled(SSE4_1));
3997 EnsureSpace ensure_space(this);
3998 emit(prefix);
3999 emit_optional_rex_32(dst, src);
4000 emit(escape1);
4001 emit(escape2);
4002 emit(opcode);
4003 emit_sse_operand(dst, src);
4004 }
4005
sse4_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4006 void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
4007 byte escape1, byte escape2, byte opcode) {
4008 DCHECK(IsEnabled(SSE4_1));
4009 EnsureSpace ensure_space(this);
4010 emit(prefix);
4011 emit_optional_rex_32(dst, src);
4012 emit(escape1);
4013 emit(escape2);
4014 emit(opcode);
4015 emit_sse_operand(dst, src);
4016 }
4017
sse4_instr(Register dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)4018 void Assembler::sse4_instr(Register dst, XMMRegister src, byte prefix,
4019 byte escape1, byte escape2, byte opcode,
4020 int8_t imm8) {
4021 DCHECK(is_uint8(imm8));
4022 DCHECK(IsEnabled(SSE4_1));
4023 EnsureSpace ensure_space(this);
4024 emit(prefix);
4025 emit_optional_rex_32(src, dst);
4026 emit(escape1);
4027 emit(escape2);
4028 emit(opcode);
4029 emit_sse_operand(src, dst);
4030 emit(imm8);
4031 }
4032
sse4_instr(Operand dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode,int8_t imm8)4033 void Assembler::sse4_instr(Operand dst, XMMRegister src, byte prefix,
4034 byte escape1, byte escape2, byte opcode,
4035 int8_t imm8) {
4036 DCHECK(is_uint8(imm8));
4037 DCHECK(IsEnabled(SSE4_1));
4038 EnsureSpace ensure_space(this);
4039 emit(prefix);
4040 emit_optional_rex_32(src, dst);
4041 emit(escape1);
4042 emit(escape2);
4043 emit(opcode);
4044 emit_sse_operand(src, dst);
4045 emit(imm8);
4046 }
4047
sse4_2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4048 void Assembler::sse4_2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4049 byte escape1, byte escape2, byte opcode) {
4050 DCHECK(IsEnabled(SSE4_2));
4051 EnsureSpace ensure_space(this);
4052 emit(prefix);
4053 emit_optional_rex_32(dst, src);
4054 emit(escape1);
4055 emit(escape2);
4056 emit(opcode);
4057 emit_sse_operand(dst, src);
4058 }
4059
sse4_2_instr(XMMRegister dst,Operand src,byte prefix,byte escape1,byte escape2,byte opcode)4060 void Assembler::sse4_2_instr(XMMRegister dst, Operand src, byte prefix,
4061 byte escape1, byte escape2, byte opcode) {
4062 DCHECK(IsEnabled(SSE4_2));
4063 EnsureSpace ensure_space(this);
4064 emit(prefix);
4065 emit_optional_rex_32(dst, src);
4066 emit(escape1);
4067 emit(escape2);
4068 emit(opcode);
4069 emit_sse_operand(dst, src);
4070 }
4071
lddqu(XMMRegister dst,Operand src)4072 void Assembler::lddqu(XMMRegister dst, Operand src) {
4073 DCHECK(IsEnabled(SSE3));
4074 EnsureSpace ensure_space(this);
4075 emit(0xF2);
4076 emit_optional_rex_32(dst, src);
4077 emit(0x0F);
4078 emit(0xF0);
4079 emit_sse_operand(dst, src);
4080 }
4081
movddup(XMMRegister dst,XMMRegister src)4082 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
4083 DCHECK(IsEnabled(SSE3));
4084 EnsureSpace ensure_space(this);
4085 emit(0xF2);
4086 emit_optional_rex_32(dst, src);
4087 emit(0x0F);
4088 emit(0x12);
4089 emit_sse_operand(dst, src);
4090 }
4091
movddup(XMMRegister dst,Operand src)4092 void Assembler::movddup(XMMRegister dst, Operand src) {
4093 DCHECK(IsEnabled(SSE3));
4094 EnsureSpace ensure_space(this);
4095 emit(0xF2);
4096 emit_optional_rex_32(dst, src);
4097 emit(0x0F);
4098 emit(0x12);
4099 emit_sse_operand(dst, src);
4100 }
4101
psrldq(XMMRegister dst,uint8_t shift)4102 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4103 EnsureSpace ensure_space(this);
4104 emit(0x66);
4105 emit_optional_rex_32(dst);
4106 emit(0x0F);
4107 emit(0x73);
4108 emit_sse_operand(dst);
4109 emit(shift);
4110 }
4111
pshufhw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4112 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4113 EnsureSpace ensure_space(this);
4114 emit(0xF3);
4115 emit_optional_rex_32(dst, src);
4116 emit(0x0F);
4117 emit(0x70);
4118 emit_sse_operand(dst, src);
4119 emit(shuffle);
4120 }
4121
pshufhw(XMMRegister dst,Operand src,uint8_t shuffle)4122 void Assembler::pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
4123 EnsureSpace ensure_space(this);
4124 emit(0xF3);
4125 emit_optional_rex_32(dst, src);
4126 emit(0x0F);
4127 emit(0x70);
4128 emit_sse_operand(dst, src);
4129 emit(shuffle);
4130 }
4131
pshuflw(XMMRegister dst,XMMRegister src,uint8_t shuffle)4132 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4133 EnsureSpace ensure_space(this);
4134 emit(0xF2);
4135 emit_optional_rex_32(dst, src);
4136 emit(0x0F);
4137 emit(0x70);
4138 emit_sse_operand(dst, src);
4139 emit(shuffle);
4140 }
4141
pshuflw(XMMRegister dst,Operand src,uint8_t shuffle)4142 void Assembler::pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
4143 EnsureSpace ensure_space(this);
4144 emit(0xF2);
4145 emit_optional_rex_32(dst, src);
4146 emit(0x0F);
4147 emit(0x70);
4148 emit_sse_operand(dst, src);
4149 emit(shuffle);
4150 }
4151
pshufd(XMMRegister dst,XMMRegister src,uint8_t shuffle)4152 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4153 EnsureSpace ensure_space(this);
4154 emit(0x66);
4155 emit_optional_rex_32(dst, src);
4156 emit(0x0F);
4157 emit(0x70);
4158 emit_sse_operand(dst, src);
4159 emit(shuffle);
4160 }
4161
pshufd(XMMRegister dst,Operand src,uint8_t shuffle)4162 void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
4163 EnsureSpace ensure_space(this);
4164 emit(0x66);
4165 emit_optional_rex_32(dst, src);
4166 emit(0x0F);
4167 emit(0x70);
4168 emit_sse_operand(dst, src);
4169 emit(shuffle);
4170 }
4171
emit_sse_operand(XMMRegister reg,Operand adr)4172 void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
4173 Register ireg = Register::from_code(reg.code());
4174 emit_operand(ireg, adr);
4175 }
4176
emit_sse_operand(Register reg,Operand adr)4177 void Assembler::emit_sse_operand(Register reg, Operand adr) {
4178 emit_operand(reg, adr);
4179 }
4180
emit_sse_operand(XMMRegister dst,XMMRegister src)4181 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4182 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4183 }
4184
emit_sse_operand(XMMRegister dst,Register src)4185 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4186 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4187 }
4188
emit_sse_operand(Register dst,XMMRegister src)4189 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4190 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4191 }
4192
emit_sse_operand(XMMRegister dst)4193 void Assembler::emit_sse_operand(XMMRegister dst) {
4194 emit(0xD8 | dst.low_bits());
4195 }
4196
db(uint8_t data)4197 void Assembler::db(uint8_t data) {
4198 EnsureSpace ensure_space(this);
4199 emit(data);
4200 }
4201
dd(uint32_t data)4202 void Assembler::dd(uint32_t data) {
4203 EnsureSpace ensure_space(this);
4204 emitl(data);
4205 }
4206
dq(uint64_t data)4207 void Assembler::dq(uint64_t data) {
4208 EnsureSpace ensure_space(this);
4209 emitq(data);
4210 }
4211
dq(Label * label)4212 void Assembler::dq(Label* label) {
4213 EnsureSpace ensure_space(this);
4214 if (label->is_bound()) {
4215 internal_reference_positions_.push_back(pc_offset());
4216 emit(Immediate64(reinterpret_cast<Address>(buffer_start_) + label->pos(),
4217 RelocInfo::INTERNAL_REFERENCE));
4218 } else {
4219 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4220 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4221 if (label->is_linked()) {
4222 emitl(label->pos());
4223 label->link_to(pc_offset() - sizeof(int32_t));
4224 } else {
4225 DCHECK(label->is_unused());
4226 int32_t current = pc_offset();
4227 emitl(current);
4228 label->link_to(current);
4229 }
4230 }
4231 }
4232
4233 // Relocation information implementations.
4234
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4235 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4236 if (!ShouldRecordRelocInfo(rmode)) return;
4237 RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
4238 reloc_info_writer.Write(&rinfo);
4239 }
4240
4241 const int RelocInfo::kApplyMask =
4242 RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
4243 RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
4244 RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
4245 RelocInfo::ModeMask(RelocInfo::WASM_CALL);
4246
IsCodedSpecially()4247 bool RelocInfo::IsCodedSpecially() {
4248 // The deserializer needs to know whether a pointer is specially coded. Being
4249 // specially coded on x64 means that it is a relative 32 bit address, as used
4250 // by branch instructions.
4251 return (1 << rmode_) & kApplyMask;
4252 }
4253
IsInConstantPool()4254 bool RelocInfo::IsInConstantPool() { return false; }
4255
4256 } // namespace internal
4257 } // namespace v8
4258
4259 #endif // V8_TARGET_ARCH_X64
4260