1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #if V8_TARGET_ARCH_IA32
6
7 #include "src/base/bits.h"
8 #include "src/base/division-by-constant.h"
9 #include "src/base/utils/random-number-generator.h"
10 #include "src/codegen/callable.h"
11 #include "src/codegen/code-factory.h"
12 #include "src/codegen/external-reference-table.h"
13 #include "src/codegen/ia32/assembler-ia32-inl.h"
14 #include "src/codegen/macro-assembler.h"
15 #include "src/debug/debug.h"
16 #include "src/execution/frame-constants.h"
17 #include "src/execution/frames-inl.h"
18 #include "src/heap/memory-chunk.h"
19 #include "src/init/bootstrapper.h"
20 #include "src/logging/counters.h"
21 #include "src/runtime/runtime.h"
22 #include "src/snapshot/embedded/embedded-data.h"
23 #include "src/snapshot/snapshot.h"
24
25 // Satisfy cpplint check, but don't include platform-specific header. It is
26 // included recursively via macro-assembler.h.
27 #if 0
28 #include "src/codegen/ia32/macro-assembler-ia32.h"
29 #endif
30
31 namespace v8 {
32 namespace internal {
33
GetArgumentOperand(int index) const34 Operand StackArgumentsAccessor::GetArgumentOperand(int index) const {
35 DCHECK_GE(index, 0);
36 // arg[0] = esp + kPCOnStackSize;
37 // arg[i] = arg[0] + i * kSystemPointerSize;
38 return Operand(esp, kPCOnStackSize + index * kSystemPointerSize);
39 }
40
41 // -------------------------------------------------------------------------
42 // MacroAssembler implementation.
43
InitializeRootRegister()44 void TurboAssembler::InitializeRootRegister() {
45 ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
46 Move(kRootRegister, Immediate(isolate_root));
47 }
48
LoadRoot(Register destination,RootIndex index)49 void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
50 if (root_array_available()) {
51 mov(destination,
52 Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
53 return;
54 }
55
56 if (RootsTable::IsImmortalImmovable(index)) {
57 Handle<Object> object = isolate()->root_handle(index);
58 if (object->IsSmi()) {
59 mov(destination, Immediate(Smi::cast(*object)));
60 return;
61 } else {
62 DCHECK(object->IsHeapObject());
63 mov(destination, Handle<HeapObject>::cast(object));
64 return;
65 }
66 }
67
68 ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
69 lea(destination,
70 Operand(isolate_root.address(), RelocInfo::EXTERNAL_REFERENCE));
71 mov(destination, Operand(destination, RootRegisterOffsetForRootIndex(index)));
72 }
73
CompareRoot(Register with,Register scratch,RootIndex index)74 void TurboAssembler::CompareRoot(Register with, Register scratch,
75 RootIndex index) {
76 if (root_array_available()) {
77 CompareRoot(with, index);
78 } else {
79 ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
80 lea(scratch,
81 Operand(isolate_root.address(), RelocInfo::EXTERNAL_REFERENCE));
82 cmp(with, Operand(scratch, RootRegisterOffsetForRootIndex(index)));
83 }
84 }
85
CompareRoot(Register with,RootIndex index)86 void TurboAssembler::CompareRoot(Register with, RootIndex index) {
87 if (root_array_available()) {
88 cmp(with, Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
89 return;
90 }
91
92 DCHECK(RootsTable::IsImmortalImmovable(index));
93 Handle<Object> object = isolate()->root_handle(index);
94 if (object->IsHeapObject()) {
95 cmp(with, Handle<HeapObject>::cast(object));
96 } else {
97 cmp(with, Immediate(Smi::cast(*object)));
98 }
99 }
100
PushRoot(RootIndex index)101 void MacroAssembler::PushRoot(RootIndex index) {
102 if (root_array_available()) {
103 DCHECK(RootsTable::IsImmortalImmovable(index));
104 push(Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
105 return;
106 }
107
108 // TODO(v8:6666): Add a scratch register or remove all uses.
109 DCHECK(RootsTable::IsImmortalImmovable(index));
110 Handle<Object> object = isolate()->root_handle(index);
111 if (object->IsHeapObject()) {
112 Push(Handle<HeapObject>::cast(object));
113 } else {
114 Push(Smi::cast(*object));
115 }
116 }
117
JumpIfIsInRange(Register value,unsigned lower_limit,unsigned higher_limit,Register scratch,Label * on_in_range,Label::Distance near_jump)118 void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
119 unsigned higher_limit, Register scratch,
120 Label* on_in_range,
121 Label::Distance near_jump) {
122 if (lower_limit != 0) {
123 lea(scratch, Operand(value, 0u - lower_limit));
124 cmp(scratch, Immediate(higher_limit - lower_limit));
125 } else {
126 cmp(value, Immediate(higher_limit));
127 }
128 j(below_equal, on_in_range, near_jump);
129 }
130
PushArray(Register array,Register size,Register scratch,PushArrayOrder order)131 void TurboAssembler::PushArray(Register array, Register size, Register scratch,
132 PushArrayOrder order) {
133 DCHECK(!AreAliased(array, size, scratch));
134 Register counter = scratch;
135 Label loop, entry;
136 if (order == PushArrayOrder::kReverse) {
137 mov(counter, 0);
138 jmp(&entry);
139 bind(&loop);
140 Push(Operand(array, counter, times_system_pointer_size, 0));
141 inc(counter);
142 bind(&entry);
143 cmp(counter, size);
144 j(less, &loop, Label::kNear);
145 } else {
146 mov(counter, size);
147 jmp(&entry);
148 bind(&loop);
149 Push(Operand(array, counter, times_system_pointer_size, 0));
150 bind(&entry);
151 dec(counter);
152 j(greater_equal, &loop, Label::kNear);
153 }
154 }
155
ExternalReferenceAsOperand(ExternalReference reference,Register scratch)156 Operand TurboAssembler::ExternalReferenceAsOperand(ExternalReference reference,
157 Register scratch) {
158 // TODO(jgruber): Add support for enable_root_array_delta_access.
159 if (root_array_available() && options().isolate_independent_code) {
160 if (IsAddressableThroughRootRegister(isolate(), reference)) {
161 // Some external references can be efficiently loaded as an offset from
162 // kRootRegister.
163 intptr_t offset =
164 RootRegisterOffsetForExternalReference(isolate(), reference);
165 return Operand(kRootRegister, offset);
166 } else {
167 // Otherwise, do a memory load from the external reference table.
168 mov(scratch, Operand(kRootRegister,
169 RootRegisterOffsetForExternalReferenceTableEntry(
170 isolate(), reference)));
171 return Operand(scratch, 0);
172 }
173 }
174 Move(scratch, Immediate(reference));
175 return Operand(scratch, 0);
176 }
177
178 // TODO(v8:6666): If possible, refactor into a platform-independent function in
179 // TurboAssembler.
ExternalReferenceAddressAsOperand(ExternalReference reference)180 Operand TurboAssembler::ExternalReferenceAddressAsOperand(
181 ExternalReference reference) {
182 DCHECK(root_array_available());
183 DCHECK(options().isolate_independent_code);
184 return Operand(
185 kRootRegister,
186 RootRegisterOffsetForExternalReferenceTableEntry(isolate(), reference));
187 }
188
189 // TODO(v8:6666): If possible, refactor into a platform-independent function in
190 // TurboAssembler.
HeapObjectAsOperand(Handle<HeapObject> object)191 Operand TurboAssembler::HeapObjectAsOperand(Handle<HeapObject> object) {
192 DCHECK(root_array_available());
193
194 int builtin_index;
195 RootIndex root_index;
196 if (isolate()->roots_table().IsRootHandle(object, &root_index)) {
197 return Operand(kRootRegister, RootRegisterOffsetForRootIndex(root_index));
198 } else if (isolate()->builtins()->IsBuiltinHandle(object, &builtin_index)) {
199 return Operand(kRootRegister,
200 RootRegisterOffsetForBuiltinIndex(builtin_index));
201 } else if (object.is_identical_to(code_object_) &&
202 Builtins::IsBuiltinId(maybe_builtin_index_)) {
203 return Operand(kRootRegister,
204 RootRegisterOffsetForBuiltinIndex(maybe_builtin_index_));
205 } else {
206 // Objects in the constants table need an additional indirection, which
207 // cannot be represented as a single Operand.
208 UNREACHABLE();
209 }
210 }
211
LoadFromConstantsTable(Register destination,int constant_index)212 void TurboAssembler::LoadFromConstantsTable(Register destination,
213 int constant_index) {
214 DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
215 LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
216 mov(destination,
217 FieldOperand(destination, FixedArray::OffsetOfElementAt(constant_index)));
218 }
219
LoadRootRegisterOffset(Register destination,intptr_t offset)220 void TurboAssembler::LoadRootRegisterOffset(Register destination,
221 intptr_t offset) {
222 DCHECK(is_int32(offset));
223 DCHECK(root_array_available());
224 if (offset == 0) {
225 mov(destination, kRootRegister);
226 } else {
227 lea(destination, Operand(kRootRegister, static_cast<int32_t>(offset)));
228 }
229 }
230
LoadRootRelative(Register destination,int32_t offset)231 void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
232 DCHECK(root_array_available());
233 mov(destination, Operand(kRootRegister, offset));
234 }
235
LoadAddress(Register destination,ExternalReference source)236 void TurboAssembler::LoadAddress(Register destination,
237 ExternalReference source) {
238 // TODO(jgruber): Add support for enable_root_array_delta_access.
239 if (root_array_available() && options().isolate_independent_code) {
240 IndirectLoadExternalReference(destination, source);
241 return;
242 }
243 mov(destination, Immediate(source));
244 }
245
246 static constexpr Register saved_regs[] = {eax, ecx, edx};
247
248 static constexpr int kNumberOfSavedRegs = sizeof(saved_regs) / sizeof(Register);
249
RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,Register exclusion1,Register exclusion2,Register exclusion3) const250 int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
251 Register exclusion1,
252 Register exclusion2,
253 Register exclusion3) const {
254 int bytes = 0;
255 for (int i = 0; i < kNumberOfSavedRegs; i++) {
256 Register reg = saved_regs[i];
257 if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
258 bytes += kSystemPointerSize;
259 }
260 }
261
262 if (fp_mode == kSaveFPRegs) {
263 // Count all XMM registers except XMM0.
264 bytes += kDoubleSize * (XMMRegister::kNumRegisters - 1);
265 }
266
267 return bytes;
268 }
269
PushCallerSaved(SaveFPRegsMode fp_mode,Register exclusion1,Register exclusion2,Register exclusion3)270 int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
271 Register exclusion2, Register exclusion3) {
272 // We don't allow a GC during a store buffer overflow so there is no need to
273 // store the registers in any particular way, but we do have to store and
274 // restore them.
275 int bytes = 0;
276 for (int i = 0; i < kNumberOfSavedRegs; i++) {
277 Register reg = saved_regs[i];
278 if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
279 push(reg);
280 bytes += kSystemPointerSize;
281 }
282 }
283
284 if (fp_mode == kSaveFPRegs) {
285 // Save all XMM registers except XMM0.
286 int delta = kDoubleSize * (XMMRegister::kNumRegisters - 1);
287 AllocateStackSpace(delta);
288 for (int i = XMMRegister::kNumRegisters - 1; i > 0; i--) {
289 XMMRegister reg = XMMRegister::from_code(i);
290 movsd(Operand(esp, (i - 1) * kDoubleSize), reg);
291 }
292 bytes += delta;
293 }
294
295 return bytes;
296 }
297
PopCallerSaved(SaveFPRegsMode fp_mode,Register exclusion1,Register exclusion2,Register exclusion3)298 int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
299 Register exclusion2, Register exclusion3) {
300 int bytes = 0;
301 if (fp_mode == kSaveFPRegs) {
302 // Restore all XMM registers except XMM0.
303 int delta = kDoubleSize * (XMMRegister::kNumRegisters - 1);
304 for (int i = XMMRegister::kNumRegisters - 1; i > 0; i--) {
305 XMMRegister reg = XMMRegister::from_code(i);
306 movsd(reg, Operand(esp, (i - 1) * kDoubleSize));
307 }
308 add(esp, Immediate(delta));
309 bytes += delta;
310 }
311
312 for (int i = kNumberOfSavedRegs - 1; i >= 0; i--) {
313 Register reg = saved_regs[i];
314 if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
315 pop(reg);
316 bytes += kSystemPointerSize;
317 }
318 }
319
320 return bytes;
321 }
322
RecordWriteField(Register object,int offset,Register value,Register dst,SaveFPRegsMode save_fp,RememberedSetAction remembered_set_action,SmiCheck smi_check)323 void MacroAssembler::RecordWriteField(Register object, int offset,
324 Register value, Register dst,
325 SaveFPRegsMode save_fp,
326 RememberedSetAction remembered_set_action,
327 SmiCheck smi_check) {
328 // First, check if a write barrier is even needed. The tests below
329 // catch stores of Smis.
330 Label done;
331
332 // Skip barrier if writing a smi.
333 if (smi_check == INLINE_SMI_CHECK) {
334 JumpIfSmi(value, &done);
335 }
336
337 // Although the object register is tagged, the offset is relative to the start
338 // of the object, so so offset must be a multiple of kTaggedSize.
339 DCHECK(IsAligned(offset, kTaggedSize));
340
341 lea(dst, FieldOperand(object, offset));
342 if (emit_debug_code()) {
343 Label ok;
344 test_b(dst, Immediate(kTaggedSize - 1));
345 j(zero, &ok, Label::kNear);
346 int3();
347 bind(&ok);
348 }
349
350 RecordWrite(object, dst, value, save_fp, remembered_set_action,
351 OMIT_SMI_CHECK);
352
353 bind(&done);
354
355 // Clobber clobbered input registers when running with the debug-code flag
356 // turned on to provoke errors.
357 if (emit_debug_code()) {
358 mov(value, Immediate(bit_cast<int32_t>(kZapValue)));
359 mov(dst, Immediate(bit_cast<int32_t>(kZapValue)));
360 }
361 }
362
SaveRegisters(RegList registers)363 void TurboAssembler::SaveRegisters(RegList registers) {
364 DCHECK_GT(NumRegs(registers), 0);
365 for (int i = 0; i < Register::kNumRegisters; ++i) {
366 if ((registers >> i) & 1u) {
367 push(Register::from_code(i));
368 }
369 }
370 }
371
RestoreRegisters(RegList registers)372 void TurboAssembler::RestoreRegisters(RegList registers) {
373 DCHECK_GT(NumRegs(registers), 0);
374 for (int i = Register::kNumRegisters - 1; i >= 0; --i) {
375 if ((registers >> i) & 1u) {
376 pop(Register::from_code(i));
377 }
378 }
379 }
380
CallEphemeronKeyBarrier(Register object,Register address,SaveFPRegsMode fp_mode)381 void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
382 SaveFPRegsMode fp_mode) {
383 EphemeronKeyBarrierDescriptor descriptor;
384 RegList registers = descriptor.allocatable_registers();
385
386 SaveRegisters(registers);
387
388 Register object_parameter(
389 descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
390 Register slot_parameter(descriptor.GetRegisterParameter(
391 EphemeronKeyBarrierDescriptor::kSlotAddress));
392 Register fp_mode_parameter(
393 descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
394
395 push(object);
396 push(address);
397
398 pop(slot_parameter);
399 pop(object_parameter);
400
401 Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
402 Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
403 RelocInfo::CODE_TARGET);
404
405 RestoreRegisters(registers);
406 }
407
CallRecordWriteStub(Register object,Register address,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode)408 void TurboAssembler::CallRecordWriteStub(
409 Register object, Register address,
410 RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
411 CallRecordWriteStub(
412 object, address, remembered_set_action, fp_mode,
413 isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
414 kNullAddress);
415 }
416
CallRecordWriteStub(Register object,Register address,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode,Address wasm_target)417 void TurboAssembler::CallRecordWriteStub(
418 Register object, Register address,
419 RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
420 Address wasm_target) {
421 CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
422 Handle<Code>::null(), wasm_target);
423 }
424
CallRecordWriteStub(Register object,Register address,RememberedSetAction remembered_set_action,SaveFPRegsMode fp_mode,Handle<Code> code_target,Address wasm_target)425 void TurboAssembler::CallRecordWriteStub(
426 Register object, Register address,
427 RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
428 Handle<Code> code_target, Address wasm_target) {
429 DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
430 // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
431 // i.e. always emit remember set and save FP registers in RecordWriteStub. If
432 // large performance regression is observed, we should use these values to
433 // avoid unnecessary work.
434
435 RecordWriteDescriptor descriptor;
436 RegList registers = descriptor.allocatable_registers();
437
438 SaveRegisters(registers);
439
440 Register object_parameter(
441 descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
442 Register slot_parameter(
443 descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
444 Register remembered_set_parameter(
445 descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
446 Register fp_mode_parameter(
447 descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
448
449 push(object);
450 push(address);
451
452 pop(slot_parameter);
453 pop(object_parameter);
454
455 Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
456 Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
457 if (code_target.is_null()) {
458 // Use {wasm_call} for direct Wasm call within a module.
459 wasm_call(wasm_target, RelocInfo::WASM_STUB_CALL);
460 } else {
461 Call(code_target, RelocInfo::CODE_TARGET);
462 }
463
464 RestoreRegisters(registers);
465 }
466
RecordWrite(Register object,Register address,Register value,SaveFPRegsMode fp_mode,RememberedSetAction remembered_set_action,SmiCheck smi_check)467 void MacroAssembler::RecordWrite(Register object, Register address,
468 Register value, SaveFPRegsMode fp_mode,
469 RememberedSetAction remembered_set_action,
470 SmiCheck smi_check) {
471 DCHECK(object != value);
472 DCHECK(object != address);
473 DCHECK(value != address);
474 AssertNotSmi(object);
475
476 if ((remembered_set_action == OMIT_REMEMBERED_SET &&
477 !FLAG_incremental_marking) ||
478 FLAG_disable_write_barriers) {
479 return;
480 }
481
482 if (emit_debug_code()) {
483 Label ok;
484 cmp(value, Operand(address, 0));
485 j(equal, &ok, Label::kNear);
486 int3();
487 bind(&ok);
488 }
489
490 // First, check if a write barrier is even needed. The tests below
491 // catch stores of Smis and stores into young gen.
492 Label done;
493
494 if (smi_check == INLINE_SMI_CHECK) {
495 // Skip barrier if writing a smi.
496 JumpIfSmi(value, &done, Label::kNear);
497 }
498
499 CheckPageFlag(value,
500 value, // Used as scratch.
501 MemoryChunk::kPointersToHereAreInterestingMask, zero, &done,
502 Label::kNear);
503 CheckPageFlag(object,
504 value, // Used as scratch.
505 MemoryChunk::kPointersFromHereAreInterestingMask, zero, &done,
506 Label::kNear);
507
508 CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
509
510 bind(&done);
511
512 // Clobber clobbered registers when running with the debug-code flag
513 // turned on to provoke errors.
514 if (emit_debug_code()) {
515 mov(address, Immediate(bit_cast<int32_t>(kZapValue)));
516 mov(value, Immediate(bit_cast<int32_t>(kZapValue)));
517 }
518 }
519
MaybeDropFrames()520 void MacroAssembler::MaybeDropFrames() {
521 // Check whether we need to drop frames to restart a function on the stack.
522 Label dont_drop;
523 ExternalReference restart_fp =
524 ExternalReference::debug_restart_fp_address(isolate());
525 mov(eax, ExternalReferenceAsOperand(restart_fp, eax));
526 test(eax, eax);
527 j(zero, &dont_drop, Label::kNear);
528
529 Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET);
530 bind(&dont_drop);
531 }
532
Cvtsi2ss(XMMRegister dst,Operand src)533 void TurboAssembler::Cvtsi2ss(XMMRegister dst, Operand src) {
534 xorps(dst, dst);
535 cvtsi2ss(dst, src);
536 }
537
Cvtsi2sd(XMMRegister dst,Operand src)538 void TurboAssembler::Cvtsi2sd(XMMRegister dst, Operand src) {
539 xorpd(dst, dst);
540 cvtsi2sd(dst, src);
541 }
542
Cvtui2ss(XMMRegister dst,Operand src,Register tmp)543 void TurboAssembler::Cvtui2ss(XMMRegister dst, Operand src, Register tmp) {
544 Label done;
545 Register src_reg = src.is_reg_only() ? src.reg() : tmp;
546 if (src_reg == tmp) mov(tmp, src);
547 cvtsi2ss(dst, src_reg);
548 test(src_reg, src_reg);
549 j(positive, &done, Label::kNear);
550
551 // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
552 if (src_reg != tmp) mov(tmp, src_reg);
553 shr(tmp, 1);
554 // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
555 Label msb_not_set;
556 j(not_carry, &msb_not_set, Label::kNear);
557 or_(tmp, Immediate(1));
558 bind(&msb_not_set);
559 cvtsi2ss(dst, tmp);
560 addss(dst, dst);
561 bind(&done);
562 }
563
Cvttss2ui(Register dst,Operand src,XMMRegister tmp)564 void TurboAssembler::Cvttss2ui(Register dst, Operand src, XMMRegister tmp) {
565 Label done;
566 cvttss2si(dst, src);
567 test(dst, dst);
568 j(positive, &done);
569 Move(tmp, static_cast<float>(INT32_MIN));
570 addss(tmp, src);
571 cvttss2si(dst, tmp);
572 or_(dst, Immediate(0x80000000));
573 bind(&done);
574 }
575
Cvtui2sd(XMMRegister dst,Operand src,Register scratch)576 void TurboAssembler::Cvtui2sd(XMMRegister dst, Operand src, Register scratch) {
577 Label done;
578 cmp(src, Immediate(0));
579 ExternalReference uint32_bias = ExternalReference::address_of_uint32_bias();
580 Cvtsi2sd(dst, src);
581 j(not_sign, &done, Label::kNear);
582 addsd(dst, ExternalReferenceAsOperand(uint32_bias, scratch));
583 bind(&done);
584 }
585
Cvttsd2ui(Register dst,Operand src,XMMRegister tmp)586 void TurboAssembler::Cvttsd2ui(Register dst, Operand src, XMMRegister tmp) {
587 Move(tmp, -2147483648.0);
588 addsd(tmp, src);
589 cvttsd2si(dst, tmp);
590 add(dst, Immediate(0x80000000));
591 }
592
Roundps(XMMRegister dst,XMMRegister src,RoundingMode mode)593 void TurboAssembler::Roundps(XMMRegister dst, XMMRegister src,
594 RoundingMode mode) {
595 if (CpuFeatures::IsSupported(AVX)) {
596 CpuFeatureScope scope(this, AVX);
597 vroundps(dst, src, mode);
598 } else {
599 CpuFeatureScope scope(this, SSE4_1);
600 roundps(dst, src, mode);
601 }
602 }
603
Roundpd(XMMRegister dst,XMMRegister src,RoundingMode mode)604 void TurboAssembler::Roundpd(XMMRegister dst, XMMRegister src,
605 RoundingMode mode) {
606 if (CpuFeatures::IsSupported(AVX)) {
607 CpuFeatureScope scope(this, AVX);
608 vroundpd(dst, src, mode);
609 } else {
610 CpuFeatureScope scope(this, SSE4_1);
611 roundpd(dst, src, mode);
612 }
613 }
614
ShlPair(Register high,Register low,uint8_t shift)615 void TurboAssembler::ShlPair(Register high, Register low, uint8_t shift) {
616 DCHECK_GE(63, shift);
617 if (shift >= 32) {
618 mov(high, low);
619 if (shift != 32) shl(high, shift - 32);
620 xor_(low, low);
621 } else {
622 shld(high, low, shift);
623 shl(low, shift);
624 }
625 }
626
ShlPair_cl(Register high,Register low)627 void TurboAssembler::ShlPair_cl(Register high, Register low) {
628 shld_cl(high, low);
629 shl_cl(low);
630 Label done;
631 test(ecx, Immediate(0x20));
632 j(equal, &done, Label::kNear);
633 mov(high, low);
634 xor_(low, low);
635 bind(&done);
636 }
637
ShrPair(Register high,Register low,uint8_t shift)638 void TurboAssembler::ShrPair(Register high, Register low, uint8_t shift) {
639 DCHECK_GE(63, shift);
640 if (shift >= 32) {
641 mov(low, high);
642 if (shift != 32) shr(low, shift - 32);
643 xor_(high, high);
644 } else {
645 shrd(low, high, shift);
646 shr(high, shift);
647 }
648 }
649
ShrPair_cl(Register high,Register low)650 void TurboAssembler::ShrPair_cl(Register high, Register low) {
651 shrd_cl(low, high);
652 shr_cl(high);
653 Label done;
654 test(ecx, Immediate(0x20));
655 j(equal, &done, Label::kNear);
656 mov(low, high);
657 xor_(high, high);
658 bind(&done);
659 }
660
SarPair(Register high,Register low,uint8_t shift)661 void TurboAssembler::SarPair(Register high, Register low, uint8_t shift) {
662 DCHECK_GE(63, shift);
663 if (shift >= 32) {
664 mov(low, high);
665 if (shift != 32) sar(low, shift - 32);
666 sar(high, 31);
667 } else {
668 shrd(low, high, shift);
669 sar(high, shift);
670 }
671 }
672
SarPair_cl(Register high,Register low)673 void TurboAssembler::SarPair_cl(Register high, Register low) {
674 shrd_cl(low, high);
675 sar_cl(high);
676 Label done;
677 test(ecx, Immediate(0x20));
678 j(equal, &done, Label::kNear);
679 mov(low, high);
680 sar(high, 31);
681 bind(&done);
682 }
683
LoadMap(Register destination,Register object)684 void TurboAssembler::LoadMap(Register destination, Register object) {
685 mov(destination, FieldOperand(object, HeapObject::kMapOffset));
686 }
687
CmpObjectType(Register heap_object,InstanceType type,Register map)688 void MacroAssembler::CmpObjectType(Register heap_object, InstanceType type,
689 Register map) {
690 LoadMap(map, heap_object);
691 CmpInstanceType(map, type);
692 }
693
CmpInstanceType(Register map,InstanceType type)694 void MacroAssembler::CmpInstanceType(Register map, InstanceType type) {
695 cmpw(FieldOperand(map, Map::kInstanceTypeOffset), Immediate(type));
696 }
697
AssertSmi(Register object)698 void MacroAssembler::AssertSmi(Register object) {
699 if (emit_debug_code()) {
700 test(object, Immediate(kSmiTagMask));
701 Check(equal, AbortReason::kOperandIsNotASmi);
702 }
703 }
704
AssertConstructor(Register object)705 void MacroAssembler::AssertConstructor(Register object) {
706 if (emit_debug_code()) {
707 test(object, Immediate(kSmiTagMask));
708 Check(not_equal, AbortReason::kOperandIsASmiAndNotAConstructor);
709 Push(object);
710 LoadMap(object, object);
711 test_b(FieldOperand(object, Map::kBitFieldOffset),
712 Immediate(Map::Bits1::IsConstructorBit::kMask));
713 Pop(object);
714 Check(not_zero, AbortReason::kOperandIsNotAConstructor);
715 }
716 }
717
AssertFunction(Register object)718 void MacroAssembler::AssertFunction(Register object) {
719 if (emit_debug_code()) {
720 test(object, Immediate(kSmiTagMask));
721 Check(not_equal, AbortReason::kOperandIsASmiAndNotAFunction);
722 Push(object);
723 CmpObjectType(object, JS_FUNCTION_TYPE, object);
724 Pop(object);
725 Check(equal, AbortReason::kOperandIsNotAFunction);
726 }
727 }
728
AssertBoundFunction(Register object)729 void MacroAssembler::AssertBoundFunction(Register object) {
730 if (emit_debug_code()) {
731 test(object, Immediate(kSmiTagMask));
732 Check(not_equal, AbortReason::kOperandIsASmiAndNotABoundFunction);
733 Push(object);
734 CmpObjectType(object, JS_BOUND_FUNCTION_TYPE, object);
735 Pop(object);
736 Check(equal, AbortReason::kOperandIsNotABoundFunction);
737 }
738 }
739
AssertGeneratorObject(Register object)740 void MacroAssembler::AssertGeneratorObject(Register object) {
741 if (!emit_debug_code()) return;
742
743 test(object, Immediate(kSmiTagMask));
744 Check(not_equal, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
745
746 {
747 Push(object);
748 Register map = object;
749
750 LoadMap(map, object);
751
752 Label do_check;
753 // Check if JSGeneratorObject
754 CmpInstanceType(map, JS_GENERATOR_OBJECT_TYPE);
755 j(equal, &do_check, Label::kNear);
756
757 // Check if JSAsyncFunctionObject.
758 CmpInstanceType(map, JS_ASYNC_FUNCTION_OBJECT_TYPE);
759 j(equal, &do_check, Label::kNear);
760
761 // Check if JSAsyncGeneratorObject
762 CmpInstanceType(map, JS_ASYNC_GENERATOR_OBJECT_TYPE);
763
764 bind(&do_check);
765 Pop(object);
766 }
767
768 Check(equal, AbortReason::kOperandIsNotAGeneratorObject);
769 }
770
AssertUndefinedOrAllocationSite(Register object,Register scratch)771 void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
772 Register scratch) {
773 if (emit_debug_code()) {
774 Label done_checking;
775 AssertNotSmi(object);
776 CompareRoot(object, scratch, RootIndex::kUndefinedValue);
777 j(equal, &done_checking);
778 LoadRoot(scratch, RootIndex::kAllocationSiteWithWeakNextMap);
779 cmp(FieldOperand(object, 0), scratch);
780 Assert(equal, AbortReason::kExpectedUndefinedOrCell);
781 bind(&done_checking);
782 }
783 }
784
AssertNotSmi(Register object)785 void MacroAssembler::AssertNotSmi(Register object) {
786 if (emit_debug_code()) {
787 test(object, Immediate(kSmiTagMask));
788 Check(not_equal, AbortReason::kOperandIsASmi);
789 }
790 }
791
StubPrologue(StackFrame::Type type)792 void TurboAssembler::StubPrologue(StackFrame::Type type) {
793 push(ebp); // Caller's frame pointer.
794 mov(ebp, esp);
795 push(Immediate(StackFrame::TypeToMarker(type)));
796 }
797
Prologue()798 void TurboAssembler::Prologue() {
799 push(ebp); // Caller's frame pointer.
800 mov(ebp, esp);
801 push(kContextRegister); // Callee's context.
802 push(kJSFunctionRegister); // Callee's JS function.
803 push(kJavaScriptCallArgCountRegister); // Actual argument count.
804 }
805
EnterFrame(StackFrame::Type type)806 void TurboAssembler::EnterFrame(StackFrame::Type type) {
807 push(ebp);
808 mov(ebp, esp);
809 push(Immediate(StackFrame::TypeToMarker(type)));
810 }
811
LeaveFrame(StackFrame::Type type)812 void TurboAssembler::LeaveFrame(StackFrame::Type type) {
813 if (emit_debug_code()) {
814 cmp(Operand(ebp, CommonFrameConstants::kContextOrFrameTypeOffset),
815 Immediate(StackFrame::TypeToMarker(type)));
816 Check(equal, AbortReason::kStackFrameTypesMustMatch);
817 }
818 leave();
819 }
820
821 #ifdef V8_OS_WIN
AllocateStackSpace(Register bytes_scratch)822 void TurboAssembler::AllocateStackSpace(Register bytes_scratch) {
823 // In windows, we cannot increment the stack size by more than one page
824 // (minimum page size is 4KB) without accessing at least one byte on the
825 // page. Check this:
826 // https://msdn.microsoft.com/en-us/library/aa227153(v=vs.60).aspx.
827 Label check_offset;
828 Label touch_next_page;
829 jmp(&check_offset);
830 bind(&touch_next_page);
831 sub(esp, Immediate(kStackPageSize));
832 // Just to touch the page, before we increment further.
833 mov(Operand(esp, 0), Immediate(0));
834 sub(bytes_scratch, Immediate(kStackPageSize));
835
836 bind(&check_offset);
837 cmp(bytes_scratch, kStackPageSize);
838 j(greater, &touch_next_page);
839
840 sub(esp, bytes_scratch);
841 }
842
AllocateStackSpace(int bytes)843 void TurboAssembler::AllocateStackSpace(int bytes) {
844 while (bytes > kStackPageSize) {
845 sub(esp, Immediate(kStackPageSize));
846 mov(Operand(esp, 0), Immediate(0));
847 bytes -= kStackPageSize;
848 }
849 sub(esp, Immediate(bytes));
850 }
851 #endif
852
EnterExitFramePrologue(StackFrame::Type frame_type,Register scratch)853 void MacroAssembler::EnterExitFramePrologue(StackFrame::Type frame_type,
854 Register scratch) {
855 DCHECK(frame_type == StackFrame::EXIT ||
856 frame_type == StackFrame::BUILTIN_EXIT);
857
858 // Set up the frame structure on the stack.
859 DCHECK_EQ(+2 * kSystemPointerSize, ExitFrameConstants::kCallerSPDisplacement);
860 DCHECK_EQ(+1 * kSystemPointerSize, ExitFrameConstants::kCallerPCOffset);
861 DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
862 push(ebp);
863 mov(ebp, esp);
864
865 // Reserve room for entry stack pointer.
866 push(Immediate(StackFrame::TypeToMarker(frame_type)));
867 DCHECK_EQ(-2 * kSystemPointerSize, ExitFrameConstants::kSPOffset);
868 push(Immediate(0)); // Saved entry sp, patched before call.
869
870 STATIC_ASSERT(edx == kRuntimeCallFunctionRegister);
871 STATIC_ASSERT(esi == kContextRegister);
872
873 // Save the frame pointer and the context in top.
874 ExternalReference c_entry_fp_address =
875 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
876 ExternalReference context_address =
877 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
878 ExternalReference c_function_address =
879 ExternalReference::Create(IsolateAddressId::kCFunctionAddress, isolate());
880
881 DCHECK(!AreAliased(scratch, ebp, esi, edx));
882 mov(ExternalReferenceAsOperand(c_entry_fp_address, scratch), ebp);
883 mov(ExternalReferenceAsOperand(context_address, scratch), esi);
884 mov(ExternalReferenceAsOperand(c_function_address, scratch), edx);
885 }
886
EnterExitFrameEpilogue(int argc,bool save_doubles)887 void MacroAssembler::EnterExitFrameEpilogue(int argc, bool save_doubles) {
888 // Optionally save all XMM registers.
889 if (save_doubles) {
890 int space =
891 XMMRegister::kNumRegisters * kDoubleSize + argc * kSystemPointerSize;
892 AllocateStackSpace(space);
893 const int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
894 for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
895 XMMRegister reg = XMMRegister::from_code(i);
896 movsd(Operand(ebp, offset - ((i + 1) * kDoubleSize)), reg);
897 }
898 } else {
899 AllocateStackSpace(argc * kSystemPointerSize);
900 }
901
902 // Get the required frame alignment for the OS.
903 const int kFrameAlignment = base::OS::ActivationFrameAlignment();
904 if (kFrameAlignment > 0) {
905 DCHECK(base::bits::IsPowerOfTwo(kFrameAlignment));
906 and_(esp, -kFrameAlignment);
907 }
908
909 // Patch the saved entry sp.
910 mov(Operand(ebp, ExitFrameConstants::kSPOffset), esp);
911 }
912
EnterExitFrame(int argc,bool save_doubles,StackFrame::Type frame_type)913 void MacroAssembler::EnterExitFrame(int argc, bool save_doubles,
914 StackFrame::Type frame_type) {
915 EnterExitFramePrologue(frame_type, edi);
916
917 // Set up argc and argv in callee-saved registers.
918 int offset = StandardFrameConstants::kCallerSPOffset - kSystemPointerSize;
919 mov(edi, eax);
920 lea(esi, Operand(ebp, eax, times_system_pointer_size, offset));
921
922 // Reserve space for argc, argv and isolate.
923 EnterExitFrameEpilogue(argc, save_doubles);
924 }
925
EnterApiExitFrame(int argc,Register scratch)926 void MacroAssembler::EnterApiExitFrame(int argc, Register scratch) {
927 EnterExitFramePrologue(StackFrame::EXIT, scratch);
928 EnterExitFrameEpilogue(argc, false);
929 }
930
LeaveExitFrame(bool save_doubles,bool pop_arguments)931 void MacroAssembler::LeaveExitFrame(bool save_doubles, bool pop_arguments) {
932 // Optionally restore all XMM registers.
933 if (save_doubles) {
934 const int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
935 for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
936 XMMRegister reg = XMMRegister::from_code(i);
937 movsd(reg, Operand(ebp, offset - ((i + 1) * kDoubleSize)));
938 }
939 }
940
941 if (pop_arguments) {
942 // Get the return address from the stack and restore the frame pointer.
943 mov(ecx, Operand(ebp, 1 * kSystemPointerSize));
944 mov(ebp, Operand(ebp, 0 * kSystemPointerSize));
945
946 // Pop the arguments and the receiver from the caller stack.
947 lea(esp, Operand(esi, 1 * kSystemPointerSize));
948
949 // Push the return address to get ready to return.
950 push(ecx);
951 } else {
952 // Otherwise just leave the exit frame.
953 leave();
954 }
955
956 LeaveExitFrameEpilogue();
957 }
958
LeaveExitFrameEpilogue()959 void MacroAssembler::LeaveExitFrameEpilogue() {
960 // Clear the top frame.
961 ExternalReference c_entry_fp_address =
962 ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
963 mov(ExternalReferenceAsOperand(c_entry_fp_address, esi), Immediate(0));
964
965 // Restore current context from top and clear it in debug mode.
966 ExternalReference context_address =
967 ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
968 mov(esi, ExternalReferenceAsOperand(context_address, esi));
969 #ifdef DEBUG
970 push(eax);
971 mov(ExternalReferenceAsOperand(context_address, eax),
972 Immediate(Context::kInvalidContext));
973 pop(eax);
974 #endif
975 }
976
LeaveApiExitFrame()977 void MacroAssembler::LeaveApiExitFrame() {
978 mov(esp, ebp);
979 pop(ebp);
980
981 LeaveExitFrameEpilogue();
982 }
983
PushStackHandler(Register scratch)984 void MacroAssembler::PushStackHandler(Register scratch) {
985 // Adjust this code if not the case.
986 STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
987 STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
988
989 push(Immediate(0)); // Padding.
990
991 // Link the current handler as the next handler.
992 ExternalReference handler_address =
993 ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
994 push(ExternalReferenceAsOperand(handler_address, scratch));
995
996 // Set this new handler as the current one.
997 mov(ExternalReferenceAsOperand(handler_address, scratch), esp);
998 }
999
PopStackHandler(Register scratch)1000 void MacroAssembler::PopStackHandler(Register scratch) {
1001 STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
1002 ExternalReference handler_address =
1003 ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
1004 pop(ExternalReferenceAsOperand(handler_address, scratch));
1005 add(esp, Immediate(StackHandlerConstants::kSize - kSystemPointerSize));
1006 }
1007
CallRuntime(const Runtime::Function * f,int num_arguments,SaveFPRegsMode save_doubles)1008 void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
1009 SaveFPRegsMode save_doubles) {
1010 // If the expected number of arguments of the runtime function is
1011 // constant, we check that the actual number of arguments match the
1012 // expectation.
1013 CHECK(f->nargs < 0 || f->nargs == num_arguments);
1014
1015 // TODO(1236192): Most runtime routines don't need the number of
1016 // arguments passed in because it is constant. At some point we
1017 // should remove this need and make the runtime routine entry code
1018 // smarter.
1019 Move(kRuntimeCallArgCountRegister, Immediate(num_arguments));
1020 Move(kRuntimeCallFunctionRegister, Immediate(ExternalReference::Create(f)));
1021 Handle<Code> code =
1022 CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
1023 Call(code, RelocInfo::CODE_TARGET);
1024 }
1025
TailCallRuntime(Runtime::FunctionId fid)1026 void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
1027 // ----------- S t a t e -------------
1028 // -- esp[0] : return address
1029 // -- esp[8] : argument num_arguments - 1
1030 // ...
1031 // -- esp[8 * num_arguments] : argument 0 (receiver)
1032 //
1033 // For runtime functions with variable arguments:
1034 // -- eax : number of arguments
1035 // -----------------------------------
1036
1037 const Runtime::Function* function = Runtime::FunctionForId(fid);
1038 DCHECK_EQ(1, function->result_size);
1039 if (function->nargs >= 0) {
1040 // TODO(1236192): Most runtime routines don't need the number of
1041 // arguments passed in because it is constant. At some point we
1042 // should remove this need and make the runtime routine entry code
1043 // smarter.
1044 Move(kRuntimeCallArgCountRegister, Immediate(function->nargs));
1045 }
1046 JumpToExternalReference(ExternalReference::Create(fid));
1047 }
1048
JumpToExternalReference(const ExternalReference & ext,bool builtin_exit_frame)1049 void MacroAssembler::JumpToExternalReference(const ExternalReference& ext,
1050 bool builtin_exit_frame) {
1051 // Set the entry point and jump to the C entry runtime stub.
1052 Move(kRuntimeCallFunctionRegister, Immediate(ext));
1053 Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
1054 kArgvOnStack, builtin_exit_frame);
1055 Jump(code, RelocInfo::CODE_TARGET);
1056 }
1057
JumpToInstructionStream(Address entry)1058 void MacroAssembler::JumpToInstructionStream(Address entry) {
1059 jmp(entry, RelocInfo::OFF_HEAP_TARGET);
1060 }
1061
PrepareForTailCall(Register callee_args_count,Register caller_args_count,Register scratch0,Register scratch1,int number_of_temp_values_after_return_address)1062 void TurboAssembler::PrepareForTailCall(
1063 Register callee_args_count, Register caller_args_count, Register scratch0,
1064 Register scratch1, int number_of_temp_values_after_return_address) {
1065 DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
1066
1067 // Calculate the destination address where we will put the return address
1068 // after we drop current frame.
1069 Register new_sp_reg = scratch0;
1070 sub(caller_args_count, callee_args_count);
1071 lea(new_sp_reg, Operand(ebp, caller_args_count, times_system_pointer_size,
1072 StandardFrameConstants::kCallerPCOffset -
1073 number_of_temp_values_after_return_address *
1074 kSystemPointerSize));
1075
1076 if (FLAG_debug_code) {
1077 cmp(esp, new_sp_reg);
1078 Check(below, AbortReason::kStackAccessBelowStackPointer);
1079 }
1080
1081 // Copy return address from caller's frame to current frame's return address
1082 // to avoid its trashing and let the following loop copy it to the right
1083 // place.
1084 Register tmp_reg = scratch1;
1085 mov(tmp_reg, Operand(ebp, StandardFrameConstants::kCallerPCOffset));
1086 mov(Operand(esp,
1087 number_of_temp_values_after_return_address * kSystemPointerSize),
1088 tmp_reg);
1089
1090 // Restore caller's frame pointer now as it could be overwritten by
1091 // the copying loop.
1092 mov(ebp, Operand(ebp, StandardFrameConstants::kCallerFPOffset));
1093
1094 // +2 here is to copy both receiver and return address.
1095 Register count_reg = caller_args_count;
1096 lea(count_reg, Operand(callee_args_count,
1097 2 + number_of_temp_values_after_return_address));
1098
1099 // Now copy callee arguments to the caller frame going backwards to avoid
1100 // callee arguments corruption (source and destination areas could overlap).
1101 Label loop, entry;
1102 jmp(&entry, Label::kNear);
1103 bind(&loop);
1104 dec(count_reg);
1105 mov(tmp_reg, Operand(esp, count_reg, times_system_pointer_size, 0));
1106 mov(Operand(new_sp_reg, count_reg, times_system_pointer_size, 0), tmp_reg);
1107 bind(&entry);
1108 cmp(count_reg, Immediate(0));
1109 j(not_equal, &loop, Label::kNear);
1110
1111 // Leave current frame.
1112 mov(esp, new_sp_reg);
1113 }
1114
CompareStackLimit(Register with,StackLimitKind kind)1115 void MacroAssembler::CompareStackLimit(Register with, StackLimitKind kind) {
1116 DCHECK(root_array_available());
1117 Isolate* isolate = this->isolate();
1118 // Address through the root register. No load is needed.
1119 ExternalReference limit =
1120 kind == StackLimitKind::kRealStackLimit
1121 ? ExternalReference::address_of_real_jslimit(isolate)
1122 : ExternalReference::address_of_jslimit(isolate);
1123 DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
1124
1125 intptr_t offset =
1126 TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
1127 cmp(with, Operand(kRootRegister, offset));
1128 }
1129
StackOverflowCheck(Register num_args,Register scratch,Label * stack_overflow,bool include_receiver)1130 void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch,
1131 Label* stack_overflow,
1132 bool include_receiver) {
1133 DCHECK_NE(num_args, scratch);
1134 // Check the stack for overflow. We are not trying to catch
1135 // interruptions (e.g. debug break and preemption) here, so the "real stack
1136 // limit" is checked.
1137 ExternalReference real_stack_limit =
1138 ExternalReference::address_of_real_jslimit(isolate());
1139 // Compute the space that is left as a negative number in scratch. If
1140 // we already overflowed, this will be a positive number.
1141 mov(scratch, ExternalReferenceAsOperand(real_stack_limit, scratch));
1142 sub(scratch, esp);
1143 // TODO(victorgomes): Remove {include_receiver} and always require one extra
1144 // word of the stack space.
1145 lea(scratch, Operand(scratch, num_args, times_system_pointer_size, 0));
1146 if (include_receiver) {
1147 add(scratch, Immediate(kSystemPointerSize));
1148 }
1149 // See if we overflowed, i.e. scratch is positive.
1150 cmp(scratch, Immediate(0));
1151 // TODO(victorgomes): Save some bytes in the builtins that use stack checks
1152 // by jumping to a builtin that throws the exception.
1153 j(greater, stack_overflow); // Signed comparison.
1154 }
1155
InvokePrologue(Register expected_parameter_count,Register actual_parameter_count,Label * done,InvokeFlag flag)1156 void MacroAssembler::InvokePrologue(Register expected_parameter_count,
1157 Register actual_parameter_count,
1158 Label* done, InvokeFlag flag) {
1159 if (expected_parameter_count != actual_parameter_count) {
1160 DCHECK_EQ(actual_parameter_count, eax);
1161 DCHECK_EQ(expected_parameter_count, ecx);
1162 Label regular_invoke;
1163 #ifdef V8_NO_ARGUMENTS_ADAPTOR
1164 // If the expected parameter count is equal to the adaptor sentinel, no need
1165 // to push undefined value as arguments.
1166 cmp(expected_parameter_count, Immediate(kDontAdaptArgumentsSentinel));
1167 j(equal, ®ular_invoke, Label::kFar);
1168
1169 // If overapplication or if the actual argument count is equal to the
1170 // formal parameter count, no need to push extra undefined values.
1171 sub(expected_parameter_count, actual_parameter_count);
1172 j(less_equal, ®ular_invoke, Label::kFar);
1173
1174 // We need to preserve edx, edi, esi and ebx.
1175 movd(xmm0, edx);
1176 movd(xmm1, edi);
1177 movd(xmm2, esi);
1178 movd(xmm3, ebx);
1179
1180 Label stack_overflow;
1181 StackOverflowCheck(expected_parameter_count, edx, &stack_overflow);
1182
1183 Register scratch = esi;
1184
1185 // Underapplication. Move the arguments already in the stack, including the
1186 // receiver and the return address.
1187 {
1188 Label copy, check;
1189 Register src = edx, dest = esp, num = edi, current = ebx;
1190 mov(src, esp);
1191 lea(scratch,
1192 Operand(expected_parameter_count, times_system_pointer_size, 0));
1193 AllocateStackSpace(scratch);
1194 // Extra words are the receiver and the return address (if a jump).
1195 int extra_words = flag == CALL_FUNCTION ? 1 : 2;
1196 lea(num, Operand(eax, extra_words)); // Number of words to copy.
1197 Set(current, 0);
1198 // Fall-through to the loop body because there are non-zero words to copy.
1199 bind(©);
1200 mov(scratch, Operand(src, current, times_system_pointer_size, 0));
1201 mov(Operand(dest, current, times_system_pointer_size, 0), scratch);
1202 inc(current);
1203 bind(&check);
1204 cmp(current, num);
1205 j(less, ©);
1206 lea(edx, Operand(esp, num, times_system_pointer_size, 0));
1207 }
1208
1209 // Fill remaining expected arguments with undefined values.
1210 movd(ebx, xmm3); // Restore root.
1211 LoadRoot(scratch, RootIndex::kUndefinedValue);
1212 {
1213 Label loop;
1214 bind(&loop);
1215 dec(expected_parameter_count);
1216 mov(Operand(edx, expected_parameter_count, times_system_pointer_size, 0),
1217 scratch);
1218 j(greater, &loop, Label::kNear);
1219 }
1220
1221 // Restore remaining registers.
1222 movd(esi, xmm2);
1223 movd(edi, xmm1);
1224 movd(edx, xmm0);
1225
1226 jmp(®ular_invoke);
1227
1228 bind(&stack_overflow);
1229 {
1230 FrameScope frame(this,
1231 has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
1232 CallRuntime(Runtime::kThrowStackOverflow);
1233 int3(); // This should be unreachable.
1234 }
1235 #else
1236 cmp(expected_parameter_count, actual_parameter_count);
1237 j(equal, ®ular_invoke);
1238 Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
1239 if (flag == CALL_FUNCTION) {
1240 Call(adaptor, RelocInfo::CODE_TARGET);
1241 jmp(done, Label::kNear);
1242 } else {
1243 Jump(adaptor, RelocInfo::CODE_TARGET);
1244 }
1245 #endif
1246 bind(®ular_invoke);
1247 }
1248 }
1249
CallDebugOnFunctionCall(Register fun,Register new_target,Register expected_parameter_count,Register actual_parameter_count)1250 void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
1251 Register expected_parameter_count,
1252 Register actual_parameter_count) {
1253 FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
1254 SmiTag(expected_parameter_count);
1255 Push(expected_parameter_count);
1256
1257 SmiTag(actual_parameter_count);
1258 Push(actual_parameter_count);
1259 SmiUntag(actual_parameter_count);
1260
1261 if (new_target.is_valid()) {
1262 Push(new_target);
1263 }
1264 Push(fun);
1265 Push(fun);
1266 // Arguments are located 2 words below the base pointer.
1267 Operand receiver_op = Operand(ebp, kSystemPointerSize * 2);
1268 Push(receiver_op);
1269 CallRuntime(Runtime::kDebugOnFunctionCall);
1270 Pop(fun);
1271 if (new_target.is_valid()) {
1272 Pop(new_target);
1273 }
1274 Pop(actual_parameter_count);
1275 SmiUntag(actual_parameter_count);
1276
1277 Pop(expected_parameter_count);
1278 SmiUntag(expected_parameter_count);
1279 }
1280
InvokeFunctionCode(Register function,Register new_target,Register expected_parameter_count,Register actual_parameter_count,InvokeFlag flag)1281 void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
1282 Register expected_parameter_count,
1283 Register actual_parameter_count,
1284 InvokeFlag flag) {
1285 // You can't call a function without a valid frame.
1286 DCHECK_IMPLIES(flag == CALL_FUNCTION, has_frame());
1287 DCHECK_EQ(function, edi);
1288 DCHECK_IMPLIES(new_target.is_valid(), new_target == edx);
1289 DCHECK(expected_parameter_count == ecx || expected_parameter_count == eax);
1290 DCHECK_EQ(actual_parameter_count, eax);
1291
1292 // On function call, call into the debugger if necessary.
1293 Label debug_hook, continue_after_hook;
1294 {
1295 ExternalReference debug_hook_active =
1296 ExternalReference::debug_hook_on_function_call_address(isolate());
1297 push(eax);
1298 cmpb(ExternalReferenceAsOperand(debug_hook_active, eax), Immediate(0));
1299 pop(eax);
1300 j(not_equal, &debug_hook);
1301 }
1302 bind(&continue_after_hook);
1303
1304 // Clear the new.target register if not given.
1305 if (!new_target.is_valid()) {
1306 Move(edx, isolate()->factory()->undefined_value());
1307 }
1308
1309 Label done;
1310 InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
1311 // We call indirectly through the code field in the function to
1312 // allow recompilation to take effect without changing any of the
1313 // call sites.
1314 static_assert(kJavaScriptCallCodeStartRegister == ecx, "ABI mismatch");
1315 mov(ecx, FieldOperand(function, JSFunction::kCodeOffset));
1316 if (flag == CALL_FUNCTION) {
1317 CallCodeObject(ecx);
1318 } else {
1319 DCHECK(flag == JUMP_FUNCTION);
1320 JumpCodeObject(ecx);
1321 }
1322 jmp(&done, Label::kNear);
1323
1324 // Deferred debug hook.
1325 bind(&debug_hook);
1326 CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
1327 actual_parameter_count);
1328 jmp(&continue_after_hook);
1329
1330 bind(&done);
1331 }
1332
InvokeFunction(Register fun,Register new_target,Register actual_parameter_count,InvokeFlag flag)1333 void MacroAssembler::InvokeFunction(Register fun, Register new_target,
1334 Register actual_parameter_count,
1335 InvokeFlag flag) {
1336 // You can't call a function without a valid frame.
1337 DCHECK(flag == JUMP_FUNCTION || has_frame());
1338
1339 DCHECK(fun == edi);
1340 mov(ecx, FieldOperand(edi, JSFunction::kSharedFunctionInfoOffset));
1341 mov(esi, FieldOperand(edi, JSFunction::kContextOffset));
1342 movzx_w(ecx,
1343 FieldOperand(ecx, SharedFunctionInfo::kFormalParameterCountOffset));
1344
1345 InvokeFunctionCode(edi, new_target, ecx, actual_parameter_count, flag);
1346 }
1347
LoadGlobalProxy(Register dst)1348 void MacroAssembler::LoadGlobalProxy(Register dst) {
1349 LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX);
1350 }
1351
LoadNativeContextSlot(Register destination,int index)1352 void MacroAssembler::LoadNativeContextSlot(Register destination, int index) {
1353 // Load the native context from the current context.
1354 LoadMap(destination, esi);
1355 mov(destination,
1356 FieldOperand(destination,
1357 Map::kConstructorOrBackPointerOrNativeContextOffset));
1358 // Load the function from the native context.
1359 mov(destination, Operand(destination, Context::SlotOffset(index)));
1360 }
1361
SafepointRegisterStackIndex(int reg_code)1362 int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
1363 // The registers are pushed starting with the lowest encoding,
1364 // which means that lowest encodings are furthest away from
1365 // the stack pointer.
1366 DCHECK(reg_code >= 0 && reg_code < kNumSafepointRegisters);
1367 return kNumSafepointRegisters - reg_code - 1;
1368 }
1369
Ret()1370 void TurboAssembler::Ret() { ret(0); }
1371
Ret(int bytes_dropped,Register scratch)1372 void TurboAssembler::Ret(int bytes_dropped, Register scratch) {
1373 if (is_uint16(bytes_dropped)) {
1374 ret(bytes_dropped);
1375 } else {
1376 pop(scratch);
1377 add(esp, Immediate(bytes_dropped));
1378 push(scratch);
1379 ret(0);
1380 }
1381 }
1382
Push(Immediate value)1383 void TurboAssembler::Push(Immediate value) {
1384 if (root_array_available() && options().isolate_independent_code) {
1385 if (value.is_embedded_object()) {
1386 Push(HeapObjectAsOperand(value.embedded_object()));
1387 return;
1388 } else if (value.is_external_reference()) {
1389 Push(ExternalReferenceAddressAsOperand(value.external_reference()));
1390 return;
1391 }
1392 }
1393 push(value);
1394 }
1395
Drop(int stack_elements)1396 void MacroAssembler::Drop(int stack_elements) {
1397 if (stack_elements > 0) {
1398 add(esp, Immediate(stack_elements * kSystemPointerSize));
1399 }
1400 }
1401
Move(Register dst,Register src)1402 void TurboAssembler::Move(Register dst, Register src) {
1403 if (dst != src) {
1404 mov(dst, src);
1405 }
1406 }
1407
Move(Register dst,const Immediate & src)1408 void TurboAssembler::Move(Register dst, const Immediate& src) {
1409 if (!src.is_heap_object_request() && src.is_zero()) {
1410 xor_(dst, dst); // Shorter than mov of 32-bit immediate 0.
1411 } else if (src.is_external_reference()) {
1412 LoadAddress(dst, src.external_reference());
1413 } else {
1414 mov(dst, src);
1415 }
1416 }
1417
Move(Operand dst,const Immediate & src)1418 void TurboAssembler::Move(Operand dst, const Immediate& src) {
1419 // Since there's no scratch register available, take a detour through the
1420 // stack.
1421 if (root_array_available() && options().isolate_independent_code) {
1422 if (src.is_embedded_object() || src.is_external_reference() ||
1423 src.is_heap_object_request()) {
1424 Push(src);
1425 pop(dst);
1426 return;
1427 }
1428 }
1429
1430 if (src.is_embedded_object()) {
1431 mov(dst, src.embedded_object());
1432 } else {
1433 mov(dst, src);
1434 }
1435 }
1436
Move(Register dst,Handle<HeapObject> src)1437 void TurboAssembler::Move(Register dst, Handle<HeapObject> src) {
1438 if (root_array_available() && options().isolate_independent_code) {
1439 IndirectLoadConstant(dst, src);
1440 return;
1441 }
1442 mov(dst, src);
1443 }
1444
Move(XMMRegister dst,uint32_t src)1445 void TurboAssembler::Move(XMMRegister dst, uint32_t src) {
1446 if (src == 0) {
1447 pxor(dst, dst);
1448 } else {
1449 unsigned cnt = base::bits::CountPopulation(src);
1450 unsigned nlz = base::bits::CountLeadingZeros32(src);
1451 unsigned ntz = base::bits::CountTrailingZeros32(src);
1452 if (nlz + cnt + ntz == 32) {
1453 pcmpeqd(dst, dst);
1454 if (ntz == 0) {
1455 psrld(dst, 32 - cnt);
1456 } else {
1457 pslld(dst, 32 - cnt);
1458 if (nlz != 0) psrld(dst, nlz);
1459 }
1460 } else {
1461 push(eax);
1462 mov(eax, Immediate(src));
1463 movd(dst, Operand(eax));
1464 pop(eax);
1465 }
1466 }
1467 }
1468
Move(XMMRegister dst,uint64_t src)1469 void TurboAssembler::Move(XMMRegister dst, uint64_t src) {
1470 if (src == 0) {
1471 pxor(dst, dst);
1472 } else {
1473 uint32_t lower = static_cast<uint32_t>(src);
1474 uint32_t upper = static_cast<uint32_t>(src >> 32);
1475 unsigned cnt = base::bits::CountPopulation(src);
1476 unsigned nlz = base::bits::CountLeadingZeros64(src);
1477 unsigned ntz = base::bits::CountTrailingZeros64(src);
1478 if (nlz + cnt + ntz == 64) {
1479 pcmpeqd(dst, dst);
1480 if (ntz == 0) {
1481 psrlq(dst, 64 - cnt);
1482 } else {
1483 psllq(dst, 64 - cnt);
1484 if (nlz != 0) psrlq(dst, nlz);
1485 }
1486 } else if (lower == 0) {
1487 Move(dst, upper);
1488 psllq(dst, 32);
1489 } else if (CpuFeatures::IsSupported(SSE4_1)) {
1490 CpuFeatureScope scope(this, SSE4_1);
1491 push(eax);
1492 Move(eax, Immediate(lower));
1493 movd(dst, Operand(eax));
1494 if (upper != lower) {
1495 Move(eax, Immediate(upper));
1496 }
1497 pinsrd(dst, Operand(eax), 1);
1498 pop(eax);
1499 } else {
1500 push(Immediate(upper));
1501 push(Immediate(lower));
1502 movsd(dst, Operand(esp, 0));
1503 add(esp, Immediate(kDoubleSize));
1504 }
1505 }
1506 }
1507
Pshufhw(XMMRegister dst,Operand src,uint8_t shuffle)1508 void TurboAssembler::Pshufhw(XMMRegister dst, Operand src, uint8_t shuffle) {
1509 if (CpuFeatures::IsSupported(AVX)) {
1510 CpuFeatureScope scope(this, AVX);
1511 vpshufhw(dst, src, shuffle);
1512 } else {
1513 pshufhw(dst, src, shuffle);
1514 }
1515 }
1516
Pshuflw(XMMRegister dst,Operand src,uint8_t shuffle)1517 void TurboAssembler::Pshuflw(XMMRegister dst, Operand src, uint8_t shuffle) {
1518 if (CpuFeatures::IsSupported(AVX)) {
1519 CpuFeatureScope scope(this, AVX);
1520 vpshuflw(dst, src, shuffle);
1521 } else {
1522 pshuflw(dst, src, shuffle);
1523 }
1524 }
1525
Pshufd(XMMRegister dst,Operand src,uint8_t shuffle)1526 void TurboAssembler::Pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
1527 if (CpuFeatures::IsSupported(AVX)) {
1528 CpuFeatureScope scope(this, AVX);
1529 vpshufd(dst, src, shuffle);
1530 } else {
1531 pshufd(dst, src, shuffle);
1532 }
1533 }
1534
Psraw(XMMRegister dst,uint8_t shift)1535 void TurboAssembler::Psraw(XMMRegister dst, uint8_t shift) {
1536 if (CpuFeatures::IsSupported(AVX)) {
1537 CpuFeatureScope scope(this, AVX);
1538 vpsraw(dst, dst, shift);
1539 } else {
1540 psraw(dst, shift);
1541 }
1542 }
1543
Psrlw(XMMRegister dst,uint8_t shift)1544 void TurboAssembler::Psrlw(XMMRegister dst, uint8_t shift) {
1545 if (CpuFeatures::IsSupported(AVX)) {
1546 CpuFeatureScope scope(this, AVX);
1547 vpsrlw(dst, dst, shift);
1548 } else {
1549 psrlw(dst, shift);
1550 }
1551 }
1552
Psrlq(XMMRegister dst,uint8_t shift)1553 void TurboAssembler::Psrlq(XMMRegister dst, uint8_t shift) {
1554 if (CpuFeatures::IsSupported(AVX)) {
1555 CpuFeatureScope scope(this, AVX);
1556 vpsrlq(dst, dst, shift);
1557 } else {
1558 psrlq(dst, shift);
1559 }
1560 }
1561
Psignb(XMMRegister dst,Operand src)1562 void TurboAssembler::Psignb(XMMRegister dst, Operand src) {
1563 if (CpuFeatures::IsSupported(AVX)) {
1564 CpuFeatureScope scope(this, AVX);
1565 vpsignb(dst, dst, src);
1566 return;
1567 }
1568 if (CpuFeatures::IsSupported(SSSE3)) {
1569 CpuFeatureScope sse_scope(this, SSSE3);
1570 psignb(dst, src);
1571 return;
1572 }
1573 FATAL("no AVX or SSE3 support");
1574 }
1575
Psignw(XMMRegister dst,Operand src)1576 void TurboAssembler::Psignw(XMMRegister dst, Operand src) {
1577 if (CpuFeatures::IsSupported(AVX)) {
1578 CpuFeatureScope scope(this, AVX);
1579 vpsignw(dst, dst, src);
1580 return;
1581 }
1582 if (CpuFeatures::IsSupported(SSSE3)) {
1583 CpuFeatureScope sse_scope(this, SSSE3);
1584 psignw(dst, src);
1585 return;
1586 }
1587 FATAL("no AVX or SSE3 support");
1588 }
1589
Psignd(XMMRegister dst,Operand src)1590 void TurboAssembler::Psignd(XMMRegister dst, Operand src) {
1591 if (CpuFeatures::IsSupported(AVX)) {
1592 CpuFeatureScope scope(this, AVX);
1593 vpsignd(dst, dst, src);
1594 return;
1595 }
1596 if (CpuFeatures::IsSupported(SSSE3)) {
1597 CpuFeatureScope sse_scope(this, SSSE3);
1598 psignd(dst, src);
1599 return;
1600 }
1601 FATAL("no AVX or SSE3 support");
1602 }
1603
Pshufb(XMMRegister dst,XMMRegister src,Operand mask)1604 void TurboAssembler::Pshufb(XMMRegister dst, XMMRegister src, Operand mask) {
1605 if (CpuFeatures::IsSupported(AVX)) {
1606 CpuFeatureScope scope(this, AVX);
1607 vpshufb(dst, src, mask);
1608 return;
1609 }
1610 if (CpuFeatures::IsSupported(SSSE3)) {
1611 // Make sure these are different so that we won't overwrite mask.
1612 DCHECK(!mask.is_reg(dst));
1613 CpuFeatureScope sse_scope(this, SSSE3);
1614 if (dst != src) {
1615 movapd(dst, src);
1616 }
1617 pshufb(dst, mask);
1618 return;
1619 }
1620 FATAL("no AVX or SSE3 support");
1621 }
1622
Pblendw(XMMRegister dst,Operand src,uint8_t imm8)1623 void TurboAssembler::Pblendw(XMMRegister dst, Operand src, uint8_t imm8) {
1624 if (CpuFeatures::IsSupported(AVX)) {
1625 CpuFeatureScope scope(this, AVX);
1626 vpblendw(dst, dst, src, imm8);
1627 return;
1628 }
1629 if (CpuFeatures::IsSupported(SSE4_1)) {
1630 CpuFeatureScope sse_scope(this, SSE4_1);
1631 pblendw(dst, src, imm8);
1632 return;
1633 }
1634 FATAL("no AVX or SSE4.1 support");
1635 }
1636
Palignr(XMMRegister dst,Operand src,uint8_t imm8)1637 void TurboAssembler::Palignr(XMMRegister dst, Operand src, uint8_t imm8) {
1638 if (CpuFeatures::IsSupported(AVX)) {
1639 CpuFeatureScope scope(this, AVX);
1640 vpalignr(dst, dst, src, imm8);
1641 return;
1642 }
1643 if (CpuFeatures::IsSupported(SSSE3)) {
1644 CpuFeatureScope sse_scope(this, SSSE3);
1645 palignr(dst, src, imm8);
1646 return;
1647 }
1648 FATAL("no AVX or SSE3 support");
1649 }
1650
Pextrb(Register dst,XMMRegister src,uint8_t imm8)1651 void TurboAssembler::Pextrb(Register dst, XMMRegister src, uint8_t imm8) {
1652 if (CpuFeatures::IsSupported(AVX)) {
1653 CpuFeatureScope scope(this, AVX);
1654 vpextrb(dst, src, imm8);
1655 return;
1656 }
1657 if (CpuFeatures::IsSupported(SSE4_1)) {
1658 CpuFeatureScope sse_scope(this, SSE4_1);
1659 pextrb(dst, src, imm8);
1660 return;
1661 }
1662 FATAL("no AVX or SSE4.1 support");
1663 }
1664
Pextrw(Register dst,XMMRegister src,uint8_t imm8)1665 void TurboAssembler::Pextrw(Register dst, XMMRegister src, uint8_t imm8) {
1666 if (CpuFeatures::IsSupported(AVX)) {
1667 CpuFeatureScope scope(this, AVX);
1668 vpextrw(dst, src, imm8);
1669 return;
1670 }
1671 if (CpuFeatures::IsSupported(SSE4_1)) {
1672 CpuFeatureScope sse_scope(this, SSE4_1);
1673 pextrw(dst, src, imm8);
1674 return;
1675 }
1676 FATAL("no AVX or SSE4.1 support");
1677 }
1678
Pextrd(Register dst,XMMRegister src,uint8_t imm8)1679 void TurboAssembler::Pextrd(Register dst, XMMRegister src, uint8_t imm8) {
1680 if (imm8 == 0) {
1681 Movd(dst, src);
1682 return;
1683 }
1684 if (CpuFeatures::IsSupported(AVX)) {
1685 CpuFeatureScope scope(this, AVX);
1686 vpextrd(dst, src, imm8);
1687 return;
1688 }
1689 if (CpuFeatures::IsSupported(SSE4_1)) {
1690 CpuFeatureScope sse_scope(this, SSE4_1);
1691 pextrd(dst, src, imm8);
1692 return;
1693 }
1694 // Without AVX or SSE, we can only have 64-bit values in xmm registers.
1695 // We don't have an xmm scratch register, so move the data via the stack. This
1696 // path is rarely required, so it's acceptable to be slow.
1697 DCHECK_LT(imm8, 2);
1698 AllocateStackSpace(kDoubleSize);
1699 movsd(Operand(esp, 0), src);
1700 mov(dst, Operand(esp, imm8 * kUInt32Size));
1701 add(esp, Immediate(kDoubleSize));
1702 }
1703
Pinsrb(XMMRegister dst,Operand src,int8_t imm8)1704 void TurboAssembler::Pinsrb(XMMRegister dst, Operand src, int8_t imm8) {
1705 if (CpuFeatures::IsSupported(AVX)) {
1706 CpuFeatureScope scope(this, AVX);
1707 vpinsrb(dst, dst, src, imm8);
1708 return;
1709 }
1710 if (CpuFeatures::IsSupported(SSE4_1)) {
1711 CpuFeatureScope sse_scope(this, SSE4_1);
1712 pinsrb(dst, src, imm8);
1713 return;
1714 }
1715 FATAL("no AVX or SSE4.1 support");
1716 }
1717
Pinsrd(XMMRegister dst,Operand src,uint8_t imm8)1718 void TurboAssembler::Pinsrd(XMMRegister dst, Operand src, uint8_t imm8) {
1719 if (CpuFeatures::IsSupported(AVX)) {
1720 CpuFeatureScope scope(this, AVX);
1721 vpinsrd(dst, dst, src, imm8);
1722 return;
1723 }
1724 if (CpuFeatures::IsSupported(SSE4_1)) {
1725 CpuFeatureScope sse_scope(this, SSE4_1);
1726 pinsrd(dst, src, imm8);
1727 return;
1728 }
1729 // Without AVX or SSE, we can only have 64-bit values in xmm registers.
1730 // We don't have an xmm scratch register, so move the data via the stack. This
1731 // path is rarely required, so it's acceptable to be slow.
1732 DCHECK_LT(imm8, 2);
1733 AllocateStackSpace(kDoubleSize);
1734 // Write original content of {dst} to the stack.
1735 movsd(Operand(esp, 0), dst);
1736 // Overwrite the portion specified in {imm8}.
1737 if (src.is_reg_only()) {
1738 mov(Operand(esp, imm8 * kUInt32Size), src.reg());
1739 } else {
1740 movss(dst, src);
1741 movss(Operand(esp, imm8 * kUInt32Size), dst);
1742 }
1743 // Load back the full value into {dst}.
1744 movsd(dst, Operand(esp, 0));
1745 add(esp, Immediate(kDoubleSize));
1746 }
1747
Pinsrw(XMMRegister dst,Operand src,int8_t imm8)1748 void TurboAssembler::Pinsrw(XMMRegister dst, Operand src, int8_t imm8) {
1749 if (CpuFeatures::IsSupported(AVX)) {
1750 CpuFeatureScope scope(this, AVX);
1751 vpinsrw(dst, dst, src, imm8);
1752 return;
1753 } else {
1754 pinsrw(dst, src, imm8);
1755 return;
1756 }
1757 }
1758
Vbroadcastss(XMMRegister dst,Operand src)1759 void TurboAssembler::Vbroadcastss(XMMRegister dst, Operand src) {
1760 if (CpuFeatures::IsSupported(AVX)) {
1761 CpuFeatureScope avx_scope(this, AVX);
1762 vbroadcastss(dst, src);
1763 return;
1764 }
1765 movss(dst, src);
1766 shufps(dst, dst, static_cast<byte>(0));
1767 }
1768
Lzcnt(Register dst,Operand src)1769 void TurboAssembler::Lzcnt(Register dst, Operand src) {
1770 if (CpuFeatures::IsSupported(LZCNT)) {
1771 CpuFeatureScope scope(this, LZCNT);
1772 lzcnt(dst, src);
1773 return;
1774 }
1775 Label not_zero_src;
1776 bsr(dst, src);
1777 j(not_zero, ¬_zero_src, Label::kNear);
1778 mov(dst, 63); // 63^31 == 32
1779 bind(¬_zero_src);
1780 xor_(dst, Immediate(31)); // for x in [0..31], 31^x == 31-x.
1781 }
1782
Tzcnt(Register dst,Operand src)1783 void TurboAssembler::Tzcnt(Register dst, Operand src) {
1784 if (CpuFeatures::IsSupported(BMI1)) {
1785 CpuFeatureScope scope(this, BMI1);
1786 tzcnt(dst, src);
1787 return;
1788 }
1789 Label not_zero_src;
1790 bsf(dst, src);
1791 j(not_zero, ¬_zero_src, Label::kNear);
1792 mov(dst, 32); // The result of tzcnt is 32 if src = 0.
1793 bind(¬_zero_src);
1794 }
1795
Popcnt(Register dst,Operand src)1796 void TurboAssembler::Popcnt(Register dst, Operand src) {
1797 if (CpuFeatures::IsSupported(POPCNT)) {
1798 CpuFeatureScope scope(this, POPCNT);
1799 popcnt(dst, src);
1800 return;
1801 }
1802 FATAL("no POPCNT support");
1803 }
1804
LoadWeakValue(Register in_out,Label * target_if_cleared)1805 void MacroAssembler::LoadWeakValue(Register in_out, Label* target_if_cleared) {
1806 cmp(in_out, Immediate(kClearedWeakHeapObjectLower32));
1807 j(equal, target_if_cleared);
1808
1809 and_(in_out, Immediate(~kWeakHeapObjectMask));
1810 }
1811
IncrementCounter(StatsCounter * counter,int value,Register scratch)1812 void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
1813 Register scratch) {
1814 DCHECK_GT(value, 0);
1815 if (FLAG_native_code_counters && counter->Enabled()) {
1816 Operand operand =
1817 ExternalReferenceAsOperand(ExternalReference::Create(counter), scratch);
1818 if (value == 1) {
1819 inc(operand);
1820 } else {
1821 add(operand, Immediate(value));
1822 }
1823 }
1824 }
1825
DecrementCounter(StatsCounter * counter,int value,Register scratch)1826 void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
1827 Register scratch) {
1828 DCHECK_GT(value, 0);
1829 if (FLAG_native_code_counters && counter->Enabled()) {
1830 Operand operand =
1831 ExternalReferenceAsOperand(ExternalReference::Create(counter), scratch);
1832 if (value == 1) {
1833 dec(operand);
1834 } else {
1835 sub(operand, Immediate(value));
1836 }
1837 }
1838 }
1839
Assert(Condition cc,AbortReason reason)1840 void TurboAssembler::Assert(Condition cc, AbortReason reason) {
1841 if (emit_debug_code()) Check(cc, reason);
1842 }
1843
AssertUnreachable(AbortReason reason)1844 void TurboAssembler::AssertUnreachable(AbortReason reason) {
1845 if (emit_debug_code()) Abort(reason);
1846 }
1847
Check(Condition cc,AbortReason reason)1848 void TurboAssembler::Check(Condition cc, AbortReason reason) {
1849 Label L;
1850 j(cc, &L);
1851 Abort(reason);
1852 // will not return here
1853 bind(&L);
1854 }
1855
CheckStackAlignment()1856 void TurboAssembler::CheckStackAlignment() {
1857 int frame_alignment = base::OS::ActivationFrameAlignment();
1858 int frame_alignment_mask = frame_alignment - 1;
1859 if (frame_alignment > kSystemPointerSize) {
1860 DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
1861 Label alignment_as_expected;
1862 test(esp, Immediate(frame_alignment_mask));
1863 j(zero, &alignment_as_expected);
1864 // Abort if stack is not aligned.
1865 int3();
1866 bind(&alignment_as_expected);
1867 }
1868 }
1869
Abort(AbortReason reason)1870 void TurboAssembler::Abort(AbortReason reason) {
1871 #ifdef DEBUG
1872 const char* msg = GetAbortReason(reason);
1873 RecordComment("Abort message: ");
1874 RecordComment(msg);
1875 #endif
1876
1877 // Avoid emitting call to builtin if requested.
1878 if (trap_on_abort()) {
1879 int3();
1880 return;
1881 }
1882
1883 if (should_abort_hard()) {
1884 // We don't care if we constructed a frame. Just pretend we did.
1885 FrameScope assume_frame(this, StackFrame::NONE);
1886 PrepareCallCFunction(1, eax);
1887 mov(Operand(esp, 0), Immediate(static_cast<int>(reason)));
1888 CallCFunction(ExternalReference::abort_with_reason(), 1);
1889 return;
1890 }
1891
1892 Move(edx, Smi::FromInt(static_cast<int>(reason)));
1893
1894 // Disable stub call restrictions to always allow calls to abort.
1895 if (!has_frame()) {
1896 // We don't actually want to generate a pile of code for this, so just
1897 // claim there is a stack frame, without generating one.
1898 FrameScope scope(this, StackFrame::NONE);
1899 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
1900 } else {
1901 Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
1902 }
1903 // will not return here
1904 int3();
1905 }
1906
PrepareCallCFunction(int num_arguments,Register scratch)1907 void TurboAssembler::PrepareCallCFunction(int num_arguments, Register scratch) {
1908 int frame_alignment = base::OS::ActivationFrameAlignment();
1909 if (frame_alignment != 0) {
1910 // Make stack end at alignment and make room for num_arguments words
1911 // and the original value of esp.
1912 mov(scratch, esp);
1913 AllocateStackSpace((num_arguments + 1) * kSystemPointerSize);
1914 DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
1915 and_(esp, -frame_alignment);
1916 mov(Operand(esp, num_arguments * kSystemPointerSize), scratch);
1917 } else {
1918 AllocateStackSpace(num_arguments * kSystemPointerSize);
1919 }
1920 }
1921
CallCFunction(ExternalReference function,int num_arguments)1922 void TurboAssembler::CallCFunction(ExternalReference function,
1923 int num_arguments) {
1924 // Trashing eax is ok as it will be the return value.
1925 Move(eax, Immediate(function));
1926 CallCFunction(eax, num_arguments);
1927 }
1928
CallCFunction(Register function,int num_arguments)1929 void TurboAssembler::CallCFunction(Register function, int num_arguments) {
1930 DCHECK_LE(num_arguments, kMaxCParameters);
1931 DCHECK(has_frame());
1932 // Check stack alignment.
1933 if (emit_debug_code()) {
1934 CheckStackAlignment();
1935 }
1936
1937 // Save the frame pointer and PC so that the stack layout remains iterable,
1938 // even without an ExitFrame which normally exists between JS and C frames.
1939 // Find two caller-saved scratch registers.
1940 Register pc_scratch = eax;
1941 Register scratch = ecx;
1942 if (function == eax) pc_scratch = edx;
1943 if (function == ecx) scratch = edx;
1944 PushPC();
1945 pop(pc_scratch);
1946
1947 // See x64 code for reasoning about how to address the isolate data fields.
1948 DCHECK_IMPLIES(!root_array_available(), isolate() != nullptr);
1949 mov(root_array_available()
1950 ? Operand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset())
1951 : ExternalReferenceAsOperand(
1952 ExternalReference::fast_c_call_caller_pc_address(isolate()),
1953 scratch),
1954 pc_scratch);
1955 mov(root_array_available()
1956 ? Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset())
1957 : ExternalReferenceAsOperand(
1958 ExternalReference::fast_c_call_caller_fp_address(isolate()),
1959 scratch),
1960 ebp);
1961
1962 call(function);
1963
1964 // We don't unset the PC; the FP is the source of truth.
1965 mov(root_array_available()
1966 ? Operand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset())
1967 : ExternalReferenceAsOperand(
1968 ExternalReference::fast_c_call_caller_fp_address(isolate()),
1969 scratch),
1970 Immediate(0));
1971
1972 if (base::OS::ActivationFrameAlignment() != 0) {
1973 mov(esp, Operand(esp, num_arguments * kSystemPointerSize));
1974 } else {
1975 add(esp, Immediate(num_arguments * kSystemPointerSize));
1976 }
1977 }
1978
PushPC()1979 void TurboAssembler::PushPC() {
1980 // Push the current PC onto the stack as "return address" via calling
1981 // the next instruction.
1982 Label get_pc;
1983 call(&get_pc);
1984 bind(&get_pc);
1985 }
1986
Call(Handle<Code> code_object,RelocInfo::Mode rmode)1987 void TurboAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
1988 DCHECK_IMPLIES(options().isolate_independent_code,
1989 Builtins::IsIsolateIndependentBuiltin(*code_object));
1990 if (options().inline_offheap_trampolines) {
1991 int builtin_index = Builtins::kNoBuiltinId;
1992 if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
1993 // Inline the trampoline.
1994 CallBuiltin(builtin_index);
1995 return;
1996 }
1997 }
1998 DCHECK(RelocInfo::IsCodeTarget(rmode));
1999 call(code_object, rmode);
2000 }
2001
LoadEntryFromBuiltinIndex(Register builtin_index)2002 void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
2003 STATIC_ASSERT(kSystemPointerSize == 4);
2004 STATIC_ASSERT(kSmiShiftSize == 0);
2005 STATIC_ASSERT(kSmiTagSize == 1);
2006 STATIC_ASSERT(kSmiTag == 0);
2007
2008 // The builtin_index register contains the builtin index as a Smi.
2009 // Untagging is folded into the indexing operand below (we use
2010 // times_half_system_pointer_size instead of times_system_pointer_size since
2011 // smis are already shifted by one).
2012 mov(builtin_index,
2013 Operand(kRootRegister, builtin_index, times_half_system_pointer_size,
2014 IsolateData::builtin_entry_table_offset()));
2015 }
2016
CallBuiltinByIndex(Register builtin_index)2017 void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
2018 LoadEntryFromBuiltinIndex(builtin_index);
2019 call(builtin_index);
2020 }
2021
CallBuiltin(int builtin_index)2022 void TurboAssembler::CallBuiltin(int builtin_index) {
2023 DCHECK(Builtins::IsBuiltinId(builtin_index));
2024 RecordCommentForOffHeapTrampoline(builtin_index);
2025 CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
2026 EmbeddedData d = EmbeddedData::FromBlob();
2027 Address entry = d.InstructionStartOfBuiltin(builtin_index);
2028 call(entry, RelocInfo::OFF_HEAP_TARGET);
2029 }
2030
LoadCodeObjectEntry(Register destination,Register code_object)2031 void TurboAssembler::LoadCodeObjectEntry(Register destination,
2032 Register code_object) {
2033 // Code objects are called differently depending on whether we are generating
2034 // builtin code (which will later be embedded into the binary) or compiling
2035 // user JS code at runtime.
2036 // * Builtin code runs in --jitless mode and thus must not call into on-heap
2037 // Code targets. Instead, we dispatch through the builtins entry table.
2038 // * Codegen at runtime does not have this restriction and we can use the
2039 // shorter, branchless instruction sequence. The assumption here is that
2040 // targets are usually generated code and not builtin Code objects.
2041
2042 if (options().isolate_independent_code) {
2043 DCHECK(root_array_available());
2044 Label if_code_is_off_heap, out;
2045
2046 // Check whether the Code object is an off-heap trampoline. If so, call its
2047 // (off-heap) entry point directly without going through the (on-heap)
2048 // trampoline. Otherwise, just call the Code object as always.
2049 test(FieldOperand(code_object, Code::kFlagsOffset),
2050 Immediate(Code::IsOffHeapTrampoline::kMask));
2051 j(not_equal, &if_code_is_off_heap);
2052
2053 // Not an off-heap trampoline, the entry point is at
2054 // Code::raw_instruction_start().
2055 Move(destination, code_object);
2056 add(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
2057 jmp(&out);
2058
2059 // An off-heap trampoline, the entry point is loaded from the builtin entry
2060 // table.
2061 bind(&if_code_is_off_heap);
2062 mov(destination, FieldOperand(code_object, Code::kBuiltinIndexOffset));
2063 mov(destination,
2064 Operand(kRootRegister, destination, times_system_pointer_size,
2065 IsolateData::builtin_entry_table_offset()));
2066
2067 bind(&out);
2068 } else {
2069 Move(destination, code_object);
2070 add(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
2071 }
2072 }
2073
CallCodeObject(Register code_object)2074 void TurboAssembler::CallCodeObject(Register code_object) {
2075 LoadCodeObjectEntry(code_object, code_object);
2076 call(code_object);
2077 }
2078
JumpCodeObject(Register code_object)2079 void TurboAssembler::JumpCodeObject(Register code_object) {
2080 LoadCodeObjectEntry(code_object, code_object);
2081 jmp(code_object);
2082 }
2083
Jump(const ExternalReference & reference)2084 void TurboAssembler::Jump(const ExternalReference& reference) {
2085 DCHECK(root_array_available());
2086 jmp(Operand(kRootRegister, RootRegisterOffsetForExternalReferenceTableEntry(
2087 isolate(), reference)));
2088 }
2089
Jump(Handle<Code> code_object,RelocInfo::Mode rmode)2090 void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode) {
2091 DCHECK_IMPLIES(options().isolate_independent_code,
2092 Builtins::IsIsolateIndependentBuiltin(*code_object));
2093 if (options().inline_offheap_trampolines) {
2094 int builtin_index = Builtins::kNoBuiltinId;
2095 if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index)) {
2096 // Inline the trampoline.
2097 RecordCommentForOffHeapTrampoline(builtin_index);
2098 CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
2099 EmbeddedData d = EmbeddedData::FromBlob();
2100 Address entry = d.InstructionStartOfBuiltin(builtin_index);
2101 jmp(entry, RelocInfo::OFF_HEAP_TARGET);
2102 return;
2103 }
2104 }
2105 DCHECK(RelocInfo::IsCodeTarget(rmode));
2106 jmp(code_object, rmode);
2107 }
2108
RetpolineCall(Register reg)2109 void TurboAssembler::RetpolineCall(Register reg) {
2110 Label setup_return, setup_target, inner_indirect_branch, capture_spec;
2111
2112 jmp(&setup_return); // Jump past the entire retpoline below.
2113
2114 bind(&inner_indirect_branch);
2115 call(&setup_target);
2116
2117 bind(&capture_spec);
2118 pause();
2119 jmp(&capture_spec);
2120
2121 bind(&setup_target);
2122 mov(Operand(esp, 0), reg);
2123 ret(0);
2124
2125 bind(&setup_return);
2126 call(&inner_indirect_branch); // Callee will return after this instruction.
2127 }
2128
RetpolineCall(Address destination,RelocInfo::Mode rmode)2129 void TurboAssembler::RetpolineCall(Address destination, RelocInfo::Mode rmode) {
2130 Label setup_return, setup_target, inner_indirect_branch, capture_spec;
2131
2132 jmp(&setup_return); // Jump past the entire retpoline below.
2133
2134 bind(&inner_indirect_branch);
2135 call(&setup_target);
2136
2137 bind(&capture_spec);
2138 pause();
2139 jmp(&capture_spec);
2140
2141 bind(&setup_target);
2142 mov(Operand(esp, 0), destination, rmode);
2143 ret(0);
2144
2145 bind(&setup_return);
2146 call(&inner_indirect_branch); // Callee will return after this instruction.
2147 }
2148
RetpolineJump(Register reg)2149 void TurboAssembler::RetpolineJump(Register reg) {
2150 Label setup_target, capture_spec;
2151
2152 call(&setup_target);
2153
2154 bind(&capture_spec);
2155 pause();
2156 jmp(&capture_spec);
2157
2158 bind(&setup_target);
2159 mov(Operand(esp, 0), reg);
2160 ret(0);
2161 }
2162
CheckPageFlag(Register object,Register scratch,int mask,Condition cc,Label * condition_met,Label::Distance condition_met_distance)2163 void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
2164 Condition cc, Label* condition_met,
2165 Label::Distance condition_met_distance) {
2166 DCHECK(cc == zero || cc == not_zero);
2167 if (scratch == object) {
2168 and_(scratch, Immediate(~kPageAlignmentMask));
2169 } else {
2170 mov(scratch, Immediate(~kPageAlignmentMask));
2171 and_(scratch, object);
2172 }
2173 if (mask < (1 << kBitsPerByte)) {
2174 test_b(Operand(scratch, BasicMemoryChunk::kFlagsOffset), Immediate(mask));
2175 } else {
2176 test(Operand(scratch, BasicMemoryChunk::kFlagsOffset), Immediate(mask));
2177 }
2178 j(cc, condition_met, condition_met_distance);
2179 }
2180
ComputeCodeStartAddress(Register dst)2181 void TurboAssembler::ComputeCodeStartAddress(Register dst) {
2182 // In order to get the address of the current instruction, we first need
2183 // to use a call and then use a pop, thus pushing the return address to
2184 // the stack and then popping it into the register.
2185 Label current;
2186 call(¤t);
2187 int pc = pc_offset();
2188 bind(¤t);
2189 pop(dst);
2190 if (pc != 0) {
2191 sub(dst, Immediate(pc));
2192 }
2193 }
2194
CallForDeoptimization(Builtins::Name target,int,Label * exit,DeoptimizeKind kind,Label *)2195 void TurboAssembler::CallForDeoptimization(Builtins::Name target, int,
2196 Label* exit, DeoptimizeKind kind,
2197 Label*) {
2198 CallBuiltin(target);
2199 DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
2200 (kind == DeoptimizeKind::kLazy)
2201 ? Deoptimizer::kLazyDeoptExitSize
2202 : Deoptimizer::kNonLazyDeoptExitSize);
2203 USE(exit, kind);
2204 }
2205
Trap()2206 void TurboAssembler::Trap() { int3(); }
DebugBreak()2207 void TurboAssembler::DebugBreak() { int3(); }
2208
2209 } // namespace internal
2210 } // namespace v8
2211
2212 #endif // V8_TARGET_ARCH_IA32
2213