1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/arm64/assembler-arm64-inl.h"
6 #include "src/codegen/arm64/macro-assembler-arm64-inl.h"
7 #include "src/codegen/optimized-compilation-info.h"
8 #include "src/compiler/backend/code-generator-impl.h"
9 #include "src/compiler/backend/code-generator.h"
10 #include "src/compiler/backend/gap-resolver.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/osr.h"
13 #include "src/execution/frame-constants.h"
14 #include "src/heap/memory-chunk.h"
15
16 #if V8_ENABLE_WEBASSEMBLY
17 #include "src/wasm/wasm-code-manager.h"
18 #include "src/wasm/wasm-objects.h"
19 #endif // V8_ENABLE_WEBASSEMBLY
20
21 namespace v8 {
22 namespace internal {
23 namespace compiler {
24
25 #define __ tasm()->
26
27 // Adds Arm64-specific methods to convert InstructionOperands.
28 class Arm64OperandConverter final : public InstructionOperandConverter {
29 public:
Arm64OperandConverter(CodeGenerator * gen,Instruction * instr)30 Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
31 : InstructionOperandConverter(gen, instr) {}
32
InputFloat32Register(size_t index)33 DoubleRegister InputFloat32Register(size_t index) {
34 return InputDoubleRegister(index).S();
35 }
36
InputFloat64Register(size_t index)37 DoubleRegister InputFloat64Register(size_t index) {
38 return InputDoubleRegister(index);
39 }
40
InputSimd128Register(size_t index)41 DoubleRegister InputSimd128Register(size_t index) {
42 return InputDoubleRegister(index).Q();
43 }
44
InputFloat32OrZeroRegister(size_t index)45 CPURegister InputFloat32OrZeroRegister(size_t index) {
46 if (instr_->InputAt(index)->IsImmediate()) {
47 DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
48 return wzr;
49 }
50 DCHECK(instr_->InputAt(index)->IsFPRegister());
51 return InputDoubleRegister(index).S();
52 }
53
InputFloat64OrZeroRegister(size_t index)54 CPURegister InputFloat64OrZeroRegister(size_t index) {
55 if (instr_->InputAt(index)->IsImmediate()) {
56 DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
57 return xzr;
58 }
59 DCHECK(instr_->InputAt(index)->IsDoubleRegister());
60 return InputDoubleRegister(index);
61 }
62
OutputCount()63 size_t OutputCount() { return instr_->OutputCount(); }
64
OutputFloat32Register()65 DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
66
OutputFloat64Register()67 DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
68
OutputSimd128Register()69 DoubleRegister OutputSimd128Register() { return OutputDoubleRegister().Q(); }
70
InputRegister32(size_t index)71 Register InputRegister32(size_t index) {
72 return ToRegister(instr_->InputAt(index)).W();
73 }
74
InputOrZeroRegister32(size_t index)75 Register InputOrZeroRegister32(size_t index) {
76 DCHECK(instr_->InputAt(index)->IsRegister() ||
77 (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
78 if (instr_->InputAt(index)->IsImmediate()) {
79 return wzr;
80 }
81 return InputRegister32(index);
82 }
83
InputRegister64(size_t index)84 Register InputRegister64(size_t index) { return InputRegister(index); }
85
InputOrZeroRegister64(size_t index)86 Register InputOrZeroRegister64(size_t index) {
87 DCHECK(instr_->InputAt(index)->IsRegister() ||
88 (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
89 if (instr_->InputAt(index)->IsImmediate()) {
90 return xzr;
91 }
92 return InputRegister64(index);
93 }
94
InputOperand(size_t index)95 Operand InputOperand(size_t index) {
96 return ToOperand(instr_->InputAt(index));
97 }
98
InputOperand64(size_t index)99 Operand InputOperand64(size_t index) { return InputOperand(index); }
100
InputOperand32(size_t index)101 Operand InputOperand32(size_t index) {
102 return ToOperand32(instr_->InputAt(index));
103 }
104
OutputRegister64()105 Register OutputRegister64() { return OutputRegister(); }
106
OutputRegister32()107 Register OutputRegister32() { return OutputRegister().W(); }
108
TempRegister32(size_t index)109 Register TempRegister32(size_t index) {
110 return ToRegister(instr_->TempAt(index)).W();
111 }
112
InputOperand2_32(size_t index)113 Operand InputOperand2_32(size_t index) {
114 switch (AddressingModeField::decode(instr_->opcode())) {
115 case kMode_None:
116 return InputOperand32(index);
117 case kMode_Operand2_R_LSL_I:
118 return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
119 case kMode_Operand2_R_LSR_I:
120 return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
121 case kMode_Operand2_R_ASR_I:
122 return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
123 case kMode_Operand2_R_ROR_I:
124 return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
125 case kMode_Operand2_R_UXTB:
126 return Operand(InputRegister32(index), UXTB);
127 case kMode_Operand2_R_UXTH:
128 return Operand(InputRegister32(index), UXTH);
129 case kMode_Operand2_R_SXTB:
130 return Operand(InputRegister32(index), SXTB);
131 case kMode_Operand2_R_SXTH:
132 return Operand(InputRegister32(index), SXTH);
133 case kMode_Operand2_R_SXTW:
134 return Operand(InputRegister32(index), SXTW);
135 case kMode_MRI:
136 case kMode_MRR:
137 case kMode_Root:
138 break;
139 }
140 UNREACHABLE();
141 }
142
InputOperand2_64(size_t index)143 Operand InputOperand2_64(size_t index) {
144 switch (AddressingModeField::decode(instr_->opcode())) {
145 case kMode_None:
146 return InputOperand64(index);
147 case kMode_Operand2_R_LSL_I:
148 return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
149 case kMode_Operand2_R_LSR_I:
150 return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
151 case kMode_Operand2_R_ASR_I:
152 return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
153 case kMode_Operand2_R_ROR_I:
154 return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
155 case kMode_Operand2_R_UXTB:
156 return Operand(InputRegister64(index), UXTB);
157 case kMode_Operand2_R_UXTH:
158 return Operand(InputRegister64(index), UXTH);
159 case kMode_Operand2_R_SXTB:
160 return Operand(InputRegister64(index), SXTB);
161 case kMode_Operand2_R_SXTH:
162 return Operand(InputRegister64(index), SXTH);
163 case kMode_Operand2_R_SXTW:
164 return Operand(InputRegister64(index), SXTW);
165 case kMode_MRI:
166 case kMode_MRR:
167 case kMode_Root:
168 break;
169 }
170 UNREACHABLE();
171 }
172
MemoryOperand(size_t index=0)173 MemOperand MemoryOperand(size_t index = 0) {
174 switch (AddressingModeField::decode(instr_->opcode())) {
175 case kMode_None:
176 case kMode_Operand2_R_LSR_I:
177 case kMode_Operand2_R_ASR_I:
178 case kMode_Operand2_R_ROR_I:
179 case kMode_Operand2_R_UXTB:
180 case kMode_Operand2_R_UXTH:
181 case kMode_Operand2_R_SXTB:
182 case kMode_Operand2_R_SXTH:
183 case kMode_Operand2_R_SXTW:
184 break;
185 case kMode_Root:
186 return MemOperand(kRootRegister, InputInt64(index));
187 case kMode_Operand2_R_LSL_I:
188 return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
189 LSL, InputInt32(index + 2));
190 case kMode_MRI:
191 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
192 case kMode_MRR:
193 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
194 }
195 UNREACHABLE();
196 }
197
ToOperand(InstructionOperand * op)198 Operand ToOperand(InstructionOperand* op) {
199 if (op->IsRegister()) {
200 return Operand(ToRegister(op));
201 }
202 return ToImmediate(op);
203 }
204
ToOperand32(InstructionOperand * op)205 Operand ToOperand32(InstructionOperand* op) {
206 if (op->IsRegister()) {
207 return Operand(ToRegister(op).W());
208 }
209 return ToImmediate(op);
210 }
211
ToImmediate(InstructionOperand * operand)212 Operand ToImmediate(InstructionOperand* operand) {
213 Constant constant = ToConstant(operand);
214 switch (constant.type()) {
215 case Constant::kInt32:
216 return Operand(constant.ToInt32());
217 case Constant::kInt64:
218 #if V8_ENABLE_WEBASSEMBLY
219 if (RelocInfo::IsWasmReference(constant.rmode())) {
220 return Operand(constant.ToInt64(), constant.rmode());
221 }
222 #endif // V8_ENABLE_WEBASSEMBLY
223 return Operand(constant.ToInt64());
224 case Constant::kFloat32:
225 return Operand(Operand::EmbeddedNumber(constant.ToFloat32()));
226 case Constant::kFloat64:
227 return Operand(Operand::EmbeddedNumber(constant.ToFloat64().value()));
228 case Constant::kExternalReference:
229 return Operand(constant.ToExternalReference());
230 case Constant::kCompressedHeapObject: // Fall through.
231 case Constant::kHeapObject:
232 return Operand(constant.ToHeapObject());
233 case Constant::kDelayedStringConstant:
234 return Operand::EmbeddedStringConstant(
235 constant.ToDelayedStringConstant());
236 case Constant::kRpoNumber:
237 UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
238 }
239 UNREACHABLE();
240 }
241
ToMemOperand(InstructionOperand * op,TurboAssembler * tasm) const242 MemOperand ToMemOperand(InstructionOperand* op, TurboAssembler* tasm) const {
243 DCHECK_NOT_NULL(op);
244 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
245 return SlotToMemOperand(AllocatedOperand::cast(op)->index(), tasm);
246 }
247
SlotToMemOperand(int slot,TurboAssembler * tasm) const248 MemOperand SlotToMemOperand(int slot, TurboAssembler* tasm) const {
249 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
250 if (offset.from_frame_pointer()) {
251 int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
252 // Convert FP-offsets to SP-offsets if it results in better code.
253 if (Assembler::IsImmLSUnscaled(from_sp) ||
254 Assembler::IsImmLSScaled(from_sp, 3)) {
255 offset = FrameOffset::FromStackPointer(from_sp);
256 }
257 }
258 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
259 }
260 };
261
262 namespace {
263
264 class OutOfLineRecordWrite final : public OutOfLineCode {
265 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand offset,Register value,RecordWriteMode mode,StubCallMode stub_mode,UnwindingInfoWriter * unwinding_info_writer)266 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand offset,
267 Register value, RecordWriteMode mode,
268 StubCallMode stub_mode,
269 UnwindingInfoWriter* unwinding_info_writer)
270 : OutOfLineCode(gen),
271 object_(object),
272 offset_(offset),
273 value_(value),
274 mode_(mode),
275 #if V8_ENABLE_WEBASSEMBLY
276 stub_mode_(stub_mode),
277 #endif // V8_ENABLE_WEBASSEMBLY
278 must_save_lr_(!gen->frame_access_state()->has_frame()),
279 unwinding_info_writer_(unwinding_info_writer),
280 zone_(gen->zone()) {
281 }
282
Generate()283 void Generate() final {
284 if (COMPRESS_POINTERS_BOOL) {
285 __ DecompressTaggedPointer(value_, value_);
286 }
287 __ CheckPageFlag(value_, MemoryChunk::kPointersToHereAreInterestingMask, ne,
288 exit());
289 RememberedSetAction const remembered_set_action =
290 mode_ > RecordWriteMode::kValueIsMap ? RememberedSetAction::kEmit
291 : RememberedSetAction::kOmit;
292 SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
293 ? SaveFPRegsMode::kSave
294 : SaveFPRegsMode::kIgnore;
295 if (must_save_lr_) {
296 // We need to save and restore lr if the frame was elided.
297 __ Push<TurboAssembler::kSignLR>(lr, padreg);
298 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), sp);
299 }
300 if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
301 __ CallEphemeronKeyBarrier(object_, offset_, save_fp_mode);
302 #if V8_ENABLE_WEBASSEMBLY
303 } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
304 // A direct call to a wasm runtime stub defined in this module.
305 // Just encode the stub index. This will be patched when the code
306 // is added to the native module and copied into wasm code space.
307 __ CallRecordWriteStubSaveRegisters(object_, offset_,
308 remembered_set_action, save_fp_mode,
309 StubCallMode::kCallWasmRuntimeStub);
310 #endif // V8_ENABLE_WEBASSEMBLY
311 } else {
312 __ CallRecordWriteStubSaveRegisters(object_, offset_,
313 remembered_set_action, save_fp_mode);
314 }
315 if (must_save_lr_) {
316 __ Pop<TurboAssembler::kAuthLR>(padreg, lr);
317 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
318 }
319 }
320
321 private:
322 Register const object_;
323 Operand const offset_;
324 Register const value_;
325 RecordWriteMode const mode_;
326 #if V8_ENABLE_WEBASSEMBLY
327 StubCallMode const stub_mode_;
328 #endif // V8_ENABLE_WEBASSEMBLY
329 bool must_save_lr_;
330 UnwindingInfoWriter* const unwinding_info_writer_;
331 Zone* zone_;
332 };
333
FlagsConditionToCondition(FlagsCondition condition)334 Condition FlagsConditionToCondition(FlagsCondition condition) {
335 switch (condition) {
336 case kEqual:
337 return eq;
338 case kNotEqual:
339 return ne;
340 case kSignedLessThan:
341 return lt;
342 case kSignedGreaterThanOrEqual:
343 return ge;
344 case kSignedLessThanOrEqual:
345 return le;
346 case kSignedGreaterThan:
347 return gt;
348 case kUnsignedLessThan:
349 return lo;
350 case kUnsignedGreaterThanOrEqual:
351 return hs;
352 case kUnsignedLessThanOrEqual:
353 return ls;
354 case kUnsignedGreaterThan:
355 return hi;
356 case kFloatLessThanOrUnordered:
357 return lt;
358 case kFloatGreaterThanOrEqual:
359 return ge;
360 case kFloatLessThanOrEqual:
361 return ls;
362 case kFloatGreaterThanOrUnordered:
363 return hi;
364 case kFloatLessThan:
365 return lo;
366 case kFloatGreaterThanOrEqualOrUnordered:
367 return hs;
368 case kFloatLessThanOrEqualOrUnordered:
369 return le;
370 case kFloatGreaterThan:
371 return gt;
372 case kOverflow:
373 return vs;
374 case kNotOverflow:
375 return vc;
376 case kUnorderedEqual:
377 case kUnorderedNotEqual:
378 break;
379 case kPositiveOrZero:
380 return pl;
381 case kNegative:
382 return mi;
383 }
384 UNREACHABLE();
385 }
386
387 #if V8_ENABLE_WEBASSEMBLY
388 class WasmOutOfLineTrap : public OutOfLineCode {
389 public:
WasmOutOfLineTrap(CodeGenerator * gen,Instruction * instr)390 WasmOutOfLineTrap(CodeGenerator* gen, Instruction* instr)
391 : OutOfLineCode(gen), gen_(gen), instr_(instr) {}
Generate()392 void Generate() override {
393 Arm64OperandConverter i(gen_, instr_);
394 TrapId trap_id =
395 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
396 GenerateCallToTrap(trap_id);
397 }
398
399 protected:
400 CodeGenerator* gen_;
401
GenerateWithTrapId(TrapId trap_id)402 void GenerateWithTrapId(TrapId trap_id) { GenerateCallToTrap(trap_id); }
403
404 private:
GenerateCallToTrap(TrapId trap_id)405 void GenerateCallToTrap(TrapId trap_id) {
406 if (!gen_->wasm_runtime_exception_support()) {
407 // We cannot test calls to the runtime in cctest/test-run-wasm.
408 // Therefore we emit a call to C here instead of a call to the runtime.
409 __ CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(),
410 0);
411 __ LeaveFrame(StackFrame::WASM);
412 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
413 int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
414 pop_count += (pop_count & 1); // align
415 __ Drop(pop_count);
416 __ Ret();
417 } else {
418 gen_->AssembleSourcePosition(instr_);
419 // A direct call to a wasm runtime stub defined in this module.
420 // Just encode the stub index. This will be patched when the code
421 // is added to the native module and copied into wasm code space.
422 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
423 ReferenceMap* reference_map =
424 gen_->zone()->New<ReferenceMap>(gen_->zone());
425 gen_->RecordSafepoint(reference_map);
426 __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
427 }
428 }
429
430 Instruction* instr_;
431 };
432
433 class WasmProtectedInstructionTrap final : public WasmOutOfLineTrap {
434 public:
WasmProtectedInstructionTrap(CodeGenerator * gen,int pc,Instruction * instr)435 WasmProtectedInstructionTrap(CodeGenerator* gen, int pc, Instruction* instr)
436 : WasmOutOfLineTrap(gen, instr), pc_(pc) {}
437
Generate()438 void Generate() override {
439 DCHECK(FLAG_wasm_bounds_checks && !FLAG_wasm_enforce_bounds_checks);
440 gen_->AddProtectedInstructionLanding(pc_, __ pc_offset());
441 GenerateWithTrapId(TrapId::kTrapMemOutOfBounds);
442 }
443
444 private:
445 int pc_;
446 };
447
EmitOOLTrapIfNeeded(Zone * zone,CodeGenerator * codegen,InstructionCode opcode,Instruction * instr,int pc)448 void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
449 InstructionCode opcode, Instruction* instr, int pc) {
450 const MemoryAccessMode access_mode = AccessModeField::decode(opcode);
451 if (access_mode == kMemoryAccessProtected) {
452 zone->New<WasmProtectedInstructionTrap>(codegen, pc, instr);
453 }
454 }
455 #else
EmitOOLTrapIfNeeded(Zone * zone,CodeGenerator * codegen,InstructionCode opcode,Instruction * instr,int pc)456 void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
457 InstructionCode opcode, Instruction* instr, int pc) {
458 DCHECK_NE(kMemoryAccessProtected, AccessModeField::decode(opcode));
459 }
460 #endif // V8_ENABLE_WEBASSEMBLY
461
462 // Handles unary ops that work for float (scalar), double (scalar), or NEON.
463 template <typename Fn>
EmitFpOrNeonUnop(TurboAssembler * tasm,Fn fn,Instruction * instr,Arm64OperandConverter i,VectorFormat scalar,VectorFormat vector)464 void EmitFpOrNeonUnop(TurboAssembler* tasm, Fn fn, Instruction* instr,
465 Arm64OperandConverter i, VectorFormat scalar,
466 VectorFormat vector) {
467 VectorFormat f = instr->InputAt(0)->IsSimd128Register() ? vector : scalar;
468
469 VRegister output = VRegister::Create(i.OutputDoubleRegister().code(), f);
470 VRegister input = VRegister::Create(i.InputDoubleRegister(0).code(), f);
471 (tasm->*fn)(output, input);
472 }
473
474 } // namespace
475
476 #define ASSEMBLE_SHIFT(asm_instr, width) \
477 do { \
478 if (instr->InputAt(1)->IsRegister()) { \
479 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
480 i.InputRegister##width(1)); \
481 } else { \
482 uint32_t imm = \
483 static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
484 __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
485 imm % (width)); \
486 } \
487 } while (0)
488
489 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr, reg) \
490 do { \
491 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
492 __ asm_instr(i.Output##reg(), i.TempRegister(0)); \
493 } while (0)
494
495 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr, reg) \
496 do { \
497 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
498 __ asm_instr(i.Input##reg(2), i.TempRegister(0)); \
499 } while (0)
500
501 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr, reg) \
502 do { \
503 Label exchange; \
504 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
505 __ Bind(&exchange); \
506 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
507 __ store_instr(i.TempRegister32(1), i.Input##reg(2), i.TempRegister(0)); \
508 __ Cbnz(i.TempRegister32(1), &exchange); \
509 } while (0)
510
511 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_instr, store_instr, ext, \
512 reg) \
513 do { \
514 Label compareExchange; \
515 Label exit; \
516 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
517 __ Bind(&compareExchange); \
518 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
519 __ Cmp(i.Output##reg(), Operand(i.Input##reg(2), ext)); \
520 __ B(ne, &exit); \
521 __ store_instr(i.TempRegister32(1), i.Input##reg(3), i.TempRegister(0)); \
522 __ Cbnz(i.TempRegister32(1), &compareExchange); \
523 __ Bind(&exit); \
524 } while (0)
525
526 #define ASSEMBLE_ATOMIC_BINOP(load_instr, store_instr, bin_instr, reg) \
527 do { \
528 Label binop; \
529 __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
530 __ Bind(&binop); \
531 __ load_instr(i.Output##reg(), i.TempRegister(0)); \
532 __ bin_instr(i.Temp##reg(1), i.Output##reg(), Operand(i.Input##reg(2))); \
533 __ store_instr(i.TempRegister32(2), i.Temp##reg(1), i.TempRegister(0)); \
534 __ Cbnz(i.TempRegister32(2), &binop); \
535 } while (0)
536
537 #define ASSEMBLE_IEEE754_BINOP(name) \
538 do { \
539 FrameScope scope(tasm(), StackFrame::MANUAL); \
540 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
541 } while (0)
542
543 #define ASSEMBLE_IEEE754_UNOP(name) \
544 do { \
545 FrameScope scope(tasm(), StackFrame::MANUAL); \
546 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
547 } while (0)
548
549 // If shift value is an immediate, we can call asm_imm, taking the shift value
550 // modulo 2^width. Otherwise, emit code to perform the modulus operation, and
551 // call asm_shl.
552 #define ASSEMBLE_SIMD_SHIFT_LEFT(asm_imm, width, format, asm_shl, gp) \
553 do { \
554 if (instr->InputAt(1)->IsImmediate()) { \
555 __ asm_imm(i.OutputSimd128Register().format(), \
556 i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
557 } else { \
558 UseScratchRegisterScope temps(tasm()); \
559 VRegister tmp = temps.AcquireQ(); \
560 Register shift = temps.Acquire##gp(); \
561 constexpr int mask = (1 << width) - 1; \
562 __ And(shift, i.InputRegister32(1), mask); \
563 __ Dup(tmp.format(), shift); \
564 __ asm_shl(i.OutputSimd128Register().format(), \
565 i.InputSimd128Register(0).format(), tmp.format()); \
566 } \
567 } while (0)
568
569 // If shift value is an immediate, we can call asm_imm, taking the shift value
570 // modulo 2^width. Otherwise, emit code to perform the modulus operation, and
571 // call asm_shl, passing in the negative shift value (treated as right shift).
572 #define ASSEMBLE_SIMD_SHIFT_RIGHT(asm_imm, width, format, asm_shl, gp) \
573 do { \
574 if (instr->InputAt(1)->IsImmediate()) { \
575 __ asm_imm(i.OutputSimd128Register().format(), \
576 i.InputSimd128Register(0).format(), i.InputInt##width(1)); \
577 } else { \
578 UseScratchRegisterScope temps(tasm()); \
579 VRegister tmp = temps.AcquireQ(); \
580 Register shift = temps.Acquire##gp(); \
581 constexpr int mask = (1 << width) - 1; \
582 __ And(shift, i.InputRegister32(1), mask); \
583 __ Dup(tmp.format(), shift); \
584 __ Neg(tmp.format(), tmp.format()); \
585 __ asm_shl(i.OutputSimd128Register().format(), \
586 i.InputSimd128Register(0).format(), tmp.format()); \
587 } \
588 } while (0)
589
AssembleDeconstructFrame()590 void CodeGenerator::AssembleDeconstructFrame() {
591 __ Mov(sp, fp);
592 __ Pop<TurboAssembler::kAuthLR>(fp, lr);
593
594 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
595 }
596
AssemblePrepareTailCall()597 void CodeGenerator::AssemblePrepareTailCall() {
598 if (frame_access_state()->has_frame()) {
599 __ RestoreFPAndLR();
600 }
601 frame_access_state()->SetFrameAccessToSP();
602 }
603
604 namespace {
605
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)606 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
607 FrameAccessState* state,
608 int new_slot_above_sp,
609 bool allow_shrinkage = true) {
610 int current_sp_offset = state->GetSPToFPSlotCount() +
611 StandardFrameConstants::kFixedSlotCountAboveFp;
612 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
613 DCHECK_EQ(stack_slot_delta % 2, 0);
614 if (stack_slot_delta > 0) {
615 tasm->Claim(stack_slot_delta);
616 state->IncreaseSPDelta(stack_slot_delta);
617 } else if (allow_shrinkage && stack_slot_delta < 0) {
618 tasm->Drop(-stack_slot_delta);
619 state->IncreaseSPDelta(stack_slot_delta);
620 }
621 }
622
623 } // namespace
624
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_slot_offset)625 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
626 int first_unused_slot_offset) {
627 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
628 first_unused_slot_offset, false);
629 }
630
AssembleTailCallAfterGap(Instruction * instr,int first_unused_slot_offset)631 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
632 int first_unused_slot_offset) {
633 DCHECK_EQ(first_unused_slot_offset % 2, 0);
634 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
635 first_unused_slot_offset);
636 DCHECK(instr->IsTailCall());
637 InstructionOperandConverter g(this, instr);
638 int optional_padding_offset = g.InputInt32(instr->InputCount() - 2);
639 if (optional_padding_offset % 2) {
640 __ Poke(padreg, optional_padding_offset * kSystemPointerSize);
641 }
642 }
643
644 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()645 void CodeGenerator::AssembleCodeStartRegisterCheck() {
646 UseScratchRegisterScope temps(tasm());
647 Register scratch = temps.AcquireX();
648 __ ComputeCodeStartAddress(scratch);
649 __ cmp(scratch, kJavaScriptCallCodeStartRegister);
650 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
651 }
652
653 // Check if the code object is marked for deoptimization. If it is, then it
654 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
655 // to:
656 // 1. read from memory the word that contains that bit, which can be found in
657 // the flags in the referenced {CodeDataContainer} object;
658 // 2. test kMarkedForDeoptimizationBit in those flags; and
659 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()660 void CodeGenerator::BailoutIfDeoptimized() {
661 UseScratchRegisterScope temps(tasm());
662 Register scratch = temps.AcquireX();
663 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
664 __ LoadTaggedPointerField(
665 scratch, MemOperand(kJavaScriptCallCodeStartRegister, offset));
666 __ Ldr(scratch.W(),
667 FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
668 Label not_deoptimized;
669 __ Tbz(scratch.W(), Code::kMarkedForDeoptimizationBit, ¬_deoptimized);
670 __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
671 RelocInfo::CODE_TARGET);
672 __ Bind(¬_deoptimized);
673 }
674
675 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)676 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
677 Instruction* instr) {
678 Arm64OperandConverter i(this, instr);
679 InstructionCode opcode = instr->opcode();
680 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
681 switch (arch_opcode) {
682 case kArchCallCodeObject: {
683 if (instr->InputAt(0)->IsImmediate()) {
684 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
685 } else {
686 Register reg = i.InputRegister(0);
687 DCHECK_IMPLIES(
688 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
689 reg == kJavaScriptCallCodeStartRegister);
690 __ CallCodeObject(reg);
691 }
692 RecordCallPosition(instr);
693 frame_access_state()->ClearSPDelta();
694 break;
695 }
696 case kArchCallBuiltinPointer: {
697 DCHECK(!instr->InputAt(0)->IsImmediate());
698 Register builtin_index = i.InputRegister(0);
699 __ CallBuiltinByIndex(builtin_index);
700 RecordCallPosition(instr);
701 frame_access_state()->ClearSPDelta();
702 break;
703 }
704 #if V8_ENABLE_WEBASSEMBLY
705 case kArchCallWasmFunction: {
706 if (instr->InputAt(0)->IsImmediate()) {
707 Constant constant = i.ToConstant(instr->InputAt(0));
708 Address wasm_code = static_cast<Address>(constant.ToInt64());
709 __ Call(wasm_code, constant.rmode());
710 } else {
711 Register target = i.InputRegister(0);
712 __ Call(target);
713 }
714 RecordCallPosition(instr);
715 frame_access_state()->ClearSPDelta();
716 break;
717 }
718 case kArchTailCallWasm: {
719 if (instr->InputAt(0)->IsImmediate()) {
720 Constant constant = i.ToConstant(instr->InputAt(0));
721 Address wasm_code = static_cast<Address>(constant.ToInt64());
722 __ Jump(wasm_code, constant.rmode());
723 } else {
724 Register target = i.InputRegister(0);
725 UseScratchRegisterScope temps(tasm());
726 temps.Exclude(x17);
727 __ Mov(x17, target);
728 __ Jump(x17);
729 }
730 unwinding_info_writer_.MarkBlockWillExit();
731 frame_access_state()->ClearSPDelta();
732 frame_access_state()->SetFrameAccessToDefault();
733 break;
734 }
735 #endif // V8_ENABLE_WEBASSEMBLY
736 case kArchTailCallCodeObject: {
737 if (instr->InputAt(0)->IsImmediate()) {
738 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
739 } else {
740 Register reg = i.InputRegister(0);
741 DCHECK_IMPLIES(
742 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
743 reg == kJavaScriptCallCodeStartRegister);
744 __ JumpCodeObject(reg);
745 }
746 unwinding_info_writer_.MarkBlockWillExit();
747 frame_access_state()->ClearSPDelta();
748 frame_access_state()->SetFrameAccessToDefault();
749 break;
750 }
751 case kArchTailCallAddress: {
752 CHECK(!instr->InputAt(0)->IsImmediate());
753 Register reg = i.InputRegister(0);
754 DCHECK_IMPLIES(
755 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
756 reg == kJavaScriptCallCodeStartRegister);
757 UseScratchRegisterScope temps(tasm());
758 temps.Exclude(x17);
759 __ Mov(x17, reg);
760 __ Jump(x17);
761 unwinding_info_writer_.MarkBlockWillExit();
762 frame_access_state()->ClearSPDelta();
763 frame_access_state()->SetFrameAccessToDefault();
764 break;
765 }
766 case kArchCallJSFunction: {
767 Register func = i.InputRegister(0);
768 if (FLAG_debug_code) {
769 // Check the function's context matches the context argument.
770 UseScratchRegisterScope scope(tasm());
771 Register temp = scope.AcquireX();
772 __ LoadTaggedPointerField(
773 temp, FieldMemOperand(func, JSFunction::kContextOffset));
774 __ cmp(cp, temp);
775 __ Assert(eq, AbortReason::kWrongFunctionContext);
776 }
777 static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
778 __ LoadTaggedPointerField(x2,
779 FieldMemOperand(func, JSFunction::kCodeOffset));
780 __ CallCodeTObject(x2);
781 RecordCallPosition(instr);
782 frame_access_state()->ClearSPDelta();
783 break;
784 }
785 case kArchPrepareCallCFunction:
786 // We don't need kArchPrepareCallCFunction on arm64 as the instruction
787 // selector has already performed a Claim to reserve space on the stack.
788 // Frame alignment is always 16 bytes, and the stack pointer is already
789 // 16-byte aligned, therefore we do not need to align the stack pointer
790 // by an unknown value, and it is safe to continue accessing the frame
791 // via the stack pointer.
792 UNREACHABLE();
793 case kArchSaveCallerRegisters: {
794 fp_mode_ =
795 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
796 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
797 fp_mode_ == SaveFPRegsMode::kSave);
798 // kReturnRegister0 should have been saved before entering the stub.
799 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
800 DCHECK(IsAligned(bytes, kSystemPointerSize));
801 DCHECK_EQ(0, frame_access_state()->sp_delta());
802 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
803 DCHECK(!caller_registers_saved_);
804 caller_registers_saved_ = true;
805 break;
806 }
807 case kArchRestoreCallerRegisters: {
808 DCHECK(fp_mode_ ==
809 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
810 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
811 fp_mode_ == SaveFPRegsMode::kSave);
812 // Don't overwrite the returned value.
813 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
814 frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
815 DCHECK_EQ(0, frame_access_state()->sp_delta());
816 DCHECK(caller_registers_saved_);
817 caller_registers_saved_ = false;
818 break;
819 }
820 case kArchPrepareTailCall:
821 AssemblePrepareTailCall();
822 break;
823 case kArchCallCFunction: {
824 int const num_gp_parameters = ParamField::decode(instr->opcode());
825 int const num_fp_parameters = FPParamField::decode(instr->opcode());
826 Label return_location;
827 #if V8_ENABLE_WEBASSEMBLY
828 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
829 // Put the return address in a stack slot.
830 __ StoreReturnAddressInWasmExitFrame(&return_location);
831 }
832 #endif // V8_ENABLE_WEBASSEMBLY
833
834 if (instr->InputAt(0)->IsImmediate()) {
835 ExternalReference ref = i.InputExternalReference(0);
836 __ CallCFunction(ref, num_gp_parameters, num_fp_parameters);
837 } else {
838 Register func = i.InputRegister(0);
839 __ CallCFunction(func, num_gp_parameters, num_fp_parameters);
840 }
841 __ Bind(&return_location);
842 #if V8_ENABLE_WEBASSEMBLY
843 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
844 RecordSafepoint(instr->reference_map());
845 }
846 #endif // V8_ENABLE_WEBASSEMBLY
847 frame_access_state()->SetFrameAccessToDefault();
848 // Ideally, we should decrement SP delta to match the change of stack
849 // pointer in CallCFunction. However, for certain architectures (e.g.
850 // ARM), there may be more strict alignment requirement, causing old SP
851 // to be saved on the stack. In those cases, we can not calculate the SP
852 // delta statically.
853 frame_access_state()->ClearSPDelta();
854 if (caller_registers_saved_) {
855 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
856 // Here, we assume the sequence to be:
857 // kArchSaveCallerRegisters;
858 // kArchCallCFunction;
859 // kArchRestoreCallerRegisters;
860 int bytes =
861 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
862 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
863 }
864 break;
865 }
866 case kArchJmp:
867 AssembleArchJump(i.InputRpo(0));
868 break;
869 case kArchTableSwitch:
870 AssembleArchTableSwitch(instr);
871 break;
872 case kArchBinarySearchSwitch:
873 AssembleArchBinarySearchSwitch(instr);
874 break;
875 case kArchAbortCSADcheck:
876 DCHECK_EQ(i.InputRegister(0), x1);
877 {
878 // We don't actually want to generate a pile of code for this, so just
879 // claim there is a stack frame, without generating one.
880 FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
881 __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck),
882 RelocInfo::CODE_TARGET);
883 }
884 __ Debug("kArchAbortCSADcheck", 0, BREAK);
885 unwinding_info_writer_.MarkBlockWillExit();
886 break;
887 case kArchDebugBreak:
888 __ DebugBreak();
889 break;
890 case kArchComment:
891 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
892 break;
893 case kArchThrowTerminator:
894 unwinding_info_writer_.MarkBlockWillExit();
895 break;
896 case kArchNop:
897 // don't emit code for nops.
898 break;
899 case kArchDeoptimize: {
900 DeoptimizationExit* exit =
901 BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
902 __ B(exit->label());
903 break;
904 }
905 case kArchRet:
906 AssembleReturn(instr->InputAt(0));
907 break;
908 case kArchFramePointer:
909 __ mov(i.OutputRegister(), fp);
910 break;
911 case kArchParentFramePointer:
912 if (frame_access_state()->has_frame()) {
913 __ ldr(i.OutputRegister(), MemOperand(fp, 0));
914 } else {
915 __ mov(i.OutputRegister(), fp);
916 }
917 break;
918 case kArchStackPointerGreaterThan: {
919 // Potentially apply an offset to the current stack pointer before the
920 // comparison to consider the size difference of an optimized frame versus
921 // the contained unoptimized frames.
922
923 Register lhs_register = sp;
924 uint32_t offset;
925
926 if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
927 lhs_register = i.TempRegister(0);
928 __ Sub(lhs_register, sp, offset);
929 }
930
931 constexpr size_t kValueIndex = 0;
932 DCHECK(instr->InputAt(kValueIndex)->IsRegister());
933 __ Cmp(lhs_register, i.InputRegister(kValueIndex));
934 break;
935 }
936 case kArchStackCheckOffset:
937 __ Move(i.OutputRegister(), Smi::FromInt(GetStackCheckOffset()));
938 break;
939 case kArchTruncateDoubleToI:
940 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
941 i.InputDoubleRegister(0), DetermineStubCallMode(),
942 frame_access_state()->has_frame()
943 ? kLRHasBeenSaved
944 : kLRHasNotBeenSaved);
945
946 break;
947 case kArchStoreWithWriteBarrier: {
948 RecordWriteMode mode =
949 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
950 AddressingMode addressing_mode =
951 AddressingModeField::decode(instr->opcode());
952 Register object = i.InputRegister(0);
953 Operand offset(0);
954 if (addressing_mode == kMode_MRI) {
955 offset = Operand(i.InputInt64(1));
956 } else {
957 DCHECK_EQ(addressing_mode, kMode_MRR);
958 offset = Operand(i.InputRegister(1));
959 }
960 Register value = i.InputRegister(2);
961 auto ool = zone()->New<OutOfLineRecordWrite>(
962 this, object, offset, value, mode, DetermineStubCallMode(),
963 &unwinding_info_writer_);
964 __ StoreTaggedField(value, MemOperand(object, offset));
965 if (mode > RecordWriteMode::kValueIsPointer) {
966 __ JumpIfSmi(value, ool->exit());
967 }
968 __ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask,
969 eq, ool->entry());
970 __ Bind(ool->exit());
971 break;
972 }
973 case kArchAtomicStoreWithWriteBarrier: {
974 DCHECK_EQ(AddressingModeField::decode(instr->opcode()), kMode_MRR);
975 RecordWriteMode mode =
976 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
977 Register object = i.InputRegister(0);
978 Register offset = i.InputRegister(1);
979 Register value = i.InputRegister(2);
980 auto ool = zone()->New<OutOfLineRecordWrite>(
981 this, object, offset, value, mode, DetermineStubCallMode(),
982 &unwinding_info_writer_);
983 __ AtomicStoreTaggedField(value, object, offset, i.TempRegister(0));
984 if (mode > RecordWriteMode::kValueIsPointer) {
985 __ JumpIfSmi(value, ool->exit());
986 }
987 __ CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask,
988 eq, ool->entry());
989 __ Bind(ool->exit());
990 break;
991 }
992 case kArchStackSlot: {
993 FrameOffset offset =
994 frame_access_state()->GetFrameOffset(i.InputInt32(0));
995 Register base = offset.from_stack_pointer() ? sp : fp;
996 __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
997 break;
998 }
999 case kIeee754Float64Acos:
1000 ASSEMBLE_IEEE754_UNOP(acos);
1001 break;
1002 case kIeee754Float64Acosh:
1003 ASSEMBLE_IEEE754_UNOP(acosh);
1004 break;
1005 case kIeee754Float64Asin:
1006 ASSEMBLE_IEEE754_UNOP(asin);
1007 break;
1008 case kIeee754Float64Asinh:
1009 ASSEMBLE_IEEE754_UNOP(asinh);
1010 break;
1011 case kIeee754Float64Atan:
1012 ASSEMBLE_IEEE754_UNOP(atan);
1013 break;
1014 case kIeee754Float64Atanh:
1015 ASSEMBLE_IEEE754_UNOP(atanh);
1016 break;
1017 case kIeee754Float64Atan2:
1018 ASSEMBLE_IEEE754_BINOP(atan2);
1019 break;
1020 case kIeee754Float64Cos:
1021 ASSEMBLE_IEEE754_UNOP(cos);
1022 break;
1023 case kIeee754Float64Cosh:
1024 ASSEMBLE_IEEE754_UNOP(cosh);
1025 break;
1026 case kIeee754Float64Cbrt:
1027 ASSEMBLE_IEEE754_UNOP(cbrt);
1028 break;
1029 case kIeee754Float64Exp:
1030 ASSEMBLE_IEEE754_UNOP(exp);
1031 break;
1032 case kIeee754Float64Expm1:
1033 ASSEMBLE_IEEE754_UNOP(expm1);
1034 break;
1035 case kIeee754Float64Log:
1036 ASSEMBLE_IEEE754_UNOP(log);
1037 break;
1038 case kIeee754Float64Log1p:
1039 ASSEMBLE_IEEE754_UNOP(log1p);
1040 break;
1041 case kIeee754Float64Log2:
1042 ASSEMBLE_IEEE754_UNOP(log2);
1043 break;
1044 case kIeee754Float64Log10:
1045 ASSEMBLE_IEEE754_UNOP(log10);
1046 break;
1047 case kIeee754Float64Pow:
1048 ASSEMBLE_IEEE754_BINOP(pow);
1049 break;
1050 case kIeee754Float64Sin:
1051 ASSEMBLE_IEEE754_UNOP(sin);
1052 break;
1053 case kIeee754Float64Sinh:
1054 ASSEMBLE_IEEE754_UNOP(sinh);
1055 break;
1056 case kIeee754Float64Tan:
1057 ASSEMBLE_IEEE754_UNOP(tan);
1058 break;
1059 case kIeee754Float64Tanh:
1060 ASSEMBLE_IEEE754_UNOP(tanh);
1061 break;
1062 case kArm64Float32RoundDown:
1063 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatS,
1064 kFormat4S);
1065 break;
1066 case kArm64Float64RoundDown:
1067 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintm, instr, i, kFormatD,
1068 kFormat2D);
1069 break;
1070 case kArm64Float32RoundUp:
1071 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatS,
1072 kFormat4S);
1073 break;
1074 case kArm64Float64RoundUp:
1075 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintp, instr, i, kFormatD,
1076 kFormat2D);
1077 break;
1078 case kArm64Float64RoundTiesAway:
1079 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frinta, instr, i, kFormatD,
1080 kFormat2D);
1081 break;
1082 case kArm64Float32RoundTruncate:
1083 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatS,
1084 kFormat4S);
1085 break;
1086 case kArm64Float64RoundTruncate:
1087 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintz, instr, i, kFormatD,
1088 kFormat2D);
1089 break;
1090 case kArm64Float32RoundTiesEven:
1091 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatS,
1092 kFormat4S);
1093 break;
1094 case kArm64Float64RoundTiesEven:
1095 EmitFpOrNeonUnop(tasm(), &TurboAssembler::Frintn, instr, i, kFormatD,
1096 kFormat2D);
1097 break;
1098 case kArm64Add:
1099 if (FlagsModeField::decode(opcode) != kFlags_none) {
1100 __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
1101 i.InputOperand2_64(1));
1102 } else {
1103 __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
1104 i.InputOperand2_64(1));
1105 }
1106 break;
1107 case kArm64Add32:
1108 if (FlagsModeField::decode(opcode) != kFlags_none) {
1109 __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1110 i.InputOperand2_32(1));
1111 } else {
1112 __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1113 i.InputOperand2_32(1));
1114 }
1115 break;
1116 case kArm64And:
1117 if (FlagsModeField::decode(opcode) != kFlags_none) {
1118 // The ands instruction only sets N and Z, so only the following
1119 // conditions make sense.
1120 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
1121 FlagsConditionField::decode(opcode) == kNotEqual ||
1122 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
1123 FlagsConditionField::decode(opcode) == kNegative);
1124 __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
1125 i.InputOperand2_64(1));
1126 } else {
1127 __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
1128 i.InputOperand2_64(1));
1129 }
1130 break;
1131 case kArm64And32:
1132 if (FlagsModeField::decode(opcode) != kFlags_none) {
1133 // The ands instruction only sets N and Z, so only the following
1134 // conditions make sense.
1135 DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
1136 FlagsConditionField::decode(opcode) == kNotEqual ||
1137 FlagsConditionField::decode(opcode) == kPositiveOrZero ||
1138 FlagsConditionField::decode(opcode) == kNegative);
1139 __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1140 i.InputOperand2_32(1));
1141 } else {
1142 __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1143 i.InputOperand2_32(1));
1144 }
1145 break;
1146 case kArm64Bic:
1147 __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
1148 i.InputOperand2_64(1));
1149 break;
1150 case kArm64Bic32:
1151 __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1152 i.InputOperand2_32(1));
1153 break;
1154 case kArm64Mul:
1155 __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1156 break;
1157 case kArm64Mul32:
1158 __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1159 break;
1160 case kArm64Sadalp: {
1161 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1162 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1163 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1164 __ Sadalp(i.OutputSimd128Register().Format(dst_f),
1165 i.InputSimd128Register(1).Format(src_f));
1166 break;
1167 }
1168 case kArm64Saddlp: {
1169 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1170 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1171 __ Saddlp(i.OutputSimd128Register().Format(dst_f),
1172 i.InputSimd128Register(0).Format(src_f));
1173 break;
1174 }
1175 case kArm64Uadalp: {
1176 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1177 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1178 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1179 __ Uadalp(i.OutputSimd128Register().Format(dst_f),
1180 i.InputSimd128Register(1).Format(src_f));
1181 break;
1182 }
1183 case kArm64Uaddlp: {
1184 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1185 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1186 __ Uaddlp(i.OutputSimd128Register().Format(dst_f),
1187 i.InputSimd128Register(0).Format(src_f));
1188 break;
1189 }
1190 case kArm64ISplat: {
1191 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1192 Register src = LaneSizeField::decode(opcode) == 64 ? i.InputRegister64(0)
1193 : i.InputRegister32(0);
1194 __ Dup(i.OutputSimd128Register().Format(f), src);
1195 break;
1196 }
1197 case kArm64FSplat: {
1198 VectorFormat src_f =
1199 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
1200 VectorFormat dst_f = VectorFormatFillQ(src_f);
1201 __ Dup(i.OutputSimd128Register().Format(dst_f),
1202 i.InputSimd128Register(0).Format(src_f), 0);
1203 break;
1204 }
1205 case kArm64Smlal: {
1206 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1207 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1208 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1209 __ Smlal(i.OutputSimd128Register().Format(dst_f),
1210 i.InputSimd128Register(1).Format(src_f),
1211 i.InputSimd128Register(2).Format(src_f));
1212 break;
1213 }
1214 case kArm64Smlal2: {
1215 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1216 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1217 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1218 __ Smlal2(i.OutputSimd128Register().Format(dst_f),
1219 i.InputSimd128Register(1).Format(src_f),
1220 i.InputSimd128Register(2).Format(src_f));
1221 break;
1222 }
1223 case kArm64Smull: {
1224 if (instr->InputAt(0)->IsRegister()) {
1225 __ Smull(i.OutputRegister(), i.InputRegister32(0),
1226 i.InputRegister32(1));
1227 } else {
1228 DCHECK(instr->InputAt(0)->IsSimd128Register());
1229 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1230 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1231 __ Smull(i.OutputSimd128Register().Format(dst_f),
1232 i.InputSimd128Register(0).Format(src_f),
1233 i.InputSimd128Register(1).Format(src_f));
1234 }
1235 break;
1236 }
1237 case kArm64Smull2: {
1238 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1239 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1240 __ Smull2(i.OutputSimd128Register().Format(dst_f),
1241 i.InputSimd128Register(0).Format(src_f),
1242 i.InputSimd128Register(1).Format(src_f));
1243 break;
1244 }
1245 case kArm64Umlal: {
1246 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1247 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1248 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1249 __ Umlal(i.OutputSimd128Register().Format(dst_f),
1250 i.InputSimd128Register(1).Format(src_f),
1251 i.InputSimd128Register(2).Format(src_f));
1252 break;
1253 }
1254 case kArm64Umlal2: {
1255 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1256 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1257 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
1258 __ Umlal2(i.OutputSimd128Register().Format(dst_f),
1259 i.InputSimd128Register(1).Format(src_f),
1260 i.InputSimd128Register(2).Format(src_f));
1261 break;
1262 }
1263 case kArm64Umull: {
1264 if (instr->InputAt(0)->IsRegister()) {
1265 __ Umull(i.OutputRegister(), i.InputRegister32(0),
1266 i.InputRegister32(1));
1267 } else {
1268 DCHECK(instr->InputAt(0)->IsSimd128Register());
1269 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1270 VectorFormat src_f = VectorFormatHalfWidth(dst_f);
1271 __ Umull(i.OutputSimd128Register().Format(dst_f),
1272 i.InputSimd128Register(0).Format(src_f),
1273 i.InputSimd128Register(1).Format(src_f));
1274 }
1275 break;
1276 }
1277 case kArm64Umull2: {
1278 VectorFormat dst_f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1279 VectorFormat src_f = VectorFormatHalfWidthDoubleLanes(dst_f);
1280 __ Umull2(i.OutputSimd128Register().Format(dst_f),
1281 i.InputSimd128Register(0).Format(src_f),
1282 i.InputSimd128Register(1).Format(src_f));
1283 break;
1284 }
1285 case kArm64Madd:
1286 __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1287 i.InputRegister(2));
1288 break;
1289 case kArm64Madd32:
1290 __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1291 i.InputRegister32(2));
1292 break;
1293 case kArm64Msub:
1294 __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1295 i.InputRegister(2));
1296 break;
1297 case kArm64Msub32:
1298 __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1299 i.InputRegister32(2));
1300 break;
1301 case kArm64Mneg:
1302 __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1303 break;
1304 case kArm64Mneg32:
1305 __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1306 break;
1307 case kArm64Idiv:
1308 __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1309 break;
1310 case kArm64Idiv32:
1311 __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1312 break;
1313 case kArm64Udiv:
1314 __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1315 break;
1316 case kArm64Udiv32:
1317 __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1318 break;
1319 case kArm64Imod: {
1320 UseScratchRegisterScope scope(tasm());
1321 Register temp = scope.AcquireX();
1322 __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1323 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1324 break;
1325 }
1326 case kArm64Imod32: {
1327 UseScratchRegisterScope scope(tasm());
1328 Register temp = scope.AcquireW();
1329 __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1330 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1331 i.InputRegister32(0));
1332 break;
1333 }
1334 case kArm64Umod: {
1335 UseScratchRegisterScope scope(tasm());
1336 Register temp = scope.AcquireX();
1337 __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1338 __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1339 break;
1340 }
1341 case kArm64Umod32: {
1342 UseScratchRegisterScope scope(tasm());
1343 Register temp = scope.AcquireW();
1344 __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1345 __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1346 i.InputRegister32(0));
1347 break;
1348 }
1349 case kArm64Not:
1350 __ Mvn(i.OutputRegister(), i.InputOperand(0));
1351 break;
1352 case kArm64Not32:
1353 __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1354 break;
1355 case kArm64Or:
1356 __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1357 i.InputOperand2_64(1));
1358 break;
1359 case kArm64Or32:
1360 __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1361 i.InputOperand2_32(1));
1362 break;
1363 case kArm64Orn:
1364 __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1365 i.InputOperand2_64(1));
1366 break;
1367 case kArm64Orn32:
1368 __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1369 i.InputOperand2_32(1));
1370 break;
1371 case kArm64Eor:
1372 __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1373 i.InputOperand2_64(1));
1374 break;
1375 case kArm64Eor32:
1376 __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1377 i.InputOperand2_32(1));
1378 break;
1379 case kArm64Eon:
1380 __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1381 i.InputOperand2_64(1));
1382 break;
1383 case kArm64Eon32:
1384 __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1385 i.InputOperand2_32(1));
1386 break;
1387 case kArm64Sub:
1388 if (FlagsModeField::decode(opcode) != kFlags_none) {
1389 __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1390 i.InputOperand2_64(1));
1391 } else {
1392 __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1393 i.InputOperand2_64(1));
1394 }
1395 break;
1396 case kArm64Sub32:
1397 if (FlagsModeField::decode(opcode) != kFlags_none) {
1398 __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1399 i.InputOperand2_32(1));
1400 } else {
1401 __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1402 i.InputOperand2_32(1));
1403 }
1404 break;
1405 case kArm64Lsl:
1406 ASSEMBLE_SHIFT(Lsl, 64);
1407 break;
1408 case kArm64Lsl32:
1409 ASSEMBLE_SHIFT(Lsl, 32);
1410 break;
1411 case kArm64Lsr:
1412 ASSEMBLE_SHIFT(Lsr, 64);
1413 break;
1414 case kArm64Lsr32:
1415 ASSEMBLE_SHIFT(Lsr, 32);
1416 break;
1417 case kArm64Asr:
1418 ASSEMBLE_SHIFT(Asr, 64);
1419 break;
1420 case kArm64Asr32:
1421 ASSEMBLE_SHIFT(Asr, 32);
1422 break;
1423 case kArm64Ror:
1424 ASSEMBLE_SHIFT(Ror, 64);
1425 break;
1426 case kArm64Ror32:
1427 ASSEMBLE_SHIFT(Ror, 32);
1428 break;
1429 case kArm64Mov32:
1430 __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1431 break;
1432 case kArm64Sxtb32:
1433 __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1434 break;
1435 case kArm64Sxth32:
1436 __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1437 break;
1438 case kArm64Sxtb:
1439 __ Sxtb(i.OutputRegister(), i.InputRegister32(0));
1440 break;
1441 case kArm64Sxth:
1442 __ Sxth(i.OutputRegister(), i.InputRegister32(0));
1443 break;
1444 case kArm64Sxtw:
1445 __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1446 break;
1447 case kArm64Sbfx:
1448 __ Sbfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1449 i.InputInt6(2));
1450 break;
1451 case kArm64Sbfx32:
1452 __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1453 i.InputInt5(2));
1454 break;
1455 case kArm64Ubfx:
1456 __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1457 i.InputInt32(2));
1458 break;
1459 case kArm64Ubfx32:
1460 __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1461 i.InputInt32(2));
1462 break;
1463 case kArm64Ubfiz32:
1464 __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1465 i.InputInt5(2));
1466 break;
1467 case kArm64Bfi:
1468 __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1469 i.InputInt6(3));
1470 break;
1471 case kArm64TestAndBranch32:
1472 case kArm64TestAndBranch:
1473 // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1474 break;
1475 case kArm64CompareAndBranch32:
1476 case kArm64CompareAndBranch:
1477 // Pseudo instruction handled in AssembleArchBranch.
1478 break;
1479 case kArm64Claim: {
1480 int count = i.InputInt32(0);
1481 DCHECK_EQ(count % 2, 0);
1482 __ AssertSpAligned();
1483 if (count > 0) {
1484 __ Claim(count);
1485 frame_access_state()->IncreaseSPDelta(count);
1486 }
1487 break;
1488 }
1489 case kArm64Poke: {
1490 Operand operand(i.InputInt32(1) * kSystemPointerSize);
1491 if (instr->InputAt(0)->IsSimd128Register()) {
1492 __ Poke(i.InputSimd128Register(0), operand);
1493 } else if (instr->InputAt(0)->IsFPRegister()) {
1494 __ Poke(i.InputFloat64Register(0), operand);
1495 } else {
1496 __ Poke(i.InputOrZeroRegister64(0), operand);
1497 }
1498 break;
1499 }
1500 case kArm64PokePair: {
1501 int slot = i.InputInt32(2) - 1;
1502 if (instr->InputAt(0)->IsFPRegister()) {
1503 __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1504 slot * kSystemPointerSize);
1505 } else {
1506 __ PokePair(i.InputRegister(1), i.InputRegister(0),
1507 slot * kSystemPointerSize);
1508 }
1509 break;
1510 }
1511 case kArm64Peek: {
1512 int reverse_slot = i.InputInt32(0);
1513 int offset =
1514 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1515 if (instr->OutputAt(0)->IsFPRegister()) {
1516 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1517 if (op->representation() == MachineRepresentation::kFloat64) {
1518 __ Ldr(i.OutputDoubleRegister(), MemOperand(fp, offset));
1519 } else if (op->representation() == MachineRepresentation::kFloat32) {
1520 __ Ldr(i.OutputFloatRegister(), MemOperand(fp, offset));
1521 } else {
1522 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1523 __ Ldr(i.OutputSimd128Register(), MemOperand(fp, offset));
1524 }
1525 } else {
1526 __ Ldr(i.OutputRegister(), MemOperand(fp, offset));
1527 }
1528 break;
1529 }
1530 case kArm64Clz:
1531 __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1532 break;
1533 case kArm64Clz32:
1534 __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1535 break;
1536 case kArm64Rbit:
1537 __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1538 break;
1539 case kArm64Rbit32:
1540 __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1541 break;
1542 case kArm64Rev:
1543 __ Rev(i.OutputRegister64(), i.InputRegister64(0));
1544 break;
1545 case kArm64Rev32:
1546 __ Rev(i.OutputRegister32(), i.InputRegister32(0));
1547 break;
1548 case kArm64Cmp:
1549 __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1550 break;
1551 case kArm64Cmp32:
1552 __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1553 break;
1554 case kArm64Cmn:
1555 __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1556 break;
1557 case kArm64Cmn32:
1558 __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1559 break;
1560 case kArm64Cnt32: {
1561 __ PopcntHelper(i.OutputRegister32(), i.InputRegister32(0));
1562 break;
1563 }
1564 case kArm64Cnt64: {
1565 __ PopcntHelper(i.OutputRegister64(), i.InputRegister64(0));
1566 break;
1567 }
1568 case kArm64Cnt: {
1569 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
1570 __ Cnt(i.OutputSimd128Register().Format(f),
1571 i.InputSimd128Register(0).Format(f));
1572 break;
1573 }
1574 case kArm64Tst:
1575 __ Tst(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1576 break;
1577 case kArm64Tst32:
1578 __ Tst(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1579 break;
1580 case kArm64Float32Cmp:
1581 if (instr->InputAt(1)->IsFPRegister()) {
1582 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1583 } else {
1584 DCHECK(instr->InputAt(1)->IsImmediate());
1585 // 0.0 is the only immediate supported by fcmp instructions.
1586 DCHECK_EQ(0.0f, i.InputFloat32(1));
1587 __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1588 }
1589 break;
1590 case kArm64Float32Add:
1591 __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1592 i.InputFloat32Register(1));
1593 break;
1594 case kArm64Float32Sub:
1595 __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1596 i.InputFloat32Register(1));
1597 break;
1598 case kArm64Float32Mul:
1599 __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1600 i.InputFloat32Register(1));
1601 break;
1602 case kArm64Float32Div:
1603 __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1604 i.InputFloat32Register(1));
1605 break;
1606 case kArm64Float32Abs:
1607 __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1608 break;
1609 case kArm64Float32Abd:
1610 __ Fabd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1611 i.InputFloat32Register(1));
1612 break;
1613 case kArm64Float32Neg:
1614 __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1615 break;
1616 case kArm64Float32Sqrt:
1617 __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1618 break;
1619 case kArm64Float32Fnmul: {
1620 __ Fnmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1621 i.InputFloat32Register(1));
1622 break;
1623 }
1624 case kArm64Float64Cmp:
1625 if (instr->InputAt(1)->IsFPRegister()) {
1626 __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1627 } else {
1628 DCHECK(instr->InputAt(1)->IsImmediate());
1629 // 0.0 is the only immediate supported by fcmp instructions.
1630 DCHECK_EQ(0.0, i.InputDouble(1));
1631 __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1632 }
1633 break;
1634 case kArm64Float64Add:
1635 __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1636 i.InputDoubleRegister(1));
1637 break;
1638 case kArm64Float64Sub:
1639 __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1640 i.InputDoubleRegister(1));
1641 break;
1642 case kArm64Float64Mul:
1643 __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1644 i.InputDoubleRegister(1));
1645 break;
1646 case kArm64Float64Div:
1647 __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1648 i.InputDoubleRegister(1));
1649 break;
1650 case kArm64Float64Mod: {
1651 // TODO(turbofan): implement directly.
1652 FrameScope scope(tasm(), StackFrame::MANUAL);
1653 DCHECK_EQ(d0, i.InputDoubleRegister(0));
1654 DCHECK_EQ(d1, i.InputDoubleRegister(1));
1655 DCHECK_EQ(d0, i.OutputDoubleRegister());
1656 // TODO(turbofan): make sure this saves all relevant registers.
1657 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1658 break;
1659 }
1660 case kArm64Float32Max: {
1661 __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1662 i.InputFloat32Register(1));
1663 break;
1664 }
1665 case kArm64Float64Max: {
1666 __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1667 i.InputDoubleRegister(1));
1668 break;
1669 }
1670 case kArm64Float32Min: {
1671 __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1672 i.InputFloat32Register(1));
1673 break;
1674 }
1675 case kArm64Float64Min: {
1676 __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1677 i.InputDoubleRegister(1));
1678 break;
1679 }
1680 case kArm64Float64Abs:
1681 __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1682 break;
1683 case kArm64Float64Abd:
1684 __ Fabd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1685 i.InputDoubleRegister(1));
1686 break;
1687 case kArm64Float64Neg:
1688 __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1689 break;
1690 case kArm64Float64Sqrt:
1691 __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1692 break;
1693 case kArm64Float64Fnmul:
1694 __ Fnmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1695 i.InputDoubleRegister(1));
1696 break;
1697 case kArm64Float32ToFloat64:
1698 __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1699 break;
1700 case kArm64Float64ToFloat32:
1701 __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1702 break;
1703 case kArm64Float32ToInt32: {
1704 __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1705 bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
1706 if (set_overflow_to_min_i32) {
1707 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1708 // because INT32_MIN allows easier out-of-bounds detection.
1709 __ Cmn(i.OutputRegister32(), 1);
1710 __ Csinc(i.OutputRegister32(), i.OutputRegister32(),
1711 i.OutputRegister32(), vc);
1712 }
1713 break;
1714 }
1715 case kArm64Float64ToInt32:
1716 __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1717 break;
1718 case kArm64Float32ToUint32: {
1719 __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1720 bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
1721 if (set_overflow_to_min_u32) {
1722 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1723 // because 0 allows easier out-of-bounds detection.
1724 __ Cmn(i.OutputRegister32(), 1);
1725 __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1726 }
1727 break;
1728 }
1729 case kArm64Float64ToUint32:
1730 __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1731 break;
1732 case kArm64Float32ToInt64:
1733 __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1734 if (i.OutputCount() > 1) {
1735 // Check for inputs below INT64_MIN and NaN.
1736 __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1737 // Check overflow.
1738 // -1 value is used to indicate a possible overflow which will occur
1739 // when subtracting (-1) from the provided INT64_MAX operand.
1740 // OutputRegister(1) is set to 0 if the input was out of range or NaN.
1741 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1742 __ Cset(i.OutputRegister(1), vc);
1743 }
1744 break;
1745 case kArm64Float64ToInt64: {
1746 __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1747 bool set_overflow_to_min_i64 = MiscField::decode(instr->opcode());
1748 DCHECK_IMPLIES(set_overflow_to_min_i64, i.OutputCount() == 1);
1749 if (set_overflow_to_min_i64) {
1750 // Avoid INT64_MAX as an overflow indicator and use INT64_MIN instead,
1751 // because INT64_MIN allows easier out-of-bounds detection.
1752 __ Cmn(i.OutputRegister64(), 1);
1753 __ Csinc(i.OutputRegister64(), i.OutputRegister64(),
1754 i.OutputRegister64(), vc);
1755 } else if (i.OutputCount() > 1) {
1756 // See kArm64Float32ToInt64 for a detailed description.
1757 __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1758 __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1759 __ Cset(i.OutputRegister(1), vc);
1760 }
1761 break;
1762 }
1763 case kArm64Float32ToUint64:
1764 __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1765 if (i.OutputCount() > 1) {
1766 // See kArm64Float32ToInt64 for a detailed description.
1767 __ Fcmp(i.InputFloat32Register(0), -1.0);
1768 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1769 __ Cset(i.OutputRegister(1), ne);
1770 }
1771 break;
1772 case kArm64Float64ToUint64:
1773 __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1774 if (i.OutputCount() > 1) {
1775 // See kArm64Float32ToInt64 for a detailed description.
1776 __ Fcmp(i.InputDoubleRegister(0), -1.0);
1777 __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1778 __ Cset(i.OutputRegister(1), ne);
1779 }
1780 break;
1781 case kArm64Int32ToFloat32:
1782 __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1783 break;
1784 case kArm64Int32ToFloat64:
1785 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1786 break;
1787 case kArm64Int64ToFloat32:
1788 __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1789 break;
1790 case kArm64Int64ToFloat64:
1791 __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1792 break;
1793 case kArm64Uint32ToFloat32:
1794 __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1795 break;
1796 case kArm64Uint32ToFloat64:
1797 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1798 break;
1799 case kArm64Uint64ToFloat32:
1800 __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1801 break;
1802 case kArm64Uint64ToFloat64:
1803 __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1804 break;
1805 case kArm64Float64ExtractLowWord32:
1806 __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1807 break;
1808 case kArm64Float64ExtractHighWord32:
1809 __ Umov(i.OutputRegister32(), i.InputFloat64Register(0).V2S(), 1);
1810 break;
1811 case kArm64Float64InsertLowWord32:
1812 DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
1813 __ Ins(i.OutputFloat64Register().V2S(), 0, i.InputRegister32(1));
1814 break;
1815 case kArm64Float64InsertHighWord32:
1816 DCHECK_EQ(i.OutputFloat64Register(), i.InputFloat64Register(0));
1817 __ Ins(i.OutputFloat64Register().V2S(), 1, i.InputRegister32(1));
1818 break;
1819 case kArm64Float64MoveU64:
1820 __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1821 break;
1822 case kArm64Float64SilenceNaN:
1823 __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1824 break;
1825 case kArm64U64MoveFloat64:
1826 __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1827 break;
1828 case kArm64Ldrb:
1829 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1830 __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1831 break;
1832 case kArm64Ldrsb:
1833 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1834 __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1835 break;
1836 case kArm64LdrsbW:
1837 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1838 __ Ldrsb(i.OutputRegister32(), i.MemoryOperand());
1839 break;
1840 case kArm64Strb:
1841 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1842 __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1843 break;
1844 case kArm64Ldrh:
1845 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1846 __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1847 break;
1848 case kArm64Ldrsh:
1849 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1850 __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1851 break;
1852 case kArm64LdrshW:
1853 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1854 __ Ldrsh(i.OutputRegister32(), i.MemoryOperand());
1855 break;
1856 case kArm64Strh:
1857 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1858 __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1859 break;
1860 case kArm64Ldrsw:
1861 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1862 __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1863 break;
1864 case kArm64LdrW:
1865 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1866 __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1867 break;
1868 case kArm64StrW:
1869 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1870 __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1871 break;
1872 case kArm64Ldr:
1873 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1874 __ Ldr(i.OutputRegister(), i.MemoryOperand());
1875 break;
1876 case kArm64LdrDecompressTaggedSigned:
1877 __ DecompressTaggedSigned(i.OutputRegister(), i.MemoryOperand());
1878 break;
1879 case kArm64LdrDecompressTaggedPointer:
1880 __ DecompressTaggedPointer(i.OutputRegister(), i.MemoryOperand());
1881 break;
1882 case kArm64LdrDecompressAnyTagged:
1883 __ DecompressAnyTagged(i.OutputRegister(), i.MemoryOperand());
1884 break;
1885 case kArm64LdarDecompressTaggedSigned:
1886 __ AtomicDecompressTaggedSigned(i.OutputRegister(), i.InputRegister(0),
1887 i.InputRegister(1), i.TempRegister(0));
1888 break;
1889 case kArm64LdarDecompressTaggedPointer:
1890 __ AtomicDecompressTaggedPointer(i.OutputRegister(), i.InputRegister(0),
1891 i.InputRegister(1), i.TempRegister(0));
1892 break;
1893 case kArm64LdarDecompressAnyTagged:
1894 __ AtomicDecompressAnyTagged(i.OutputRegister(), i.InputRegister(0),
1895 i.InputRegister(1), i.TempRegister(0));
1896 break;
1897 case kArm64Str:
1898 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1899 __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1900 break;
1901 case kArm64StrCompressTagged:
1902 __ StoreTaggedField(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1903 break;
1904 case kArm64StlrCompressTagged:
1905 // To be consistent with other STLR instructions, the value is stored at
1906 // the 3rd input register instead of the 1st.
1907 __ AtomicStoreTaggedField(i.InputRegister(2), i.InputRegister(0),
1908 i.InputRegister(1), i.TempRegister(0));
1909 break;
1910 case kArm64LdrS:
1911 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1912 __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1913 break;
1914 case kArm64StrS:
1915 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1916 __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1917 break;
1918 case kArm64LdrD:
1919 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1920 __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1921 break;
1922 case kArm64StrD:
1923 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1924 __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1925 break;
1926 case kArm64LdrQ:
1927 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1928 __ Ldr(i.OutputSimd128Register(), i.MemoryOperand());
1929 break;
1930 case kArm64StrQ:
1931 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
1932 __ Str(i.InputSimd128Register(0), i.MemoryOperand(1));
1933 break;
1934 case kArm64DmbIsh:
1935 __ Dmb(InnerShareable, BarrierAll);
1936 break;
1937 case kArm64DsbIsb:
1938 __ Dsb(FullSystem, BarrierAll);
1939 __ Isb();
1940 break;
1941 case kAtomicLoadInt8:
1942 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1943 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1944 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1945 break;
1946 case kAtomicLoadUint8:
1947 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1948 break;
1949 case kAtomicLoadInt16:
1950 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1951 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1952 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1953 break;
1954 case kAtomicLoadUint16:
1955 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1956 break;
1957 case kAtomicLoadWord32:
1958 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register32);
1959 break;
1960 case kArm64Word64AtomicLoadUint64:
1961 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register);
1962 break;
1963 case kAtomicStoreWord8:
1964 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb, Register32);
1965 break;
1966 case kAtomicStoreWord16:
1967 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh, Register32);
1968 break;
1969 case kAtomicStoreWord32:
1970 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register32);
1971 break;
1972 case kArm64Word64AtomicStoreWord64:
1973 ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register);
1974 break;
1975 case kAtomicExchangeInt8:
1976 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1977 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1978 break;
1979 case kAtomicExchangeUint8:
1980 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1981 break;
1982 case kAtomicExchangeInt16:
1983 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1984 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1985 break;
1986 case kAtomicExchangeUint16:
1987 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1988 break;
1989 case kAtomicExchangeWord32:
1990 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register32);
1991 break;
1992 case kArm64Word64AtomicExchangeUint64:
1993 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register);
1994 break;
1995 case kAtomicCompareExchangeInt8:
1996 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
1997 Register32);
1998 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1999 break;
2000 case kAtomicCompareExchangeUint8:
2001 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
2002 Register32);
2003 break;
2004 case kAtomicCompareExchangeInt16:
2005 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
2006 Register32);
2007 __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
2008 break;
2009 case kAtomicCompareExchangeUint16:
2010 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
2011 Register32);
2012 break;
2013 case kAtomicCompareExchangeWord32:
2014 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTW, Register32);
2015 break;
2016 case kArm64Word64AtomicCompareExchangeUint64:
2017 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTX, Register);
2018 break;
2019 #define ATOMIC_BINOP_CASE(op, inst) \
2020 case kAtomic##op##Int8: \
2021 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
2022 __ Sxtb(i.OutputRegister(0), i.OutputRegister(0)); \
2023 break; \
2024 case kAtomic##op##Uint8: \
2025 ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
2026 break; \
2027 case kAtomic##op##Int16: \
2028 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
2029 __ Sxth(i.OutputRegister(0), i.OutputRegister(0)); \
2030 break; \
2031 case kAtomic##op##Uint16: \
2032 ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
2033 break; \
2034 case kAtomic##op##Word32: \
2035 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register32); \
2036 break; \
2037 case kArm64Word64Atomic##op##Uint64: \
2038 ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register); \
2039 break;
2040 ATOMIC_BINOP_CASE(Add, Add)
2041 ATOMIC_BINOP_CASE(Sub, Sub)
2042 ATOMIC_BINOP_CASE(And, And)
2043 ATOMIC_BINOP_CASE(Or, Orr)
2044 ATOMIC_BINOP_CASE(Xor, Eor)
2045 #undef ATOMIC_BINOP_CASE
2046 #undef ASSEMBLE_SHIFT
2047 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
2048 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
2049 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
2050 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
2051 #undef ASSEMBLE_ATOMIC_BINOP
2052 #undef ASSEMBLE_IEEE754_BINOP
2053 #undef ASSEMBLE_IEEE754_UNOP
2054
2055 #define SIMD_UNOP_CASE(Op, Instr, FORMAT) \
2056 case Op: \
2057 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
2058 i.InputSimd128Register(0).V##FORMAT()); \
2059 break;
2060 #define SIMD_UNOP_LANE_SIZE_CASE(Op, Instr) \
2061 case Op: { \
2062 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2063 __ Instr(i.OutputSimd128Register().Format(f), \
2064 i.InputSimd128Register(0).Format(f)); \
2065 break; \
2066 }
2067 #define SIMD_BINOP_CASE(Op, Instr, FORMAT) \
2068 case Op: \
2069 __ Instr(i.OutputSimd128Register().V##FORMAT(), \
2070 i.InputSimd128Register(0).V##FORMAT(), \
2071 i.InputSimd128Register(1).V##FORMAT()); \
2072 break;
2073 #define SIMD_BINOP_LANE_SIZE_CASE(Op, Instr) \
2074 case Op: { \
2075 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2076 __ Instr(i.OutputSimd128Register().Format(f), \
2077 i.InputSimd128Register(0).Format(f), \
2078 i.InputSimd128Register(1).Format(f)); \
2079 break; \
2080 }
2081 #define SIMD_FCM_L_CASE(Op, ImmOp, RegOp) \
2082 case Op: { \
2083 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2084 if (instr->InputCount() == 1) { \
2085 __ Fcm##ImmOp(i.OutputSimd128Register().Format(f), \
2086 i.InputSimd128Register(0).Format(f), +0.0); \
2087 } else { \
2088 __ Fcm##RegOp(i.OutputSimd128Register().Format(f), \
2089 i.InputSimd128Register(1).Format(f), \
2090 i.InputSimd128Register(0).Format(f)); \
2091 } \
2092 break; \
2093 }
2094 #define SIMD_FCM_G_CASE(Op, ImmOp) \
2095 case Op: { \
2096 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2097 /* Currently Gt/Ge instructions are only used with zero */ \
2098 DCHECK_EQ(instr->InputCount(), 1); \
2099 __ Fcm##ImmOp(i.OutputSimd128Register().Format(f), \
2100 i.InputSimd128Register(0).Format(f), +0.0); \
2101 break; \
2102 }
2103 #define SIMD_DESTRUCTIVE_BINOP_CASE(Op, Instr, FORMAT) \
2104 case Op: { \
2105 VRegister dst = i.OutputSimd128Register().V##FORMAT(); \
2106 DCHECK_EQ(dst, i.InputSimd128Register(0).V##FORMAT()); \
2107 __ Instr(dst, i.InputSimd128Register(1).V##FORMAT(), \
2108 i.InputSimd128Register(2).V##FORMAT()); \
2109 break; \
2110 }
2111 #define SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(Op, Instr) \
2112 case Op: { \
2113 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode)); \
2114 VRegister dst = i.OutputSimd128Register().Format(f); \
2115 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f)); \
2116 __ Instr(dst, i.InputSimd128Register(1).Format(f), \
2117 i.InputSimd128Register(2).Format(f)); \
2118 break; \
2119 }
2120 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMin, Fmin);
2121 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMax, Fmax);
2122 SIMD_UNOP_LANE_SIZE_CASE(kArm64FAbs, Fabs);
2123 SIMD_UNOP_LANE_SIZE_CASE(kArm64FSqrt, Fsqrt);
2124 SIMD_BINOP_LANE_SIZE_CASE(kArm64FAdd, Fadd);
2125 SIMD_BINOP_LANE_SIZE_CASE(kArm64FSub, Fsub);
2126 SIMD_BINOP_LANE_SIZE_CASE(kArm64FMul, Fmul);
2127 SIMD_BINOP_LANE_SIZE_CASE(kArm64FDiv, Fdiv);
2128 SIMD_UNOP_LANE_SIZE_CASE(kArm64FNeg, Fneg);
2129 SIMD_UNOP_LANE_SIZE_CASE(kArm64IAbs, Abs);
2130 SIMD_UNOP_LANE_SIZE_CASE(kArm64INeg, Neg);
2131 SIMD_BINOP_LANE_SIZE_CASE(kArm64RoundingAverageU, Urhadd);
2132 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMinS, Smin);
2133 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMaxS, Smax);
2134 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMinU, Umin);
2135 SIMD_BINOP_LANE_SIZE_CASE(kArm64IMaxU, Umax);
2136 SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(kArm64Mla, Mla);
2137 SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE(kArm64Mls, Mls);
2138 case kArm64Sxtl: {
2139 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2140 VectorFormat narrow = VectorFormatHalfWidth(wide);
2141 __ Sxtl(i.OutputSimd128Register().Format(wide),
2142 i.InputSimd128Register(0).Format(narrow));
2143 break;
2144 }
2145 case kArm64Sxtl2: {
2146 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2147 VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
2148 __ Sxtl2(i.OutputSimd128Register().Format(wide),
2149 i.InputSimd128Register(0).Format(narrow));
2150 break;
2151 }
2152 case kArm64Uxtl: {
2153 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2154 VectorFormat narrow = VectorFormatHalfWidth(wide);
2155 __ Uxtl(i.OutputSimd128Register().Format(wide),
2156 i.InputSimd128Register(0).Format(narrow));
2157 break;
2158 }
2159 case kArm64Uxtl2: {
2160 VectorFormat wide = VectorFormatFillQ(LaneSizeField::decode(opcode));
2161 VectorFormat narrow = VectorFormatHalfWidthDoubleLanes(wide);
2162 __ Uxtl2(i.OutputSimd128Register().Format(wide),
2163 i.InputSimd128Register(0).Format(narrow));
2164 break;
2165 }
2166 case kArm64F64x2ConvertLowI32x4S: {
2167 VRegister dst = i.OutputSimd128Register().V2D();
2168 __ Sxtl(dst, i.InputSimd128Register(0).V2S());
2169 __ Scvtf(dst, dst);
2170 break;
2171 }
2172 case kArm64F64x2ConvertLowI32x4U: {
2173 VRegister dst = i.OutputSimd128Register().V2D();
2174 __ Uxtl(dst, i.InputSimd128Register(0).V2S());
2175 __ Ucvtf(dst, dst);
2176 break;
2177 }
2178 case kArm64I32x4TruncSatF64x2SZero: {
2179 VRegister dst = i.OutputSimd128Register();
2180 __ Fcvtzs(dst.V2D(), i.InputSimd128Register(0).V2D());
2181 __ Sqxtn(dst.V2S(), dst.V2D());
2182 break;
2183 }
2184 case kArm64I32x4TruncSatF64x2UZero: {
2185 VRegister dst = i.OutputSimd128Register();
2186 __ Fcvtzu(dst.V2D(), i.InputSimd128Register(0).V2D());
2187 __ Uqxtn(dst.V2S(), dst.V2D());
2188 break;
2189 }
2190 case kArm64F32x4DemoteF64x2Zero: {
2191 __ Fcvtn(i.OutputSimd128Register().V2S(),
2192 i.InputSimd128Register(0).V2D());
2193 break;
2194 }
2195 case kArm64F64x2PromoteLowF32x4: {
2196 __ Fcvtl(i.OutputSimd128Register().V2D(),
2197 i.InputSimd128Register(0).V2S());
2198 break;
2199 }
2200 case kArm64FExtractLane: {
2201 VectorFormat dst_f =
2202 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
2203 VectorFormat src_f = VectorFormatFillQ(dst_f);
2204 __ Mov(i.OutputSimd128Register().Format(dst_f),
2205 i.InputSimd128Register(0).Format(src_f), i.InputInt8(1));
2206 break;
2207 }
2208 case kArm64FReplaceLane: {
2209 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2210 VRegister dst = i.OutputSimd128Register().Format(f),
2211 src1 = i.InputSimd128Register(0).Format(f);
2212 if (dst != src1) {
2213 __ Mov(dst, src1);
2214 }
2215 __ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).Format(f), 0);
2216 break;
2217 }
2218 SIMD_FCM_L_CASE(kArm64FEq, eq, eq);
2219 case kArm64FNe: {
2220 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2221 VRegister dst = i.OutputSimd128Register().Format(f);
2222 if (instr->InputCount() == 1) {
2223 __ Fcmeq(dst, i.InputSimd128Register(0).Format(f), +0.0);
2224 } else {
2225 __ Fcmeq(dst, i.InputSimd128Register(0).Format(f),
2226 i.InputSimd128Register(1).Format(f));
2227 }
2228 __ Mvn(dst, dst);
2229 break;
2230 }
2231 SIMD_FCM_L_CASE(kArm64FLt, lt, gt);
2232 SIMD_FCM_L_CASE(kArm64FLe, le, ge);
2233 SIMD_FCM_G_CASE(kArm64FGt, gt);
2234 SIMD_FCM_G_CASE(kArm64FGe, ge);
2235 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfma, Fmla, 2D);
2236 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F64x2Qfms, Fmls, 2D);
2237 case kArm64F64x2Pmin: {
2238 VRegister dst = i.OutputSimd128Register().V2D();
2239 VRegister lhs = i.InputSimd128Register(0).V2D();
2240 VRegister rhs = i.InputSimd128Register(1).V2D();
2241 // f64x2.pmin(lhs, rhs)
2242 // = v128.bitselect(rhs, lhs, f64x2.lt(rhs,lhs))
2243 // = v128.bitselect(rhs, lhs, f64x2.gt(lhs,rhs))
2244 __ Fcmgt(dst, lhs, rhs);
2245 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2246 break;
2247 }
2248 case kArm64F64x2Pmax: {
2249 VRegister dst = i.OutputSimd128Register().V2D();
2250 VRegister lhs = i.InputSimd128Register(0).V2D();
2251 VRegister rhs = i.InputSimd128Register(1).V2D();
2252 // f64x2.pmax(lhs, rhs)
2253 // = v128.bitselect(rhs, lhs, f64x2.gt(rhs, lhs))
2254 __ Fcmgt(dst, rhs, lhs);
2255 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2256 break;
2257 }
2258 SIMD_UNOP_CASE(kArm64F32x4SConvertI32x4, Scvtf, 4S);
2259 SIMD_UNOP_CASE(kArm64F32x4UConvertI32x4, Ucvtf, 4S);
2260 SIMD_UNOP_CASE(kArm64F32x4RecipApprox, Frecpe, 4S);
2261 SIMD_UNOP_CASE(kArm64F32x4RecipSqrtApprox, Frsqrte, 4S);
2262 case kArm64FMulElement: {
2263 VectorFormat s_f =
2264 ScalarFormatFromLaneSize(LaneSizeField::decode(opcode));
2265 VectorFormat v_f = VectorFormatFillQ(s_f);
2266 __ Fmul(i.OutputSimd128Register().Format(v_f),
2267 i.InputSimd128Register(0).Format(v_f),
2268 i.InputSimd128Register(1).Format(s_f), i.InputInt8(2));
2269 break;
2270 }
2271 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfma, Fmla, 4S);
2272 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64F32x4Qfms, Fmls, 4S);
2273 case kArm64F32x4Pmin: {
2274 VRegister dst = i.OutputSimd128Register().V4S();
2275 VRegister lhs = i.InputSimd128Register(0).V4S();
2276 VRegister rhs = i.InputSimd128Register(1).V4S();
2277 // f32x4.pmin(lhs, rhs)
2278 // = v128.bitselect(rhs, lhs, f32x4.lt(rhs, lhs))
2279 // = v128.bitselect(rhs, lhs, f32x4.gt(lhs, rhs))
2280 __ Fcmgt(dst, lhs, rhs);
2281 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2282 break;
2283 }
2284 case kArm64F32x4Pmax: {
2285 VRegister dst = i.OutputSimd128Register().V4S();
2286 VRegister lhs = i.InputSimd128Register(0).V4S();
2287 VRegister rhs = i.InputSimd128Register(1).V4S();
2288 // f32x4.pmax(lhs, rhs)
2289 // = v128.bitselect(rhs, lhs, f32x4.gt(rhs, lhs))
2290 __ Fcmgt(dst, rhs, lhs);
2291 __ Bsl(dst.V16B(), rhs.V16B(), lhs.V16B());
2292 break;
2293 }
2294 case kArm64IExtractLane: {
2295 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2296 Register dst =
2297 f == kFormat2D ? i.OutputRegister64() : i.OutputRegister32();
2298 __ Mov(dst, i.InputSimd128Register(0).Format(f), i.InputInt8(1));
2299 break;
2300 }
2301 case kArm64IReplaceLane: {
2302 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2303 VRegister dst = i.OutputSimd128Register().Format(f),
2304 src1 = i.InputSimd128Register(0).Format(f);
2305 Register src2 =
2306 f == kFormat2D ? i.InputRegister64(2) : i.InputRegister32(2);
2307 if (dst != src1) {
2308 __ Mov(dst, src1);
2309 }
2310 __ Mov(dst, i.InputInt8(1), src2);
2311 break;
2312 }
2313 case kArm64I64x2Shl: {
2314 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 6, V2D, Sshl, X);
2315 break;
2316 }
2317 case kArm64I64x2ShrS: {
2318 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 6, V2D, Sshl, X);
2319 break;
2320 }
2321 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAdd, Add);
2322 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISub, Sub);
2323 case kArm64I64x2Mul: {
2324 UseScratchRegisterScope scope(tasm());
2325 VRegister dst = i.OutputSimd128Register();
2326 VRegister src1 = i.InputSimd128Register(0);
2327 VRegister src2 = i.InputSimd128Register(1);
2328 VRegister tmp1 = scope.AcquireSameSizeAs(dst);
2329 VRegister tmp2 = scope.AcquireSameSizeAs(dst);
2330 VRegister tmp3 = i.ToSimd128Register(instr->TempAt(0));
2331
2332 // This 2x64-bit multiplication is performed with several 32-bit
2333 // multiplications.
2334
2335 // 64-bit numbers x and y, can be represented as:
2336 // x = a + 2^32(b)
2337 // y = c + 2^32(d)
2338
2339 // A 64-bit multiplication is:
2340 // x * y = ac + 2^32(ad + bc) + 2^64(bd)
2341 // note: `2^64(bd)` can be ignored, the value is too large to fit in
2342 // 64-bits.
2343
2344 // This sequence implements a 2x64bit multiply, where the registers
2345 // `src1` and `src2` are split up into 32-bit components:
2346 // src1 = |d|c|b|a|
2347 // src2 = |h|g|f|e|
2348 //
2349 // src1 * src2 = |cg + 2^32(ch + dg)|ae + 2^32(af + be)|
2350
2351 // Reverse the 32-bit elements in the 64-bit words.
2352 // tmp2 = |g|h|e|f|
2353 __ Rev64(tmp2.V4S(), src2.V4S());
2354
2355 // Calculate the high half components.
2356 // tmp2 = |dg|ch|be|af|
2357 __ Mul(tmp2.V4S(), tmp2.V4S(), src1.V4S());
2358
2359 // Extract the low half components of src1.
2360 // tmp1 = |c|a|
2361 __ Xtn(tmp1.V2S(), src1.V2D());
2362
2363 // Sum the respective high half components.
2364 // tmp2 = |dg+ch|be+af||dg+ch|be+af|
2365 __ Addp(tmp2.V4S(), tmp2.V4S(), tmp2.V4S());
2366
2367 // Extract the low half components of src2.
2368 // tmp3 = |g|e|
2369 __ Xtn(tmp3.V2S(), src2.V2D());
2370
2371 // Shift the high half components, into the high half.
2372 // dst = |dg+ch << 32|be+af << 32|
2373 __ Shll(dst.V2D(), tmp2.V2S(), 32);
2374
2375 // Multiply the low components together, and accumulate with the high
2376 // half.
2377 // dst = |dst[1] + cg|dst[0] + ae|
2378 __ Umlal(dst.V2D(), tmp3.V2S(), tmp1.V2S());
2379
2380 break;
2381 }
2382 SIMD_BINOP_LANE_SIZE_CASE(kArm64IEq, Cmeq);
2383 case kArm64INe: {
2384 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2385 VRegister dst = i.OutputSimd128Register().Format(f);
2386 __ Cmeq(dst, i.InputSimd128Register(0).Format(f),
2387 i.InputSimd128Register(1).Format(f));
2388 __ Mvn(dst, dst);
2389 break;
2390 }
2391 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGtS, Cmgt);
2392 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGeS, Cmge);
2393 case kArm64I64x2ShrU: {
2394 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 6, V2D, Ushl, X);
2395 break;
2396 }
2397 case kArm64I64x2BitMask: {
2398 __ I64x2BitMask(i.OutputRegister32(), i.InputSimd128Register(0));
2399 break;
2400 }
2401 SIMD_UNOP_CASE(kArm64I32x4SConvertF32x4, Fcvtzs, 4S);
2402 case kArm64I32x4Shl: {
2403 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 5, V4S, Sshl, W);
2404 break;
2405 }
2406 case kArm64I32x4ShrS: {
2407 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 5, V4S, Sshl, W);
2408 break;
2409 }
2410 SIMD_BINOP_CASE(kArm64I32x4Mul, Mul, 4S);
2411 SIMD_UNOP_CASE(kArm64I32x4UConvertF32x4, Fcvtzu, 4S);
2412 case kArm64I32x4ShrU: {
2413 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 5, V4S, Ushl, W);
2414 break;
2415 }
2416 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGtU, Cmhi);
2417 SIMD_BINOP_LANE_SIZE_CASE(kArm64IGeU, Cmhs);
2418 case kArm64I32x4BitMask: {
2419 UseScratchRegisterScope scope(tasm());
2420 Register dst = i.OutputRegister32();
2421 VRegister src = i.InputSimd128Register(0);
2422 VRegister tmp = scope.AcquireQ();
2423 VRegister mask = scope.AcquireQ();
2424
2425 __ Sshr(tmp.V4S(), src.V4S(), 31);
2426 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2427 // are signed will have i-th bit set, unsigned will be 0.
2428 __ Movi(mask.V2D(), 0x0000'0008'0000'0004, 0x0000'0002'0000'0001);
2429 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2430 __ Addv(tmp.S(), tmp.V4S());
2431 __ Mov(dst.W(), tmp.V4S(), 0);
2432 break;
2433 }
2434 case kArm64I32x4DotI16x8S: {
2435 UseScratchRegisterScope scope(tasm());
2436 VRegister lhs = i.InputSimd128Register(0);
2437 VRegister rhs = i.InputSimd128Register(1);
2438 VRegister tmp1 = scope.AcquireV(kFormat4S);
2439 VRegister tmp2 = scope.AcquireV(kFormat4S);
2440 __ Smull(tmp1, lhs.V4H(), rhs.V4H());
2441 __ Smull2(tmp2, lhs.V8H(), rhs.V8H());
2442 __ Addp(i.OutputSimd128Register().V4S(), tmp1, tmp2);
2443 break;
2444 }
2445 case kArm64IExtractLaneU: {
2446 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2447 __ Umov(i.OutputRegister32(), i.InputSimd128Register(0).Format(f),
2448 i.InputInt8(1));
2449 break;
2450 }
2451 case kArm64IExtractLaneS: {
2452 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2453 __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).Format(f),
2454 i.InputInt8(1));
2455 break;
2456 }
2457 case kArm64I16x8Shl: {
2458 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 4, V8H, Sshl, W);
2459 break;
2460 }
2461 case kArm64I16x8ShrS: {
2462 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 4, V8H, Sshl, W);
2463 break;
2464 }
2465 case kArm64I16x8SConvertI32x4: {
2466 VRegister dst = i.OutputSimd128Register(),
2467 src0 = i.InputSimd128Register(0),
2468 src1 = i.InputSimd128Register(1);
2469 UseScratchRegisterScope scope(tasm());
2470 VRegister temp = scope.AcquireV(kFormat4S);
2471 if (dst == src1) {
2472 __ Mov(temp, src1.V4S());
2473 src1 = temp;
2474 }
2475 __ Sqxtn(dst.V4H(), src0.V4S());
2476 __ Sqxtn2(dst.V8H(), src1.V4S());
2477 break;
2478 }
2479 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAddSatS, Sqadd);
2480 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISubSatS, Sqsub);
2481 SIMD_BINOP_CASE(kArm64I16x8Mul, Mul, 8H);
2482 case kArm64I16x8ShrU: {
2483 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 4, V8H, Ushl, W);
2484 break;
2485 }
2486 case kArm64I16x8UConvertI32x4: {
2487 VRegister dst = i.OutputSimd128Register(),
2488 src0 = i.InputSimd128Register(0),
2489 src1 = i.InputSimd128Register(1);
2490 UseScratchRegisterScope scope(tasm());
2491 VRegister temp = scope.AcquireV(kFormat4S);
2492 if (dst == src1) {
2493 __ Mov(temp, src1.V4S());
2494 src1 = temp;
2495 }
2496 __ Sqxtun(dst.V4H(), src0.V4S());
2497 __ Sqxtun2(dst.V8H(), src1.V4S());
2498 break;
2499 }
2500 SIMD_BINOP_LANE_SIZE_CASE(kArm64IAddSatU, Uqadd);
2501 SIMD_BINOP_LANE_SIZE_CASE(kArm64ISubSatU, Uqsub);
2502 SIMD_BINOP_CASE(kArm64I16x8Q15MulRSatS, Sqrdmulh, 8H);
2503 case kArm64I16x8BitMask: {
2504 UseScratchRegisterScope scope(tasm());
2505 Register dst = i.OutputRegister32();
2506 VRegister src = i.InputSimd128Register(0);
2507 VRegister tmp = scope.AcquireQ();
2508 VRegister mask = scope.AcquireQ();
2509
2510 __ Sshr(tmp.V8H(), src.V8H(), 15);
2511 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2512 // are signed will have i-th bit set, unsigned will be 0.
2513 __ Movi(mask.V2D(), 0x0080'0040'0020'0010, 0x0008'0004'0002'0001);
2514 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2515 __ Addv(tmp.H(), tmp.V8H());
2516 __ Mov(dst.W(), tmp.V8H(), 0);
2517 break;
2518 }
2519 case kArm64I8x16Shl: {
2520 ASSEMBLE_SIMD_SHIFT_LEFT(Shl, 3, V16B, Sshl, W);
2521 break;
2522 }
2523 case kArm64I8x16ShrS: {
2524 ASSEMBLE_SIMD_SHIFT_RIGHT(Sshr, 3, V16B, Sshl, W);
2525 break;
2526 }
2527 case kArm64I8x16SConvertI16x8: {
2528 VRegister dst = i.OutputSimd128Register(),
2529 src0 = i.InputSimd128Register(0),
2530 src1 = i.InputSimd128Register(1);
2531 UseScratchRegisterScope scope(tasm());
2532 VRegister temp = scope.AcquireV(kFormat8H);
2533 if (dst == src1) {
2534 __ Mov(temp, src1.V8H());
2535 src1 = temp;
2536 }
2537 __ Sqxtn(dst.V8B(), src0.V8H());
2538 __ Sqxtn2(dst.V16B(), src1.V8H());
2539 break;
2540 }
2541 case kArm64I8x16ShrU: {
2542 ASSEMBLE_SIMD_SHIFT_RIGHT(Ushr, 3, V16B, Ushl, W);
2543 break;
2544 }
2545 case kArm64I8x16UConvertI16x8: {
2546 VRegister dst = i.OutputSimd128Register(),
2547 src0 = i.InputSimd128Register(0),
2548 src1 = i.InputSimd128Register(1);
2549 UseScratchRegisterScope scope(tasm());
2550 VRegister temp = scope.AcquireV(kFormat8H);
2551 if (dst == src1) {
2552 __ Mov(temp, src1.V8H());
2553 src1 = temp;
2554 }
2555 __ Sqxtun(dst.V8B(), src0.V8H());
2556 __ Sqxtun2(dst.V16B(), src1.V8H());
2557 break;
2558 }
2559 case kArm64I8x16BitMask: {
2560 UseScratchRegisterScope scope(tasm());
2561 Register dst = i.OutputRegister32();
2562 VRegister src = i.InputSimd128Register(0);
2563 VRegister tmp = scope.AcquireQ();
2564 VRegister mask = scope.AcquireQ();
2565
2566 // Set i-th bit of each lane i. When AND with tmp, the lanes that
2567 // are signed will have i-th bit set, unsigned will be 0.
2568 __ Sshr(tmp.V16B(), src.V16B(), 7);
2569 __ Movi(mask.V2D(), 0x8040'2010'0804'0201);
2570 __ And(tmp.V16B(), mask.V16B(), tmp.V16B());
2571 __ Ext(mask.V16B(), tmp.V16B(), tmp.V16B(), 8);
2572 __ Zip1(tmp.V16B(), tmp.V16B(), mask.V16B());
2573 __ Addv(tmp.H(), tmp.V8H());
2574 __ Mov(dst.W(), tmp.V8H(), 0);
2575 break;
2576 }
2577 case kArm64S128Const: {
2578 uint64_t imm1 = make_uint64(i.InputUint32(1), i.InputUint32(0));
2579 uint64_t imm2 = make_uint64(i.InputUint32(3), i.InputUint32(2));
2580 __ Movi(i.OutputSimd128Register().V16B(), imm2, imm1);
2581 break;
2582 }
2583 case kArm64S128Zero: {
2584 VRegister dst = i.OutputSimd128Register().V16B();
2585 __ Eor(dst, dst, dst);
2586 break;
2587 }
2588 SIMD_BINOP_CASE(kArm64S128And, And, 16B);
2589 SIMD_BINOP_CASE(kArm64S128Or, Orr, 16B);
2590 SIMD_BINOP_CASE(kArm64S128Xor, Eor, 16B);
2591 SIMD_UNOP_CASE(kArm64S128Not, Mvn, 16B);
2592 case kArm64S128Dup: {
2593 VRegister dst = i.OutputSimd128Register(),
2594 src = i.InputSimd128Register(0);
2595 int lanes = i.InputInt32(1);
2596 int index = i.InputInt32(2);
2597 switch (lanes) {
2598 case 4:
2599 __ Dup(dst.V4S(), src.V4S(), index);
2600 break;
2601 case 8:
2602 __ Dup(dst.V8H(), src.V8H(), index);
2603 break;
2604 case 16:
2605 __ Dup(dst.V16B(), src.V16B(), index);
2606 break;
2607 default:
2608 UNREACHABLE();
2609 }
2610 break;
2611 }
2612 SIMD_DESTRUCTIVE_BINOP_CASE(kArm64S128Select, Bsl, 16B);
2613 SIMD_BINOP_CASE(kArm64S128AndNot, Bic, 16B);
2614 case kArm64Ssra: {
2615 int8_t laneSize = LaneSizeField::decode(opcode);
2616 VectorFormat f = VectorFormatFillQ(laneSize);
2617 int8_t mask = laneSize - 1;
2618 VRegister dst = i.OutputSimd128Register().Format(f);
2619 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f));
2620 __ Ssra(dst, i.InputSimd128Register(1).Format(f), i.InputInt8(2) & mask);
2621 break;
2622 }
2623 case kArm64Usra: {
2624 int8_t laneSize = LaneSizeField::decode(opcode);
2625 VectorFormat f = VectorFormatFillQ(laneSize);
2626 int8_t mask = laneSize - 1;
2627 VRegister dst = i.OutputSimd128Register().Format(f);
2628 DCHECK_EQ(dst, i.InputSimd128Register(0).Format(f));
2629 __ Usra(dst, i.InputSimd128Register(1).Format(f), i.InputUint8(2) & mask);
2630 break;
2631 }
2632 case kArm64S32x4Shuffle: {
2633 Simd128Register dst = i.OutputSimd128Register().V4S(),
2634 src0 = i.InputSimd128Register(0).V4S(),
2635 src1 = i.InputSimd128Register(1).V4S();
2636 // Check for in-place shuffles.
2637 // If dst == src0 == src1, then the shuffle is unary and we only use src0.
2638 UseScratchRegisterScope scope(tasm());
2639 VRegister temp = scope.AcquireV(kFormat4S);
2640 if (dst == src0) {
2641 __ Mov(temp, src0);
2642 src0 = temp;
2643 } else if (dst == src1) {
2644 __ Mov(temp, src1);
2645 src1 = temp;
2646 }
2647 // Perform shuffle as a vmov per lane.
2648 int32_t shuffle = i.InputInt32(2);
2649 for (int i = 0; i < 4; i++) {
2650 VRegister src = src0;
2651 int lane = shuffle & 0x7;
2652 if (lane >= 4) {
2653 src = src1;
2654 lane &= 0x3;
2655 }
2656 __ Mov(dst, i, src, lane);
2657 shuffle >>= 8;
2658 }
2659 break;
2660 }
2661 SIMD_BINOP_CASE(kArm64S32x4ZipLeft, Zip1, 4S);
2662 SIMD_BINOP_CASE(kArm64S32x4ZipRight, Zip2, 4S);
2663 SIMD_BINOP_CASE(kArm64S32x4UnzipLeft, Uzp1, 4S);
2664 SIMD_BINOP_CASE(kArm64S32x4UnzipRight, Uzp2, 4S);
2665 SIMD_BINOP_CASE(kArm64S32x4TransposeLeft, Trn1, 4S);
2666 SIMD_BINOP_CASE(kArm64S32x4TransposeRight, Trn2, 4S);
2667 SIMD_BINOP_CASE(kArm64S16x8ZipLeft, Zip1, 8H);
2668 SIMD_BINOP_CASE(kArm64S16x8ZipRight, Zip2, 8H);
2669 SIMD_BINOP_CASE(kArm64S16x8UnzipLeft, Uzp1, 8H);
2670 SIMD_BINOP_CASE(kArm64S16x8UnzipRight, Uzp2, 8H);
2671 SIMD_BINOP_CASE(kArm64S16x8TransposeLeft, Trn1, 8H);
2672 SIMD_BINOP_CASE(kArm64S16x8TransposeRight, Trn2, 8H);
2673 SIMD_BINOP_CASE(kArm64S8x16ZipLeft, Zip1, 16B);
2674 SIMD_BINOP_CASE(kArm64S8x16ZipRight, Zip2, 16B);
2675 SIMD_BINOP_CASE(kArm64S8x16UnzipLeft, Uzp1, 16B);
2676 SIMD_BINOP_CASE(kArm64S8x16UnzipRight, Uzp2, 16B);
2677 SIMD_BINOP_CASE(kArm64S8x16TransposeLeft, Trn1, 16B);
2678 SIMD_BINOP_CASE(kArm64S8x16TransposeRight, Trn2, 16B);
2679 case kArm64S8x16Concat: {
2680 __ Ext(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2681 i.InputSimd128Register(1).V16B(), i.InputInt4(2));
2682 break;
2683 }
2684 case kArm64I8x16Swizzle: {
2685 __ Tbl(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2686 i.InputSimd128Register(1).V16B());
2687 break;
2688 }
2689 case kArm64I8x16Shuffle: {
2690 Simd128Register dst = i.OutputSimd128Register().V16B(),
2691 src0 = i.InputSimd128Register(0).V16B(),
2692 src1 = i.InputSimd128Register(1).V16B();
2693 // Unary shuffle table is in src0, binary shuffle table is in src0, src1,
2694 // which must be consecutive.
2695 if (src0 != src1) {
2696 DCHECK(AreConsecutive(src0, src1));
2697 }
2698
2699 int64_t imm1 = make_uint64(i.InputInt32(3), i.InputInt32(2));
2700 int64_t imm2 = make_uint64(i.InputInt32(5), i.InputInt32(4));
2701 DCHECK_EQ(0, (imm1 | imm2) & (src0 == src1 ? 0xF0F0F0F0F0F0F0F0
2702 : 0xE0E0E0E0E0E0E0E0));
2703
2704 UseScratchRegisterScope scope(tasm());
2705 VRegister temp = scope.AcquireV(kFormat16B);
2706 __ Movi(temp, imm2, imm1);
2707
2708 if (src0 == src1) {
2709 __ Tbl(dst, src0, temp.V16B());
2710 } else {
2711 __ Tbl(dst, src0, src1, temp.V16B());
2712 }
2713 break;
2714 }
2715 SIMD_UNOP_CASE(kArm64S32x2Reverse, Rev64, 4S);
2716 SIMD_UNOP_CASE(kArm64S16x4Reverse, Rev64, 8H);
2717 SIMD_UNOP_CASE(kArm64S16x2Reverse, Rev32, 8H);
2718 SIMD_UNOP_CASE(kArm64S8x8Reverse, Rev64, 16B);
2719 SIMD_UNOP_CASE(kArm64S8x4Reverse, Rev32, 16B);
2720 SIMD_UNOP_CASE(kArm64S8x2Reverse, Rev16, 16B);
2721 case kArm64LoadSplat: {
2722 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2723 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2724 __ ld1r(i.OutputSimd128Register().Format(f), i.MemoryOperand(0));
2725 break;
2726 }
2727 case kArm64LoadLane: {
2728 DCHECK_EQ(i.OutputSimd128Register(), i.InputSimd128Register(0));
2729 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2730 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2731 int laneidx = i.InputInt8(1);
2732 __ ld1(i.OutputSimd128Register().Format(f), laneidx, i.MemoryOperand(2));
2733 break;
2734 }
2735 case kArm64StoreLane: {
2736 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2737 VectorFormat f = VectorFormatFillQ(LaneSizeField::decode(opcode));
2738 int laneidx = i.InputInt8(1);
2739 __ st1(i.InputSimd128Register(0).Format(f), laneidx, i.MemoryOperand(2));
2740 break;
2741 }
2742 case kArm64S128Load8x8S: {
2743 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2744 __ Ldr(i.OutputSimd128Register().V8B(), i.MemoryOperand(0));
2745 __ Sxtl(i.OutputSimd128Register().V8H(), i.OutputSimd128Register().V8B());
2746 break;
2747 }
2748 case kArm64S128Load8x8U: {
2749 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2750 __ Ldr(i.OutputSimd128Register().V8B(), i.MemoryOperand(0));
2751 __ Uxtl(i.OutputSimd128Register().V8H(), i.OutputSimd128Register().V8B());
2752 break;
2753 }
2754 case kArm64S128Load16x4S: {
2755 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2756 __ Ldr(i.OutputSimd128Register().V4H(), i.MemoryOperand(0));
2757 __ Sxtl(i.OutputSimd128Register().V4S(), i.OutputSimd128Register().V4H());
2758 break;
2759 }
2760 case kArm64S128Load16x4U: {
2761 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2762 __ Ldr(i.OutputSimd128Register().V4H(), i.MemoryOperand(0));
2763 __ Uxtl(i.OutputSimd128Register().V4S(), i.OutputSimd128Register().V4H());
2764 break;
2765 }
2766 case kArm64S128Load32x2S: {
2767 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2768 __ Ldr(i.OutputSimd128Register().V2S(), i.MemoryOperand(0));
2769 __ Sxtl(i.OutputSimd128Register().V2D(), i.OutputSimd128Register().V2S());
2770 break;
2771 }
2772 case kArm64S128Load32x2U: {
2773 EmitOOLTrapIfNeeded(zone(), this, opcode, instr, __ pc_offset());
2774 __ Ldr(i.OutputSimd128Register().V2S(), i.MemoryOperand(0));
2775 __ Uxtl(i.OutputSimd128Register().V2D(), i.OutputSimd128Register().V2S());
2776 break;
2777 }
2778 case kArm64I64x2AllTrue: {
2779 __ I64x2AllTrue(i.OutputRegister32(), i.InputSimd128Register(0));
2780 break;
2781 }
2782 #define SIMD_REDUCE_OP_CASE(Op, Instr, format, FORMAT) \
2783 case Op: { \
2784 UseScratchRegisterScope scope(tasm()); \
2785 VRegister temp = scope.AcquireV(format); \
2786 __ Instr(temp, i.InputSimd128Register(0).V##FORMAT()); \
2787 __ Umov(i.OutputRegister32(), temp, 0); \
2788 __ Cmp(i.OutputRegister32(), 0); \
2789 __ Cset(i.OutputRegister32(), ne); \
2790 break; \
2791 }
2792 // For AnyTrue, the format does not matter.
2793 SIMD_REDUCE_OP_CASE(kArm64V128AnyTrue, Umaxv, kFormatS, 4S);
2794 SIMD_REDUCE_OP_CASE(kArm64I32x4AllTrue, Uminv, kFormatS, 4S);
2795 SIMD_REDUCE_OP_CASE(kArm64I16x8AllTrue, Uminv, kFormatH, 8H);
2796 SIMD_REDUCE_OP_CASE(kArm64I8x16AllTrue, Uminv, kFormatB, 16B);
2797 }
2798 return kSuccess;
2799 }
2800
2801 #undef SIMD_UNOP_CASE
2802 #undef SIMD_UNOP_LANE_SIZE_CASE
2803 #undef SIMD_BINOP_CASE
2804 #undef SIMD_BINOP_LANE_SIZE_CASE
2805 #undef SIMD_DESTRUCTIVE_BINOP_CASE
2806 #undef SIMD_DESTRUCTIVE_BINOP_LANE_SIZE_CASE
2807 #undef SIMD_REDUCE_OP_CASE
2808 #undef ASSEMBLE_SIMD_SHIFT_LEFT
2809 #undef ASSEMBLE_SIMD_SHIFT_RIGHT
2810
2811 // Assemble branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2812 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2813 Arm64OperandConverter i(this, instr);
2814 Label* tlabel = branch->true_label;
2815 Label* flabel = branch->false_label;
2816 FlagsCondition condition = branch->condition;
2817 ArchOpcode opcode = instr->arch_opcode();
2818
2819 if (opcode == kArm64CompareAndBranch32) {
2820 switch (condition) {
2821 case kEqual:
2822 __ Cbz(i.InputRegister32(0), tlabel);
2823 break;
2824 case kNotEqual:
2825 __ Cbnz(i.InputRegister32(0), tlabel);
2826 break;
2827 default:
2828 UNREACHABLE();
2829 }
2830 } else if (opcode == kArm64CompareAndBranch) {
2831 switch (condition) {
2832 case kEqual:
2833 __ Cbz(i.InputRegister64(0), tlabel);
2834 break;
2835 case kNotEqual:
2836 __ Cbnz(i.InputRegister64(0), tlabel);
2837 break;
2838 default:
2839 UNREACHABLE();
2840 }
2841 } else if (opcode == kArm64TestAndBranch32) {
2842 switch (condition) {
2843 case kEqual:
2844 __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2845 break;
2846 case kNotEqual:
2847 __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2848 break;
2849 default:
2850 UNREACHABLE();
2851 }
2852 } else if (opcode == kArm64TestAndBranch) {
2853 switch (condition) {
2854 case kEqual:
2855 __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2856 break;
2857 case kNotEqual:
2858 __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2859 break;
2860 default:
2861 UNREACHABLE();
2862 }
2863 } else {
2864 Condition cc = FlagsConditionToCondition(condition);
2865 __ B(cc, tlabel);
2866 }
2867 if (!branch->fallthru) __ B(flabel); // no fallthru to flabel.
2868 }
2869
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)2870 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2871 BranchInfo* branch) {
2872 AssembleArchBranch(instr, branch);
2873 }
2874
AssembleArchJump(RpoNumber target)2875 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2876 if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target));
2877 }
2878
2879 #if V8_ENABLE_WEBASSEMBLY
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2880 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2881 FlagsCondition condition) {
2882 auto ool = zone()->New<WasmOutOfLineTrap>(this, instr);
2883 Label* tlabel = ool->entry();
2884 Condition cc = FlagsConditionToCondition(condition);
2885 __ B(cc, tlabel);
2886 }
2887 #endif // V8_ENABLE_WEBASSEMBLY
2888
2889 // Assemble boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2890 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2891 FlagsCondition condition) {
2892 Arm64OperandConverter i(this, instr);
2893
2894 // Materialize a full 64-bit 1 or 0 value. The result register is always the
2895 // last output of the instruction.
2896 DCHECK_NE(0u, instr->OutputCount());
2897 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2898 Condition cc = FlagsConditionToCondition(condition);
2899 __ Cset(reg, cc);
2900 }
2901
AssembleArchSelect(Instruction * instr,FlagsCondition condition)2902 void CodeGenerator::AssembleArchSelect(Instruction* instr,
2903 FlagsCondition condition) {
2904 Arm64OperandConverter i(this, instr);
2905 MachineRepresentation rep =
2906 LocationOperand::cast(instr->OutputAt(0))->representation();
2907 Condition cc = FlagsConditionToCondition(condition);
2908 // We don't now how many inputs were consumed by the condition, so we have to
2909 // calculate the indices of the last two inputs.
2910 DCHECK_GE(instr->InputCount(), 2);
2911 size_t true_value_index = instr->InputCount() - 2;
2912 size_t false_value_index = instr->InputCount() - 1;
2913 if (rep == MachineRepresentation::kFloat32) {
2914 __ Fcsel(i.OutputFloat32Register(),
2915 i.InputFloat32Register(true_value_index),
2916 i.InputFloat32Register(false_value_index), cc);
2917 } else {
2918 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
2919 __ Fcsel(i.OutputFloat64Register(),
2920 i.InputFloat64Register(true_value_index),
2921 i.InputFloat64Register(false_value_index), cc);
2922 }
2923 }
2924
AssembleArchBinarySearchSwitch(Instruction * instr)2925 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2926 Arm64OperandConverter i(this, instr);
2927 Register input = i.InputRegister32(0);
2928 std::vector<std::pair<int32_t, Label*>> cases;
2929 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2930 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2931 }
2932 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2933 cases.data() + cases.size());
2934 }
2935
AssembleArchTableSwitch(Instruction * instr)2936 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2937 Arm64OperandConverter i(this, instr);
2938 UseScratchRegisterScope scope(tasm());
2939 Register input = i.InputRegister32(0);
2940 Register temp = scope.AcquireX();
2941 size_t const case_count = instr->InputCount() - 2;
2942 Label table;
2943 __ Cmp(input, case_count);
2944 __ B(hs, GetLabel(i.InputRpo(1)));
2945 __ Adr(temp, &table);
2946 int entry_size_log2 = 2;
2947 #ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
2948 ++entry_size_log2; // Account for BTI.
2949 #endif
2950 __ Add(temp, temp, Operand(input, UXTW, entry_size_log2));
2951 __ Br(temp);
2952 {
2953 TurboAssembler::BlockPoolsScope block_pools(tasm(),
2954 case_count * kInstrSize);
2955 __ Bind(&table);
2956 for (size_t index = 0; index < case_count; ++index) {
2957 __ JumpTarget();
2958 __ B(GetLabel(i.InputRpo(index + 2)));
2959 }
2960 __ JumpTarget();
2961 }
2962 }
2963
FinishFrame(Frame * frame)2964 void CodeGenerator::FinishFrame(Frame* frame) {
2965 auto call_descriptor = linkage()->GetIncomingDescriptor();
2966
2967 // Save FP registers.
2968 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2969 call_descriptor->CalleeSavedFPRegisters());
2970 int saved_count = saves_fp.Count();
2971 if (saved_count != 0) {
2972 DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedV().list());
2973 frame->AllocateSavedCalleeRegisterSlots(saved_count *
2974 (kDoubleSize / kSystemPointerSize));
2975 }
2976
2977 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2978 call_descriptor->CalleeSavedRegisters());
2979 saved_count = saves.Count();
2980 if (saved_count != 0) {
2981 frame->AllocateSavedCalleeRegisterSlots(saved_count);
2982 }
2983 frame->AlignFrame(16);
2984 }
2985
AssembleConstructFrame()2986 void CodeGenerator::AssembleConstructFrame() {
2987 auto call_descriptor = linkage()->GetIncomingDescriptor();
2988 __ AssertSpAligned();
2989
2990 // The frame has been previously padded in CodeGenerator::FinishFrame().
2991 DCHECK_EQ(frame()->GetTotalFrameSlotCount() % 2, 0);
2992 int required_slots =
2993 frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
2994
2995 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2996 call_descriptor->CalleeSavedRegisters());
2997 DCHECK_EQ(saves.Count() % 2, 0);
2998 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2999 call_descriptor->CalleeSavedFPRegisters());
3000 DCHECK_EQ(saves_fp.Count() % 2, 0);
3001 // The number of return slots should be even after aligning the Frame.
3002 const int returns = frame()->GetReturnSlotCount();
3003 DCHECK_EQ(returns % 2, 0);
3004
3005 if (frame_access_state()->has_frame()) {
3006 // Link the frame
3007 if (call_descriptor->IsJSFunctionCall()) {
3008 STATIC_ASSERT(InterpreterFrameConstants::kFixedFrameSize % 16 == 8);
3009 DCHECK_EQ(required_slots % 2, 1);
3010 __ Prologue();
3011 // Update required_slots count since we have just claimed one extra slot.
3012 STATIC_ASSERT(TurboAssembler::kExtraSlotClaimedByPrologue == 1);
3013 required_slots -= TurboAssembler::kExtraSlotClaimedByPrologue;
3014 } else {
3015 __ Push<TurboAssembler::kSignLR>(lr, fp);
3016 __ Mov(fp, sp);
3017 }
3018 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
3019
3020 // Create OSR entry if applicable
3021 if (info()->is_osr()) {
3022 // TurboFan OSR-compiled functions cannot be entered directly.
3023 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
3024
3025 // Unoptimized code jumps directly to this entrypoint while the
3026 // unoptimized frame is still on the stack. Optimized code uses OSR values
3027 // directly from the unoptimized frame. Thus, all that needs to be done is
3028 // to allocate the remaining stack slots.
3029 __ RecordComment("-- OSR entrypoint --");
3030 osr_pc_offset_ = __ pc_offset();
3031 __ CodeEntry();
3032 size_t unoptimized_frame_slots = osr_helper()->UnoptimizedFrameSlots();
3033 DCHECK(call_descriptor->IsJSFunctionCall());
3034 DCHECK_EQ(unoptimized_frame_slots % 2, 1);
3035 // One unoptimized frame slot has already been claimed when the actual
3036 // arguments count was pushed.
3037 required_slots -=
3038 unoptimized_frame_slots - TurboAssembler::kExtraSlotClaimedByPrologue;
3039 }
3040
3041 #if V8_ENABLE_WEBASSEMBLY
3042 if (info()->IsWasm() && required_slots * kSystemPointerSize > 4 * KB) {
3043 // For WebAssembly functions with big frames we have to do the stack
3044 // overflow check before we construct the frame. Otherwise we may not
3045 // have enough space on the stack to call the runtime for the stack
3046 // overflow.
3047 Label done;
3048 // If the frame is bigger than the stack, we throw the stack overflow
3049 // exception unconditionally. Thereby we can avoid the integer overflow
3050 // check in the condition code.
3051 if (required_slots * kSystemPointerSize < FLAG_stack_size * KB) {
3052 UseScratchRegisterScope scope(tasm());
3053 Register scratch = scope.AcquireX();
3054 __ Ldr(scratch, FieldMemOperand(
3055 kWasmInstanceRegister,
3056 WasmInstanceObject::kRealStackLimitAddressOffset));
3057 __ Ldr(scratch, MemOperand(scratch));
3058 __ Add(scratch, scratch, required_slots * kSystemPointerSize);
3059 __ Cmp(sp, scratch);
3060 __ B(hs, &done);
3061 }
3062
3063 {
3064 // Finish the frame that hasn't been fully built yet.
3065 UseScratchRegisterScope temps(tasm());
3066 Register scratch = temps.AcquireX();
3067 __ Mov(scratch,
3068 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3069 __ Push(scratch, kWasmInstanceRegister);
3070 }
3071
3072 __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
3073 // The call does not return, hence we can ignore any references and just
3074 // define an empty safepoint.
3075 ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
3076 RecordSafepoint(reference_map);
3077 if (FLAG_debug_code) __ Brk(0);
3078 __ Bind(&done);
3079 }
3080 #endif // V8_ENABLE_WEBASSEMBLY
3081
3082 // Skip callee-saved slots, which are pushed below.
3083 required_slots -= saves.Count();
3084 required_slots -= saves_fp.Count();
3085 required_slots -= returns;
3086
3087 // Build remainder of frame, including accounting for and filling-in
3088 // frame-specific header information, i.e. claiming the extra slot that
3089 // other platforms explicitly push for STUB (code object) frames and frames
3090 // recording their argument count.
3091 switch (call_descriptor->kind()) {
3092 case CallDescriptor::kCallJSFunction:
3093 __ Claim(required_slots);
3094 break;
3095 case CallDescriptor::kCallCodeObject: {
3096 UseScratchRegisterScope temps(tasm());
3097 Register scratch = temps.AcquireX();
3098 __ Mov(scratch,
3099 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3100 __ Push(scratch, padreg);
3101 // One of the extra slots has just been claimed when pushing the frame
3102 // type marker above. We also know that we have at least one slot to
3103 // claim here, as the typed frame has an odd number of fixed slots, and
3104 // all other parts of the total frame slots are even, leaving
3105 // {required_slots} to be odd.
3106 DCHECK_GE(required_slots, 1);
3107 __ Claim(required_slots - 1);
3108 } break;
3109 #if V8_ENABLE_WEBASSEMBLY
3110 case CallDescriptor::kCallWasmFunction: {
3111 UseScratchRegisterScope temps(tasm());
3112 Register scratch = temps.AcquireX();
3113 __ Mov(scratch,
3114 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3115 __ Push(scratch, kWasmInstanceRegister);
3116 __ Claim(required_slots);
3117 } break;
3118 case CallDescriptor::kCallWasmImportWrapper:
3119 case CallDescriptor::kCallWasmCapiFunction: {
3120 UseScratchRegisterScope temps(tasm());
3121 __ LoadTaggedPointerField(
3122 kJSFunctionRegister,
3123 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue2Offset));
3124 __ LoadTaggedPointerField(
3125 kWasmInstanceRegister,
3126 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue1Offset));
3127 Register scratch = temps.AcquireX();
3128 __ Mov(scratch,
3129 StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
3130 __ Push(scratch, kWasmInstanceRegister);
3131 int extra_slots =
3132 call_descriptor->kind() == CallDescriptor::kCallWasmImportWrapper
3133 ? 0 // Import wrapper: none.
3134 : 1; // C-API function: PC.
3135 __ Claim(required_slots + extra_slots);
3136 } break;
3137 #endif // V8_ENABLE_WEBASSEMBLY
3138 case CallDescriptor::kCallAddress:
3139 #if V8_ENABLE_WEBASSEMBLY
3140 if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
3141 UseScratchRegisterScope temps(tasm());
3142 Register scratch = temps.AcquireX();
3143 __ Mov(scratch, StackFrame::TypeToMarker(StackFrame::C_WASM_ENTRY));
3144 __ Push(scratch, padreg);
3145 // The additional slot will be used for the saved c_entry_fp.
3146 }
3147 #endif // V8_ENABLE_WEBASSEMBLY
3148 __ Claim(required_slots);
3149 break;
3150 default:
3151 UNREACHABLE();
3152 }
3153 }
3154
3155 // Save FP registers.
3156 DCHECK_IMPLIES(saves_fp.Count() != 0,
3157 saves_fp.list() == CPURegList::GetCalleeSavedV().list());
3158 __ PushCPURegList(saves_fp);
3159
3160 // Save registers.
3161 __ PushCPURegList<TurboAssembler::kSignLR>(saves);
3162
3163 if (returns != 0) {
3164 __ Claim(returns);
3165 }
3166 }
3167
AssembleReturn(InstructionOperand * additional_pop_count)3168 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
3169 auto call_descriptor = linkage()->GetIncomingDescriptor();
3170
3171 const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
3172 if (returns != 0) {
3173 __ Drop(returns);
3174 }
3175
3176 // Restore registers.
3177 CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
3178 call_descriptor->CalleeSavedRegisters());
3179 __ PopCPURegList<TurboAssembler::kAuthLR>(saves);
3180
3181 // Restore fp registers.
3182 CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
3183 call_descriptor->CalleeSavedFPRegisters());
3184 __ PopCPURegList(saves_fp);
3185
3186 unwinding_info_writer_.MarkBlockWillExit();
3187
3188 const int parameter_slots =
3189 static_cast<int>(call_descriptor->ParameterSlotCount());
3190 Arm64OperandConverter g(this, nullptr);
3191
3192 // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
3193 // Check RawMachineAssembler::PopAndReturn.
3194 if (parameter_slots != 0) {
3195 if (additional_pop_count->IsImmediate()) {
3196 DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
3197 } else if (FLAG_debug_code) {
3198 __ cmp(g.ToRegister(additional_pop_count), Operand(0));
3199 __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue);
3200 }
3201 }
3202
3203 Register argc_reg = x3;
3204 // Functions with JS linkage have at least one parameter (the receiver).
3205 // If {parameter_slots} == 0, it means it is a builtin with
3206 // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
3207 // itself.
3208 const bool drop_jsargs = parameter_slots != 0 &&
3209 frame_access_state()->has_frame() &&
3210 call_descriptor->IsJSFunctionCall();
3211 if (call_descriptor->IsCFunctionCall()) {
3212 AssembleDeconstructFrame();
3213 } else if (frame_access_state()->has_frame()) {
3214 // Canonicalize JSFunction return sites for now unless they have an variable
3215 // number of stack slot pops.
3216 if (additional_pop_count->IsImmediate() &&
3217 g.ToConstant(additional_pop_count).ToInt32() == 0) {
3218 if (return_label_.is_bound()) {
3219 __ B(&return_label_);
3220 return;
3221 } else {
3222 __ Bind(&return_label_);
3223 }
3224 }
3225 if (drop_jsargs) {
3226 // Get the actual argument count.
3227 DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & argc_reg.bit());
3228 __ Ldr(argc_reg, MemOperand(fp, StandardFrameConstants::kArgCOffset));
3229 }
3230 AssembleDeconstructFrame();
3231 }
3232
3233 if (drop_jsargs) {
3234 // We must pop all arguments from the stack (including the receiver). This
3235 // number of arguments is given by max(1 + argc_reg, parameter_slots).
3236 Label argc_reg_has_final_count;
3237 DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & argc_reg.bit());
3238 if (!kJSArgcIncludesReceiver) {
3239 __ Add(argc_reg, argc_reg, 1); // Consider the receiver.
3240 }
3241 if (parameter_slots > 1) {
3242 __ Cmp(argc_reg, Operand(parameter_slots));
3243 __ B(&argc_reg_has_final_count, ge);
3244 __ Mov(argc_reg, Operand(parameter_slots));
3245 __ Bind(&argc_reg_has_final_count);
3246 }
3247 __ DropArguments(argc_reg);
3248 } else if (additional_pop_count->IsImmediate()) {
3249 int additional_count = g.ToConstant(additional_pop_count).ToInt32();
3250 __ DropArguments(parameter_slots + additional_count);
3251 } else if (parameter_slots == 0) {
3252 __ DropArguments(g.ToRegister(additional_pop_count));
3253 } else {
3254 // {additional_pop_count} is guaranteed to be zero if {parameter_slots !=
3255 // 0}. Check RawMachineAssembler::PopAndReturn.
3256 __ DropArguments(parameter_slots);
3257 }
3258 __ AssertSpAligned();
3259 __ Ret();
3260 }
3261
FinishCode()3262 void CodeGenerator::FinishCode() { __ ForceConstantPoolEmissionWithoutJump(); }
3263
PrepareForDeoptimizationExits(ZoneDeque<DeoptimizationExit * > * exits)3264 void CodeGenerator::PrepareForDeoptimizationExits(
3265 ZoneDeque<DeoptimizationExit*>* exits) {
3266 __ ForceConstantPoolEmissionWithoutJump();
3267 // We are conservative here, assuming all deopts are eager with resume deopts.
3268 DCHECK_GE(Deoptimizer::kEagerWithResumeDeoptExitSize,
3269 Deoptimizer::kLazyDeoptExitSize);
3270 DCHECK_GE(Deoptimizer::kLazyDeoptExitSize,
3271 Deoptimizer::kNonLazyDeoptExitSize);
3272 __ CheckVeneerPool(false, false,
3273 static_cast<int>(exits->size()) *
3274 Deoptimizer::kEagerWithResumeDeoptExitSize);
3275
3276 // Check which deopt kinds exist in this Code object, to avoid emitting jumps
3277 // to unused entries.
3278 bool saw_deopt_kind[kDeoptimizeKindCount] = {false};
3279 bool saw_deopt_with_resume_reason[kDeoptimizeReasonCount] = {false};
3280 for (auto exit : *exits) {
3281 saw_deopt_kind[static_cast<int>(exit->kind())] = true;
3282 if (exit->kind() == DeoptimizeKind::kEagerWithResume) {
3283 saw_deopt_with_resume_reason[static_cast<int>(exit->reason())] = true;
3284 }
3285 }
3286
3287 // Emit the jumps to deoptimization entries.
3288 UseScratchRegisterScope scope(tasm());
3289 Register scratch = scope.AcquireX();
3290 STATIC_ASSERT(static_cast<int>(kFirstDeoptimizeKind) == 0);
3291 for (int i = 0; i < kDeoptimizeKindCount; i++) {
3292 if (!saw_deopt_kind[i]) continue;
3293 DeoptimizeKind kind = static_cast<DeoptimizeKind>(i);
3294 if (kind == DeoptimizeKind::kEagerWithResume) {
3295 for (int j = 0; j < kDeoptimizeReasonCount; j++) {
3296 if (!saw_deopt_with_resume_reason[j]) continue;
3297 DeoptimizeReason reason = static_cast<DeoptimizeReason>(j);
3298 __ bind(&jump_deoptimization_or_resume_entry_labels_[j]);
3299 __ LoadEntryFromBuiltin(Deoptimizer::GetDeoptWithResumeBuiltin(reason),
3300 scratch);
3301 __ Jump(scratch);
3302 }
3303 } else {
3304 __ bind(&jump_deoptimization_entry_labels_[i]);
3305 __ LoadEntryFromBuiltin(Deoptimizer::GetDeoptimizationEntry(kind),
3306 scratch);
3307 __ Jump(scratch);
3308 }
3309 }
3310 }
3311
AssembleMove(InstructionOperand * source,InstructionOperand * destination)3312 void CodeGenerator::AssembleMove(InstructionOperand* source,
3313 InstructionOperand* destination) {
3314 Arm64OperandConverter g(this, nullptr);
3315 // Helper function to write the given constant to the dst register.
3316 auto MoveConstantToRegister = [&](Register dst, Constant src) {
3317 if (src.type() == Constant::kHeapObject) {
3318 Handle<HeapObject> src_object = src.ToHeapObject();
3319 RootIndex index;
3320 if (IsMaterializableFromRoot(src_object, &index)) {
3321 __ LoadRoot(dst, index);
3322 } else {
3323 __ Mov(dst, src_object);
3324 }
3325 } else if (src.type() == Constant::kCompressedHeapObject) {
3326 Handle<HeapObject> src_object = src.ToHeapObject();
3327 RootIndex index;
3328 if (IsMaterializableFromRoot(src_object, &index)) {
3329 __ LoadRoot(dst, index);
3330 } else {
3331 // TODO(v8:8977): Even though this mov happens on 32 bits (Note the
3332 // .W()) and we are passing along the RelocInfo, we still haven't made
3333 // the address embedded in the code-stream actually be compressed.
3334 __ Mov(dst.W(),
3335 Immediate(src_object, RelocInfo::COMPRESSED_EMBEDDED_OBJECT));
3336 }
3337 } else {
3338 __ Mov(dst, g.ToImmediate(source));
3339 }
3340 };
3341 switch (MoveType::InferMove(source, destination)) {
3342 case MoveType::kRegisterToRegister:
3343 if (source->IsRegister()) {
3344 __ Mov(g.ToRegister(destination), g.ToRegister(source));
3345 } else if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3346 __ Mov(g.ToDoubleRegister(destination), g.ToDoubleRegister(source));
3347 } else {
3348 DCHECK(source->IsSimd128Register());
3349 __ Mov(g.ToDoubleRegister(destination).Q(),
3350 g.ToDoubleRegister(source).Q());
3351 }
3352 return;
3353 case MoveType::kRegisterToStack: {
3354 MemOperand dst = g.ToMemOperand(destination, tasm());
3355 if (source->IsRegister()) {
3356 __ Str(g.ToRegister(source), dst);
3357 } else {
3358 VRegister src = g.ToDoubleRegister(source);
3359 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3360 __ Str(src, dst);
3361 } else {
3362 DCHECK(source->IsSimd128Register());
3363 __ Str(src.Q(), dst);
3364 }
3365 }
3366 return;
3367 }
3368 case MoveType::kStackToRegister: {
3369 MemOperand src = g.ToMemOperand(source, tasm());
3370 if (destination->IsRegister()) {
3371 __ Ldr(g.ToRegister(destination), src);
3372 } else {
3373 VRegister dst = g.ToDoubleRegister(destination);
3374 if (destination->IsFloatRegister() || destination->IsDoubleRegister()) {
3375 __ Ldr(dst, src);
3376 } else {
3377 DCHECK(destination->IsSimd128Register());
3378 __ Ldr(dst.Q(), src);
3379 }
3380 }
3381 return;
3382 }
3383 case MoveType::kStackToStack: {
3384 MemOperand src = g.ToMemOperand(source, tasm());
3385 MemOperand dst = g.ToMemOperand(destination, tasm());
3386 if (source->IsSimd128StackSlot()) {
3387 UseScratchRegisterScope scope(tasm());
3388 VRegister temp = scope.AcquireQ();
3389 __ Ldr(temp, src);
3390 __ Str(temp, dst);
3391 } else {
3392 UseScratchRegisterScope scope(tasm());
3393 Register temp = scope.AcquireX();
3394 __ Ldr(temp, src);
3395 __ Str(temp, dst);
3396 }
3397 return;
3398 }
3399 case MoveType::kConstantToRegister: {
3400 Constant src = g.ToConstant(source);
3401 if (destination->IsRegister()) {
3402 MoveConstantToRegister(g.ToRegister(destination), src);
3403 } else {
3404 VRegister dst = g.ToDoubleRegister(destination);
3405 if (destination->IsFloatRegister()) {
3406 __ Fmov(dst.S(), src.ToFloat32());
3407 } else {
3408 DCHECK(destination->IsDoubleRegister());
3409 __ Fmov(dst, src.ToFloat64().value());
3410 }
3411 }
3412 return;
3413 }
3414 case MoveType::kConstantToStack: {
3415 Constant src = g.ToConstant(source);
3416 MemOperand dst = g.ToMemOperand(destination, tasm());
3417 if (destination->IsStackSlot()) {
3418 UseScratchRegisterScope scope(tasm());
3419 Register temp = scope.AcquireX();
3420 MoveConstantToRegister(temp, src);
3421 __ Str(temp, dst);
3422 } else if (destination->IsFloatStackSlot()) {
3423 if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
3424 __ Str(wzr, dst);
3425 } else {
3426 UseScratchRegisterScope scope(tasm());
3427 VRegister temp = scope.AcquireS();
3428 __ Fmov(temp, src.ToFloat32());
3429 __ Str(temp, dst);
3430 }
3431 } else {
3432 DCHECK(destination->IsDoubleStackSlot());
3433 if (src.ToFloat64().AsUint64() == 0) {
3434 __ Str(xzr, dst);
3435 } else {
3436 UseScratchRegisterScope scope(tasm());
3437 VRegister temp = scope.AcquireD();
3438 __ Fmov(temp, src.ToFloat64().value());
3439 __ Str(temp, dst);
3440 }
3441 }
3442 return;
3443 }
3444 }
3445 UNREACHABLE();
3446 }
3447
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)3448 void CodeGenerator::AssembleSwap(InstructionOperand* source,
3449 InstructionOperand* destination) {
3450 Arm64OperandConverter g(this, nullptr);
3451 switch (MoveType::InferSwap(source, destination)) {
3452 case MoveType::kRegisterToRegister:
3453 if (source->IsRegister()) {
3454 __ Swap(g.ToRegister(source), g.ToRegister(destination));
3455 } else {
3456 VRegister src = g.ToDoubleRegister(source);
3457 VRegister dst = g.ToDoubleRegister(destination);
3458 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3459 __ Swap(src, dst);
3460 } else {
3461 DCHECK(source->IsSimd128Register());
3462 __ Swap(src.Q(), dst.Q());
3463 }
3464 }
3465 return;
3466 case MoveType::kRegisterToStack: {
3467 UseScratchRegisterScope scope(tasm());
3468 MemOperand dst = g.ToMemOperand(destination, tasm());
3469 if (source->IsRegister()) {
3470 Register temp = scope.AcquireX();
3471 Register src = g.ToRegister(source);
3472 __ Mov(temp, src);
3473 __ Ldr(src, dst);
3474 __ Str(temp, dst);
3475 } else {
3476 UseScratchRegisterScope scope(tasm());
3477 VRegister src = g.ToDoubleRegister(source);
3478 if (source->IsFloatRegister() || source->IsDoubleRegister()) {
3479 VRegister temp = scope.AcquireD();
3480 __ Mov(temp, src);
3481 __ Ldr(src, dst);
3482 __ Str(temp, dst);
3483 } else {
3484 DCHECK(source->IsSimd128Register());
3485 VRegister temp = scope.AcquireQ();
3486 __ Mov(temp, src.Q());
3487 __ Ldr(src.Q(), dst);
3488 __ Str(temp, dst);
3489 }
3490 }
3491 return;
3492 }
3493 case MoveType::kStackToStack: {
3494 UseScratchRegisterScope scope(tasm());
3495 MemOperand src = g.ToMemOperand(source, tasm());
3496 MemOperand dst = g.ToMemOperand(destination, tasm());
3497 VRegister temp_0 = scope.AcquireD();
3498 VRegister temp_1 = scope.AcquireD();
3499 if (source->IsSimd128StackSlot()) {
3500 __ Ldr(temp_0.Q(), src);
3501 __ Ldr(temp_1.Q(), dst);
3502 __ Str(temp_0.Q(), dst);
3503 __ Str(temp_1.Q(), src);
3504 } else {
3505 __ Ldr(temp_0, src);
3506 __ Ldr(temp_1, dst);
3507 __ Str(temp_0, dst);
3508 __ Str(temp_1, src);
3509 }
3510 return;
3511 }
3512 default:
3513 UNREACHABLE();
3514 }
3515 }
3516
AssembleJumpTable(Label ** targets,size_t target_count)3517 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3518 // On 64-bit ARM we emit the jump tables inline.
3519 UNREACHABLE();
3520 }
3521
3522 #undef __
3523
3524 } // namespace compiler
3525 } // namespace internal
3526 } // namespace v8
3527