1 // Copyright 2021 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/assembler-inl.h"
6 #include "src/codegen/callable.h"
7 #include "src/codegen/macro-assembler.h"
8 #include "src/codegen/optimized-compilation-info.h"
9 #include "src/codegen/riscv64/constants-riscv64.h"
10 #include "src/compiler/backend/code-generator-impl.h"
11 #include "src/compiler/backend/code-generator.h"
12 #include "src/compiler/backend/gap-resolver.h"
13 #include "src/compiler/node-matchers.h"
14 #include "src/compiler/osr.h"
15 #include "src/heap/memory-chunk.h"
16 #include "src/wasm/wasm-code-manager.h"
17
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21
22 #define __ tasm()->
23
24 // TODO(plind): consider renaming these macros.
25 #define TRACE_MSG(msg) \
26 PrintF("code_gen: \'%s\' in function %s at line %d\n", msg, __FUNCTION__, \
27 __LINE__)
28
29 #define TRACE_UNIMPL() \
30 PrintF("UNIMPLEMENTED code_generator_riscv64: %s at line %d\n", \
31 __FUNCTION__, __LINE__)
32
33 // Adds RISC-V-specific methods to convert InstructionOperands.
34 class RiscvOperandConverter final : public InstructionOperandConverter {
35 public:
RiscvOperandConverter(CodeGenerator * gen,Instruction * instr)36 RiscvOperandConverter(CodeGenerator* gen, Instruction* instr)
37 : InstructionOperandConverter(gen, instr) {}
38
OutputSingleRegister(size_t index=0)39 FloatRegister OutputSingleRegister(size_t index = 0) {
40 return ToSingleRegister(instr_->OutputAt(index));
41 }
42
InputSingleRegister(size_t index)43 FloatRegister InputSingleRegister(size_t index) {
44 return ToSingleRegister(instr_->InputAt(index));
45 }
46
ToSingleRegister(InstructionOperand * op)47 FloatRegister ToSingleRegister(InstructionOperand* op) {
48 // Single (Float) and Double register namespace is same on RISC-V,
49 // both are typedefs of FPURegister.
50 return ToDoubleRegister(op);
51 }
52
InputOrZeroRegister(size_t index)53 Register InputOrZeroRegister(size_t index) {
54 if (instr_->InputAt(index)->IsImmediate()) {
55 Constant constant = ToConstant(instr_->InputAt(index));
56 switch (constant.type()) {
57 case Constant::kInt32:
58 case Constant::kInt64:
59 DCHECK_EQ(0, InputInt32(index));
60 break;
61 case Constant::kFloat32:
62 DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
63 break;
64 case Constant::kFloat64:
65 DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
66 break;
67 default:
68 UNREACHABLE();
69 }
70 return zero_reg;
71 }
72 return InputRegister(index);
73 }
74
InputOrZeroDoubleRegister(size_t index)75 DoubleRegister InputOrZeroDoubleRegister(size_t index) {
76 if (instr_->InputAt(index)->IsImmediate()) return kDoubleRegZero;
77
78 return InputDoubleRegister(index);
79 }
80
InputOrZeroSingleRegister(size_t index)81 DoubleRegister InputOrZeroSingleRegister(size_t index) {
82 if (instr_->InputAt(index)->IsImmediate()) return kDoubleRegZero;
83
84 return InputSingleRegister(index);
85 }
86
InputImmediate(size_t index)87 Operand InputImmediate(size_t index) {
88 Constant constant = ToConstant(instr_->InputAt(index));
89 switch (constant.type()) {
90 case Constant::kInt32:
91 return Operand(constant.ToInt32());
92 case Constant::kInt64:
93 return Operand(constant.ToInt64());
94 case Constant::kFloat32:
95 return Operand::EmbeddedNumber(constant.ToFloat32());
96 case Constant::kFloat64:
97 return Operand::EmbeddedNumber(constant.ToFloat64().value());
98 case Constant::kExternalReference:
99 case Constant::kCompressedHeapObject:
100 case Constant::kHeapObject:
101 // TODO(plind): Maybe we should handle ExtRef & HeapObj here?
102 // maybe not done on arm due to const pool ??
103 break;
104 case Constant::kDelayedStringConstant:
105 return Operand::EmbeddedStringConstant(
106 constant.ToDelayedStringConstant());
107 case Constant::kRpoNumber:
108 UNREACHABLE(); // TODO(titzer): RPO immediates
109 }
110 UNREACHABLE();
111 }
112
InputOperand(size_t index)113 Operand InputOperand(size_t index) {
114 InstructionOperand* op = instr_->InputAt(index);
115 if (op->IsRegister()) {
116 return Operand(ToRegister(op));
117 }
118 return InputImmediate(index);
119 }
120
MemoryOperand(size_t * first_index)121 MemOperand MemoryOperand(size_t* first_index) {
122 const size_t index = *first_index;
123 switch (AddressingModeField::decode(instr_->opcode())) {
124 case kMode_None:
125 break;
126 case kMode_MRI:
127 *first_index += 2;
128 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
129 case kMode_MRR:
130 // TODO(plind): r6 address mode, to be implemented ...
131 UNREACHABLE();
132 }
133 UNREACHABLE();
134 }
135
MemoryOperand(size_t index=0)136 MemOperand MemoryOperand(size_t index = 0) { return MemoryOperand(&index); }
137
ToMemOperand(InstructionOperand * op) const138 MemOperand ToMemOperand(InstructionOperand* op) const {
139 DCHECK_NOT_NULL(op);
140 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
141 return SlotToMemOperand(AllocatedOperand::cast(op)->index());
142 }
143
SlotToMemOperand(int slot) const144 MemOperand SlotToMemOperand(int slot) const {
145 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
146 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
147 }
148 };
149
HasRegisterInput(Instruction * instr,size_t index)150 static inline bool HasRegisterInput(Instruction* instr, size_t index) {
151 return instr->InputAt(index)->IsRegister();
152 }
153 namespace {
154
155 class OutOfLineRecordWrite final : public OutOfLineCode {
156 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Register index,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,StubCallMode stub_mode)157 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Register index,
158 Register value, Register scratch0, Register scratch1,
159 RecordWriteMode mode, StubCallMode stub_mode)
160 : OutOfLineCode(gen),
161 object_(object),
162 index_(index),
163 value_(value),
164 scratch0_(scratch0),
165 scratch1_(scratch1),
166 mode_(mode),
167 stub_mode_(stub_mode),
168 must_save_lr_(!gen->frame_access_state()->has_frame()),
169 zone_(gen->zone()) {
170 DCHECK(!AreAliased(object, index, scratch0, scratch1));
171 DCHECK(!AreAliased(value, index, scratch0, scratch1));
172 }
173
Generate()174 void Generate() final {
175 if (COMPRESS_POINTERS_BOOL) {
176 __ DecompressTaggedPointer(value_, value_);
177 }
178 __ CheckPageFlag(value_, scratch0_,
179 MemoryChunk::kPointersToHereAreInterestingMask, eq,
180 exit());
181 __ Add64(scratch1_, object_, index_);
182 RememberedSetAction const remembered_set_action =
183 mode_ > RecordWriteMode::kValueIsMap ? RememberedSetAction::kEmit
184 : RememberedSetAction::kOmit;
185 SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
186 ? SaveFPRegsMode::kSave
187 : SaveFPRegsMode::kIgnore;
188 if (must_save_lr_) {
189 // We need to save and restore ra if the frame was elided.
190 __ Push(ra);
191 }
192 if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
193 __ CallEphemeronKeyBarrier(object_, scratch1_, save_fp_mode);
194 } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
195 // A direct call to a wasm runtime stub defined in this module.
196 // Just encode the stub index. This will be patched when the code
197 // is added to the native module and copied into wasm code space.
198 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
199 remembered_set_action, save_fp_mode,
200 StubCallMode::kCallWasmRuntimeStub);
201 } else {
202 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
203 remembered_set_action, save_fp_mode);
204 }
205 if (must_save_lr_) {
206 __ Pop(ra);
207 }
208 }
209
210 private:
211 Register const object_;
212 Register const index_;
213 Register const value_;
214 Register const scratch0_;
215 Register const scratch1_;
216 RecordWriteMode const mode_;
217 StubCallMode const stub_mode_;
218 bool must_save_lr_;
219 Zone* zone_;
220 };
221
FlagsConditionToConditionCmp(FlagsCondition condition)222 Condition FlagsConditionToConditionCmp(FlagsCondition condition) {
223 switch (condition) {
224 case kEqual:
225 return eq;
226 case kNotEqual:
227 return ne;
228 case kSignedLessThan:
229 return lt;
230 case kSignedGreaterThanOrEqual:
231 return ge;
232 case kSignedLessThanOrEqual:
233 return le;
234 case kSignedGreaterThan:
235 return gt;
236 case kUnsignedLessThan:
237 return Uless;
238 case kUnsignedGreaterThanOrEqual:
239 return Ugreater_equal;
240 case kUnsignedLessThanOrEqual:
241 return Uless_equal;
242 case kUnsignedGreaterThan:
243 return Ugreater;
244 case kUnorderedEqual:
245 case kUnorderedNotEqual:
246 break;
247 default:
248 break;
249 }
250 UNREACHABLE();
251 }
252
FlagsConditionToConditionTst(FlagsCondition condition)253 Condition FlagsConditionToConditionTst(FlagsCondition condition) {
254 switch (condition) {
255 case kNotEqual:
256 return ne;
257 case kEqual:
258 return eq;
259 default:
260 break;
261 }
262 UNREACHABLE();
263 }
264
FlagsConditionToConditionOvf(FlagsCondition condition)265 Condition FlagsConditionToConditionOvf(FlagsCondition condition) {
266 switch (condition) {
267 case kOverflow:
268 return ne;
269 case kNotOverflow:
270 return eq;
271 default:
272 break;
273 }
274 UNREACHABLE();
275 }
276
FlagsConditionToConditionCmpFPU(bool * predicate,FlagsCondition condition)277 FPUCondition FlagsConditionToConditionCmpFPU(bool* predicate,
278 FlagsCondition condition) {
279 switch (condition) {
280 case kEqual:
281 *predicate = true;
282 return EQ;
283 case kNotEqual:
284 *predicate = false;
285 return EQ;
286 case kUnsignedLessThan:
287 *predicate = true;
288 return LT;
289 case kUnsignedGreaterThanOrEqual:
290 *predicate = false;
291 return LT;
292 case kUnsignedLessThanOrEqual:
293 *predicate = true;
294 return LE;
295 case kUnsignedGreaterThan:
296 *predicate = false;
297 return LE;
298 case kUnorderedEqual:
299 case kUnorderedNotEqual:
300 *predicate = true;
301 break;
302 default:
303 *predicate = true;
304 break;
305 }
306 UNREACHABLE();
307 }
308
309 } // namespace
310
311 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \
312 do { \
313 __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
314 __ sync(); \
315 } while (0)
316
317 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \
318 do { \
319 __ sync(); \
320 __ asm_instr(i.InputOrZeroRegister(2), i.MemoryOperand()); \
321 __ sync(); \
322 } while (0)
323
324 #define ASSEMBLE_ATOMIC_BINOP(load_linked, store_conditional, bin_instr) \
325 do { \
326 Label binop; \
327 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
328 __ sync(); \
329 __ bind(&binop); \
330 __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \
331 __ bin_instr(i.TempRegister(1), i.OutputRegister(0), \
332 Operand(i.InputRegister(2))); \
333 __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
334 __ BranchShort(&binop, ne, i.TempRegister(1), Operand(zero_reg)); \
335 __ sync(); \
336 } while (0)
337
338 #define ASSEMBLE_ATOMIC_BINOP_EXT(load_linked, store_conditional, sign_extend, \
339 size, bin_instr, representation) \
340 do { \
341 Label binop; \
342 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
343 if (representation == 32) { \
344 __ And(i.TempRegister(3), i.TempRegister(0), 0x3); \
345 } else { \
346 DCHECK_EQ(representation, 64); \
347 __ And(i.TempRegister(3), i.TempRegister(0), 0x7); \
348 } \
349 __ Sub64(i.TempRegister(0), i.TempRegister(0), \
350 Operand(i.TempRegister(3))); \
351 __ Sll32(i.TempRegister(3), i.TempRegister(3), 3); \
352 __ sync(); \
353 __ bind(&binop); \
354 __ load_linked(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
355 __ ExtractBits(i.OutputRegister(0), i.TempRegister(1), i.TempRegister(3), \
356 size, sign_extend); \
357 __ bin_instr(i.TempRegister(2), i.OutputRegister(0), \
358 Operand(i.InputRegister(2))); \
359 __ InsertBits(i.TempRegister(1), i.TempRegister(2), i.TempRegister(3), \
360 size); \
361 __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
362 __ BranchShort(&binop, ne, i.TempRegister(1), Operand(zero_reg)); \
363 __ sync(); \
364 } while (0)
365
366 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_linked, store_conditional) \
367 do { \
368 Label exchange; \
369 __ sync(); \
370 __ bind(&exchange); \
371 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
372 __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \
373 __ Move(i.TempRegister(1), i.InputRegister(2)); \
374 __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
375 __ BranchShort(&exchange, ne, i.TempRegister(1), Operand(zero_reg)); \
376 __ sync(); \
377 } while (0)
378
379 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT( \
380 load_linked, store_conditional, sign_extend, size, representation) \
381 do { \
382 Label exchange; \
383 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
384 if (representation == 32) { \
385 __ And(i.TempRegister(1), i.TempRegister(0), 0x3); \
386 } else { \
387 DCHECK_EQ(representation, 64); \
388 __ And(i.TempRegister(1), i.TempRegister(0), 0x7); \
389 } \
390 __ Sub64(i.TempRegister(0), i.TempRegister(0), \
391 Operand(i.TempRegister(1))); \
392 __ Sll32(i.TempRegister(1), i.TempRegister(1), 3); \
393 __ sync(); \
394 __ bind(&exchange); \
395 __ load_linked(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
396 __ ExtractBits(i.OutputRegister(0), i.TempRegister(2), i.TempRegister(1), \
397 size, sign_extend); \
398 __ InsertBits(i.TempRegister(2), i.InputRegister(2), i.TempRegister(1), \
399 size); \
400 __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
401 __ BranchShort(&exchange, ne, i.TempRegister(2), Operand(zero_reg)); \
402 __ sync(); \
403 } while (0)
404
405 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_linked, \
406 store_conditional) \
407 do { \
408 Label compareExchange; \
409 Label exit; \
410 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
411 __ sync(); \
412 __ bind(&compareExchange); \
413 __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \
414 __ BranchShort(&exit, ne, i.InputRegister(2), \
415 Operand(i.OutputRegister(0))); \
416 __ Move(i.TempRegister(2), i.InputRegister(3)); \
417 __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
418 __ BranchShort(&compareExchange, ne, i.TempRegister(2), \
419 Operand(zero_reg)); \
420 __ bind(&exit); \
421 __ sync(); \
422 } while (0)
423
424 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT( \
425 load_linked, store_conditional, sign_extend, size, representation) \
426 do { \
427 Label compareExchange; \
428 Label exit; \
429 __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
430 if (representation == 32) { \
431 __ And(i.TempRegister(1), i.TempRegister(0), 0x3); \
432 } else { \
433 DCHECK_EQ(representation, 64); \
434 __ And(i.TempRegister(1), i.TempRegister(0), 0x7); \
435 } \
436 __ Sub64(i.TempRegister(0), i.TempRegister(0), \
437 Operand(i.TempRegister(1))); \
438 __ Sll32(i.TempRegister(1), i.TempRegister(1), 3); \
439 __ sync(); \
440 __ bind(&compareExchange); \
441 __ load_linked(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
442 __ ExtractBits(i.OutputRegister(0), i.TempRegister(2), i.TempRegister(1), \
443 size, sign_extend); \
444 __ ExtractBits(i.InputRegister(2), i.InputRegister(2), 0, size, \
445 sign_extend); \
446 __ BranchShort(&exit, ne, i.InputRegister(2), \
447 Operand(i.OutputRegister(0))); \
448 __ InsertBits(i.TempRegister(2), i.InputRegister(3), i.TempRegister(1), \
449 size); \
450 __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
451 __ BranchShort(&compareExchange, ne, i.TempRegister(2), \
452 Operand(zero_reg)); \
453 __ bind(&exit); \
454 __ sync(); \
455 } while (0)
456
457 #define ASSEMBLE_IEEE754_BINOP(name) \
458 do { \
459 FrameScope scope(tasm(), StackFrame::MANUAL); \
460 __ PrepareCallCFunction(0, 2, kScratchReg); \
461 __ MovToFloatParameters(i.InputDoubleRegister(0), \
462 i.InputDoubleRegister(1)); \
463 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
464 /* Move the result in the double result register. */ \
465 __ MovFromFloatResult(i.OutputDoubleRegister()); \
466 } while (0)
467
468 #define ASSEMBLE_IEEE754_UNOP(name) \
469 do { \
470 FrameScope scope(tasm(), StackFrame::MANUAL); \
471 __ PrepareCallCFunction(0, 1, kScratchReg); \
472 __ MovToFloatParameter(i.InputDoubleRegister(0)); \
473 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
474 /* Move the result in the double result register. */ \
475 __ MovFromFloatResult(i.OutputDoubleRegister()); \
476 } while (0)
477
478 #define ASSEMBLE_F64X2_ARITHMETIC_BINOP(op) \
479 do { \
480 __ op(i.OutputSimd128Register(), i.InputSimd128Register(0), \
481 i.InputSimd128Register(1)); \
482 } while (0)
483
AssembleDeconstructFrame()484 void CodeGenerator::AssembleDeconstructFrame() {
485 __ Move(sp, fp);
486 __ Pop(ra, fp);
487 }
488
AssemblePrepareTailCall()489 void CodeGenerator::AssemblePrepareTailCall() {
490 if (frame_access_state()->has_frame()) {
491 __ Ld(ra, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
492 __ Ld(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
493 }
494 frame_access_state()->SetFrameAccessToSP();
495 }
496
AssembleArchSelect(Instruction * instr,FlagsCondition condition)497 void CodeGenerator::AssembleArchSelect(Instruction* instr,
498 FlagsCondition condition) {
499 UNIMPLEMENTED();
500 }
501
502 namespace {
503
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)504 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
505 FrameAccessState* state,
506 int new_slot_above_sp,
507 bool allow_shrinkage = true) {
508 int current_sp_offset = state->GetSPToFPSlotCount() +
509 StandardFrameConstants::kFixedSlotCountAboveFp;
510 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
511 if (stack_slot_delta > 0) {
512 tasm->Sub64(sp, sp, stack_slot_delta * kSystemPointerSize);
513 state->IncreaseSPDelta(stack_slot_delta);
514 } else if (allow_shrinkage && stack_slot_delta < 0) {
515 tasm->Add64(sp, sp, -stack_slot_delta * kSystemPointerSize);
516 state->IncreaseSPDelta(stack_slot_delta);
517 }
518 }
519
520 } // namespace
521
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_slot_offset)522 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
523 int first_unused_slot_offset) {
524 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
525 first_unused_slot_offset, false);
526 }
527
AssembleTailCallAfterGap(Instruction * instr,int first_unused_slot_offset)528 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
529 int first_unused_slot_offset) {
530 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
531 first_unused_slot_offset);
532 }
533
534 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()535 void CodeGenerator::AssembleCodeStartRegisterCheck() {
536 __ ComputeCodeStartAddress(kScratchReg);
537 __ Assert(eq, AbortReason::kWrongFunctionCodeStart,
538 kJavaScriptCallCodeStartRegister, Operand(kScratchReg));
539 }
540
541 // Check if the code object is marked for deoptimization. If it is, then it
542 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
543 // to:
544 // 1. read from memory the word that contains that bit, which can be found in
545 // the flags in the referenced {CodeDataContainer} object;
546 // 2. test kMarkedForDeoptimizationBit in those flags; and
547 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()548 void CodeGenerator::BailoutIfDeoptimized() {
549 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
550 __ LoadTaggedPointerField(
551 kScratchReg, MemOperand(kJavaScriptCallCodeStartRegister, offset));
552 __ Lw(kScratchReg,
553 FieldMemOperand(kScratchReg,
554 CodeDataContainer::kKindSpecificFlagsOffset));
555 __ And(kScratchReg, kScratchReg,
556 Operand(1 << Code::kMarkedForDeoptimizationBit));
557 __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
558 RelocInfo::CODE_TARGET, ne, kScratchReg, Operand(zero_reg));
559 }
560
561 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)562 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
563 Instruction* instr) {
564 RiscvOperandConverter i(this, instr);
565 InstructionCode opcode = instr->opcode();
566 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
567 switch (arch_opcode) {
568 case kArchCallCodeObject: {
569 if (instr->InputAt(0)->IsImmediate()) {
570 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
571 } else {
572 Register reg = i.InputRegister(0);
573 DCHECK_IMPLIES(
574 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
575 reg == kJavaScriptCallCodeStartRegister);
576 __ CallCodeObject(reg);
577 }
578 RecordCallPosition(instr);
579 frame_access_state()->ClearSPDelta();
580 break;
581 }
582 case kArchCallBuiltinPointer: {
583 DCHECK(!instr->InputAt(0)->IsImmediate());
584 Register builtin_index = i.InputRegister(0);
585 __ CallBuiltinByIndex(builtin_index);
586 RecordCallPosition(instr);
587 frame_access_state()->ClearSPDelta();
588 break;
589 }
590 case kArchCallWasmFunction: {
591 if (instr->InputAt(0)->IsImmediate()) {
592 Constant constant = i.ToConstant(instr->InputAt(0));
593 Address wasm_code = static_cast<Address>(constant.ToInt64());
594 __ Call(wasm_code, constant.rmode());
595 } else {
596 __ Add64(t6, i.InputOrZeroRegister(0), 0);
597 __ Call(t6);
598 }
599 RecordCallPosition(instr);
600 frame_access_state()->ClearSPDelta();
601 break;
602 }
603 case kArchTailCallCodeObject: {
604 if (instr->InputAt(0)->IsImmediate()) {
605 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
606 } else {
607 Register reg = i.InputOrZeroRegister(0);
608 DCHECK_IMPLIES(
609 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
610 reg == kJavaScriptCallCodeStartRegister);
611 __ JumpCodeObject(reg);
612 }
613 frame_access_state()->ClearSPDelta();
614 frame_access_state()->SetFrameAccessToDefault();
615 break;
616 }
617 case kArchTailCallWasm: {
618 if (instr->InputAt(0)->IsImmediate()) {
619 Constant constant = i.ToConstant(instr->InputAt(0));
620 Address wasm_code = static_cast<Address>(constant.ToInt64());
621 __ Jump(wasm_code, constant.rmode());
622 } else {
623 __ Add64(kScratchReg, i.InputOrZeroRegister(0), 0);
624 __ Jump(kScratchReg);
625 }
626 frame_access_state()->ClearSPDelta();
627 frame_access_state()->SetFrameAccessToDefault();
628 break;
629 }
630 case kArchTailCallAddress: {
631 CHECK(!instr->InputAt(0)->IsImmediate());
632 Register reg = i.InputOrZeroRegister(0);
633 DCHECK_IMPLIES(
634 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
635 reg == kJavaScriptCallCodeStartRegister);
636 __ Jump(reg);
637 frame_access_state()->ClearSPDelta();
638 frame_access_state()->SetFrameAccessToDefault();
639 break;
640 }
641 case kArchCallJSFunction: {
642 Register func = i.InputOrZeroRegister(0);
643 if (FLAG_debug_code) {
644 // Check the function's context matches the context argument.
645 __ LoadTaggedPointerField(
646 kScratchReg, FieldMemOperand(func, JSFunction::kContextOffset));
647 __ Assert(eq, AbortReason::kWrongFunctionContext, cp,
648 Operand(kScratchReg));
649 }
650 static_assert(kJavaScriptCallCodeStartRegister == a2, "ABI mismatch");
651 __ LoadTaggedPointerField(a2,
652 FieldMemOperand(func, JSFunction::kCodeOffset));
653 __ CallCodeObject(a2);
654 RecordCallPosition(instr);
655 frame_access_state()->ClearSPDelta();
656 break;
657 }
658 case kArchPrepareCallCFunction: {
659 int const num_parameters = MiscField::decode(instr->opcode());
660 __ PrepareCallCFunction(num_parameters, kScratchReg);
661 // Frame alignment requires using FP-relative frame addressing.
662 frame_access_state()->SetFrameAccessToFP();
663 break;
664 }
665 case kArchSaveCallerRegisters: {
666 fp_mode_ =
667 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
668 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
669 fp_mode_ == SaveFPRegsMode::kSave);
670 // kReturnRegister0 should have been saved before entering the stub.
671 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
672 DCHECK(IsAligned(bytes, kSystemPointerSize));
673 DCHECK_EQ(0, frame_access_state()->sp_delta());
674 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
675 DCHECK(!caller_registers_saved_);
676 caller_registers_saved_ = true;
677 break;
678 }
679 case kArchRestoreCallerRegisters: {
680 DCHECK(fp_mode_ ==
681 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
682 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
683 fp_mode_ == SaveFPRegsMode::kSave);
684 // Don't overwrite the returned value.
685 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
686 frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
687 DCHECK_EQ(0, frame_access_state()->sp_delta());
688 DCHECK(caller_registers_saved_);
689 caller_registers_saved_ = false;
690 break;
691 }
692 case kArchPrepareTailCall:
693 AssemblePrepareTailCall();
694 break;
695 case kArchCallCFunction: {
696 int const num_parameters = MiscField::decode(instr->opcode());
697 Label after_call;
698 bool isWasmCapiFunction =
699 linkage()->GetIncomingDescriptor()->IsWasmCapiFunction();
700 if (isWasmCapiFunction) {
701 // Put the return address in a stack slot.
702 __ LoadAddress(kScratchReg, &after_call, RelocInfo::EXTERNAL_REFERENCE);
703 __ Sd(kScratchReg,
704 MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
705 }
706 if (instr->InputAt(0)->IsImmediate()) {
707 ExternalReference ref = i.InputExternalReference(0);
708 __ CallCFunction(ref, num_parameters);
709 } else {
710 Register func = i.InputOrZeroRegister(0);
711 __ CallCFunction(func, num_parameters);
712 }
713 __ bind(&after_call);
714 if (isWasmCapiFunction) {
715 RecordSafepoint(instr->reference_map());
716 }
717
718 frame_access_state()->SetFrameAccessToDefault();
719 // Ideally, we should decrement SP delta to match the change of stack
720 // pointer in CallCFunction. However, for certain architectures (e.g.
721 // ARM), there may be more strict alignment requirement, causing old SP
722 // to be saved on the stack. In those cases, we can not calculate the SP
723 // delta statically.
724 frame_access_state()->ClearSPDelta();
725 if (caller_registers_saved_) {
726 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
727 // Here, we assume the sequence to be:
728 // kArchSaveCallerRegisters;
729 // kArchCallCFunction;
730 // kArchRestoreCallerRegisters;
731 int bytes =
732 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
733 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
734 }
735 break;
736 }
737 case kArchJmp:
738 AssembleArchJump(i.InputRpo(0));
739 break;
740 case kArchBinarySearchSwitch:
741 AssembleArchBinarySearchSwitch(instr);
742 break;
743 case kArchTableSwitch:
744 AssembleArchTableSwitch(instr);
745 break;
746 case kArchAbortCSADcheck:
747 DCHECK(i.InputRegister(0) == a0);
748 {
749 // We don't actually want to generate a pile of code for this, so just
750 // claim there is a stack frame, without generating one.
751 FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
752 __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck),
753 RelocInfo::CODE_TARGET);
754 }
755 __ stop();
756 break;
757 case kArchDebugBreak:
758 __ DebugBreak();
759 break;
760 case kArchComment:
761 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
762 break;
763 case kArchNop:
764 case kArchThrowTerminator:
765 // don't emit code for nops.
766 break;
767 case kArchDeoptimize: {
768 DeoptimizationExit* exit =
769 BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
770 __ Branch(exit->label());
771 break;
772 }
773 case kArchRet:
774 AssembleReturn(instr->InputAt(0));
775 break;
776 case kArchStackPointerGreaterThan:
777 // Pseudo-instruction used for cmp/branch. No opcode emitted here.
778 break;
779 case kArchStackCheckOffset:
780 __ Move(i.OutputRegister(), Smi::FromInt(GetStackCheckOffset()));
781 break;
782 case kArchFramePointer:
783 __ Move(i.OutputRegister(), fp);
784 break;
785 case kArchParentFramePointer:
786 if (frame_access_state()->has_frame()) {
787 __ Ld(i.OutputRegister(), MemOperand(fp, 0));
788 } else {
789 __ Move(i.OutputRegister(), fp);
790 }
791 break;
792 case kArchTruncateDoubleToI:
793 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
794 i.InputDoubleRegister(0), DetermineStubCallMode());
795 break;
796 case kArchStoreWithWriteBarrier: {
797 RecordWriteMode mode =
798 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
799 Register object = i.InputRegister(0);
800 Register index = i.InputRegister(1);
801 Register value = i.InputRegister(2);
802 Register scratch0 = i.TempRegister(0);
803 Register scratch1 = i.TempRegister(1);
804 auto ool = zone()->New<OutOfLineRecordWrite>(this, object, index, value,
805 scratch0, scratch1, mode,
806 DetermineStubCallMode());
807 __ Add64(kScratchReg, object, index);
808 __ StoreTaggedField(value, MemOperand(kScratchReg));
809 if (mode > RecordWriteMode::kValueIsPointer) {
810 __ JumpIfSmi(value, ool->exit());
811 }
812 __ CheckPageFlag(object, scratch0,
813 MemoryChunk::kPointersFromHereAreInterestingMask, ne,
814 ool->entry());
815 __ bind(ool->exit());
816 break;
817 }
818 case kArchStackSlot: {
819 FrameOffset offset =
820 frame_access_state()->GetFrameOffset(i.InputInt32(0));
821 Register base_reg = offset.from_stack_pointer() ? sp : fp;
822 __ Add64(i.OutputRegister(), base_reg, Operand(offset.offset()));
823 int alignment = i.InputInt32(1);
824 DCHECK(alignment == 0 || alignment == 4 || alignment == 8 ||
825 alignment == 16);
826 if (FLAG_debug_code && alignment > 0) {
827 // Verify that the output_register is properly aligned
828 __ And(kScratchReg, i.OutputRegister(),
829 Operand(kSystemPointerSize - 1));
830 __ Assert(eq, AbortReason::kAllocationIsNotDoubleAligned, kScratchReg,
831 Operand(zero_reg));
832 }
833 if (alignment == 2 * kSystemPointerSize) {
834 Label done;
835 __ Add64(kScratchReg, base_reg, Operand(offset.offset()));
836 __ And(kScratchReg, kScratchReg, Operand(alignment - 1));
837 __ BranchShort(&done, eq, kScratchReg, Operand(zero_reg));
838 __ Add64(i.OutputRegister(), i.OutputRegister(), kSystemPointerSize);
839 __ bind(&done);
840 } else if (alignment > 2 * kSystemPointerSize) {
841 Label done;
842 __ Add64(kScratchReg, base_reg, Operand(offset.offset()));
843 __ And(kScratchReg, kScratchReg, Operand(alignment - 1));
844 __ BranchShort(&done, eq, kScratchReg, Operand(zero_reg));
845 __ li(kScratchReg2, alignment);
846 __ Sub64(kScratchReg2, kScratchReg2, Operand(kScratchReg));
847 __ Add64(i.OutputRegister(), i.OutputRegister(), kScratchReg2);
848 __ bind(&done);
849 }
850
851 break;
852 }
853 case kIeee754Float64Acos:
854 ASSEMBLE_IEEE754_UNOP(acos);
855 break;
856 case kIeee754Float64Acosh:
857 ASSEMBLE_IEEE754_UNOP(acosh);
858 break;
859 case kIeee754Float64Asin:
860 ASSEMBLE_IEEE754_UNOP(asin);
861 break;
862 case kIeee754Float64Asinh:
863 ASSEMBLE_IEEE754_UNOP(asinh);
864 break;
865 case kIeee754Float64Atan:
866 ASSEMBLE_IEEE754_UNOP(atan);
867 break;
868 case kIeee754Float64Atanh:
869 ASSEMBLE_IEEE754_UNOP(atanh);
870 break;
871 case kIeee754Float64Atan2:
872 ASSEMBLE_IEEE754_BINOP(atan2);
873 break;
874 case kIeee754Float64Cos:
875 ASSEMBLE_IEEE754_UNOP(cos);
876 break;
877 case kIeee754Float64Cosh:
878 ASSEMBLE_IEEE754_UNOP(cosh);
879 break;
880 case kIeee754Float64Cbrt:
881 ASSEMBLE_IEEE754_UNOP(cbrt);
882 break;
883 case kIeee754Float64Exp:
884 ASSEMBLE_IEEE754_UNOP(exp);
885 break;
886 case kIeee754Float64Expm1:
887 ASSEMBLE_IEEE754_UNOP(expm1);
888 break;
889 case kIeee754Float64Log:
890 ASSEMBLE_IEEE754_UNOP(log);
891 break;
892 case kIeee754Float64Log1p:
893 ASSEMBLE_IEEE754_UNOP(log1p);
894 break;
895 case kIeee754Float64Log2:
896 ASSEMBLE_IEEE754_UNOP(log2);
897 break;
898 case kIeee754Float64Log10:
899 ASSEMBLE_IEEE754_UNOP(log10);
900 break;
901 case kIeee754Float64Pow:
902 ASSEMBLE_IEEE754_BINOP(pow);
903 break;
904 case kIeee754Float64Sin:
905 ASSEMBLE_IEEE754_UNOP(sin);
906 break;
907 case kIeee754Float64Sinh:
908 ASSEMBLE_IEEE754_UNOP(sinh);
909 break;
910 case kIeee754Float64Tan:
911 ASSEMBLE_IEEE754_UNOP(tan);
912 break;
913 case kIeee754Float64Tanh:
914 ASSEMBLE_IEEE754_UNOP(tanh);
915 break;
916 case kRiscvAdd32:
917 __ Add32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
918 break;
919 case kRiscvAdd64:
920 __ Add64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
921 break;
922 case kRiscvAddOvf64:
923 __ AddOverflow64(i.OutputRegister(), i.InputOrZeroRegister(0),
924 i.InputOperand(1), kScratchReg);
925 break;
926 case kRiscvSub32:
927 __ Sub32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
928 break;
929 case kRiscvSub64:
930 __ Sub64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
931 break;
932 case kRiscvSubOvf64:
933 __ SubOverflow64(i.OutputRegister(), i.InputOrZeroRegister(0),
934 i.InputOperand(1), kScratchReg);
935 break;
936 case kRiscvMul32:
937 __ Mul32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
938 break;
939 case kRiscvMulOvf32:
940 __ MulOverflow32(i.OutputRegister(), i.InputOrZeroRegister(0),
941 i.InputOperand(1), kScratchReg);
942 break;
943 case kRiscvMulHigh32:
944 __ Mulh32(i.OutputRegister(), i.InputOrZeroRegister(0),
945 i.InputOperand(1));
946 break;
947 case kRiscvMulHighU32:
948 __ Mulhu32(i.OutputRegister(), i.InputOrZeroRegister(0),
949 i.InputOperand(1), kScratchReg, kScratchReg2);
950 break;
951 case kRiscvMulHigh64:
952 __ Mulh64(i.OutputRegister(), i.InputOrZeroRegister(0),
953 i.InputOperand(1));
954 break;
955 case kRiscvDiv32: {
956 __ Div32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
957 // Set ouput to zero if divisor == 0
958 __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
959 break;
960 }
961 case kRiscvDivU32: {
962 __ Divu32(i.OutputRegister(), i.InputOrZeroRegister(0),
963 i.InputOperand(1));
964 // Set ouput to zero if divisor == 0
965 __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
966 break;
967 }
968 case kRiscvMod32:
969 __ Mod32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
970 break;
971 case kRiscvModU32:
972 __ Modu32(i.OutputRegister(), i.InputOrZeroRegister(0),
973 i.InputOperand(1));
974 break;
975 case kRiscvMul64:
976 __ Mul64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
977 break;
978 case kRiscvDiv64: {
979 __ Div64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
980 // Set ouput to zero if divisor == 0
981 __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
982 break;
983 }
984 case kRiscvDivU64: {
985 __ Divu64(i.OutputRegister(), i.InputOrZeroRegister(0),
986 i.InputOperand(1));
987 // Set ouput to zero if divisor == 0
988 __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
989 break;
990 }
991 case kRiscvMod64:
992 __ Mod64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
993 break;
994 case kRiscvModU64:
995 __ Modu64(i.OutputRegister(), i.InputOrZeroRegister(0),
996 i.InputOperand(1));
997 break;
998 case kRiscvAnd:
999 __ And(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1000 break;
1001 case kRiscvAnd32:
1002 __ And(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1003 __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1004 break;
1005 case kRiscvOr:
1006 __ Or(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1007 break;
1008 case kRiscvOr32:
1009 __ Or(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1010 __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1011 break;
1012 case kRiscvNor:
1013 if (instr->InputAt(1)->IsRegister()) {
1014 __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1015 } else {
1016 DCHECK_EQ(0, i.InputOperand(1).immediate());
1017 __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), zero_reg);
1018 }
1019 break;
1020 case kRiscvNor32:
1021 if (instr->InputAt(1)->IsRegister()) {
1022 __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1023 __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1024 } else {
1025 DCHECK_EQ(0, i.InputOperand(1).immediate());
1026 __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), zero_reg);
1027 __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1028 }
1029 break;
1030 case kRiscvXor:
1031 __ Xor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1032 break;
1033 case kRiscvXor32:
1034 __ Xor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1035 __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1036 break;
1037 case kRiscvClz32:
1038 __ Clz32(i.OutputRegister(), i.InputOrZeroRegister(0));
1039 break;
1040 case kRiscvClz64:
1041 __ Clz64(i.OutputRegister(), i.InputOrZeroRegister(0));
1042 break;
1043 case kRiscvCtz32: {
1044 Register src = i.InputRegister(0);
1045 Register dst = i.OutputRegister();
1046 __ Ctz32(dst, src);
1047 } break;
1048 case kRiscvCtz64: {
1049 Register src = i.InputRegister(0);
1050 Register dst = i.OutputRegister();
1051 __ Ctz64(dst, src);
1052 } break;
1053 case kRiscvPopcnt32: {
1054 Register src = i.InputRegister(0);
1055 Register dst = i.OutputRegister();
1056 __ Popcnt32(dst, src, kScratchReg);
1057 } break;
1058 case kRiscvPopcnt64: {
1059 Register src = i.InputRegister(0);
1060 Register dst = i.OutputRegister();
1061 __ Popcnt64(dst, src, kScratchReg);
1062 } break;
1063 case kRiscvShl32:
1064 if (instr->InputAt(1)->IsRegister()) {
1065 __ Sll32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1066 } else {
1067 int64_t imm = i.InputOperand(1).immediate();
1068 __ Sll32(i.OutputRegister(), i.InputRegister(0),
1069 static_cast<uint16_t>(imm));
1070 }
1071 break;
1072 case kRiscvShr32:
1073 if (instr->InputAt(1)->IsRegister()) {
1074 __ Srl32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1075 } else {
1076 int64_t imm = i.InputOperand(1).immediate();
1077 __ Srl32(i.OutputRegister(), i.InputRegister(0),
1078 static_cast<uint16_t>(imm));
1079 }
1080 break;
1081 case kRiscvSar32:
1082 if (instr->InputAt(1)->IsRegister()) {
1083 __ Sra32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1084 } else {
1085 int64_t imm = i.InputOperand(1).immediate();
1086 __ Sra32(i.OutputRegister(), i.InputRegister(0),
1087 static_cast<uint16_t>(imm));
1088 }
1089 break;
1090 case kRiscvZeroExtendWord: {
1091 __ ZeroExtendWord(i.OutputRegister(), i.InputRegister(0));
1092 break;
1093 }
1094 case kRiscvSignExtendWord: {
1095 __ SignExtendWord(i.OutputRegister(), i.InputRegister(0));
1096 break;
1097 }
1098 case kRiscvShl64:
1099 __ Sll64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1100 break;
1101 case kRiscvShr64:
1102 __ Srl64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1103 break;
1104 case kRiscvSar64:
1105 __ Sra64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1106 break;
1107 case kRiscvRor32:
1108 __ Ror(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1109 break;
1110 case kRiscvRor64:
1111 __ Dror(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1112 break;
1113 case kRiscvTst:
1114 __ And(kScratchReg, i.InputRegister(0), i.InputOperand(1));
1115 // Pseudo-instruction used for cmp/branch. No opcode emitted here.
1116 break;
1117 case kRiscvCmp:
1118 // Pseudo-instruction used for cmp/branch. No opcode emitted here.
1119 break;
1120 case kRiscvCmpZero:
1121 // Pseudo-instruction used for cmpzero/branch. No opcode emitted here.
1122 break;
1123 case kRiscvMov:
1124 // TODO(plind): Should we combine mov/li like this, or use separate instr?
1125 // - Also see x64 ASSEMBLE_BINOP & RegisterOrOperandType
1126 if (HasRegisterInput(instr, 0)) {
1127 __ Move(i.OutputRegister(), i.InputRegister(0));
1128 } else {
1129 __ li(i.OutputRegister(), i.InputOperand(0));
1130 }
1131 break;
1132
1133 case kRiscvCmpS: {
1134 FPURegister left = i.InputOrZeroSingleRegister(0);
1135 FPURegister right = i.InputOrZeroSingleRegister(1);
1136 bool predicate;
1137 FPUCondition cc =
1138 FlagsConditionToConditionCmpFPU(&predicate, instr->flags_condition());
1139
1140 if ((left == kDoubleRegZero || right == kDoubleRegZero) &&
1141 !__ IsSingleZeroRegSet()) {
1142 __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1143 }
1144 // compare result set to kScratchReg
1145 __ CompareF32(kScratchReg, cc, left, right);
1146 } break;
1147 case kRiscvAddS:
1148 // TODO(plind): add special case: combine mult & add.
1149 __ fadd_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1150 i.InputDoubleRegister(1));
1151 break;
1152 case kRiscvSubS:
1153 __ fsub_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1154 i.InputDoubleRegister(1));
1155 break;
1156 case kRiscvMulS:
1157 // TODO(plind): add special case: right op is -1.0, see arm port.
1158 __ fmul_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1159 i.InputDoubleRegister(1));
1160 break;
1161 case kRiscvDivS:
1162 __ fdiv_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1163 i.InputDoubleRegister(1));
1164 break;
1165 case kRiscvModS: {
1166 // TODO(bmeurer): We should really get rid of this special instruction,
1167 // and generate a CallAddress instruction instead.
1168 FrameScope scope(tasm(), StackFrame::MANUAL);
1169 __ PrepareCallCFunction(0, 2, kScratchReg);
1170 __ MovToFloatParameters(i.InputDoubleRegister(0),
1171 i.InputDoubleRegister(1));
1172 // TODO(balazs.kilvady): implement mod_two_floats_operation(isolate())
1173 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1174 // Move the result in the double result register.
1175 __ MovFromFloatResult(i.OutputSingleRegister());
1176 break;
1177 }
1178 case kRiscvAbsS:
1179 __ fabs_s(i.OutputSingleRegister(), i.InputSingleRegister(0));
1180 break;
1181 case kRiscvNegS:
1182 __ Neg_s(i.OutputSingleRegister(), i.InputSingleRegister(0));
1183 break;
1184 case kRiscvSqrtS: {
1185 __ fsqrt_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1186 break;
1187 }
1188 case kRiscvMaxS:
1189 __ fmax_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1190 i.InputDoubleRegister(1));
1191 break;
1192 case kRiscvMinS:
1193 __ fmin_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1194 i.InputDoubleRegister(1));
1195 break;
1196 case kRiscvCmpD: {
1197 FPURegister left = i.InputOrZeroDoubleRegister(0);
1198 FPURegister right = i.InputOrZeroDoubleRegister(1);
1199 bool predicate;
1200 FPUCondition cc =
1201 FlagsConditionToConditionCmpFPU(&predicate, instr->flags_condition());
1202 if ((left == kDoubleRegZero || right == kDoubleRegZero) &&
1203 !__ IsDoubleZeroRegSet()) {
1204 __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1205 }
1206 // compare result set to kScratchReg
1207 __ CompareF64(kScratchReg, cc, left, right);
1208 } break;
1209 case kRiscvAddD:
1210 // TODO(plind): add special case: combine mult & add.
1211 __ fadd_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1212 i.InputDoubleRegister(1));
1213 break;
1214 case kRiscvSubD:
1215 __ fsub_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1216 i.InputDoubleRegister(1));
1217 break;
1218 case kRiscvMulD:
1219 // TODO(plind): add special case: right op is -1.0, see arm port.
1220 __ fmul_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1221 i.InputDoubleRegister(1));
1222 break;
1223 case kRiscvDivD:
1224 __ fdiv_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1225 i.InputDoubleRegister(1));
1226 break;
1227 case kRiscvModD: {
1228 // TODO(bmeurer): We should really get rid of this special instruction,
1229 // and generate a CallAddress instruction instead.
1230 FrameScope scope(tasm(), StackFrame::MANUAL);
1231 __ PrepareCallCFunction(0, 2, kScratchReg);
1232 __ MovToFloatParameters(i.InputDoubleRegister(0),
1233 i.InputDoubleRegister(1));
1234 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1235 // Move the result in the double result register.
1236 __ MovFromFloatResult(i.OutputDoubleRegister());
1237 break;
1238 }
1239 case kRiscvAbsD:
1240 __ fabs_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1241 break;
1242 case kRiscvNegD:
1243 __ Neg_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1244 break;
1245 case kRiscvSqrtD: {
1246 __ fsqrt_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1247 break;
1248 }
1249 case kRiscvMaxD:
1250 __ fmax_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1251 i.InputDoubleRegister(1));
1252 break;
1253 case kRiscvMinD:
1254 __ fmin_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1255 i.InputDoubleRegister(1));
1256 break;
1257 case kRiscvFloat64RoundDown: {
1258 __ Floor_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1259 kScratchDoubleReg);
1260 break;
1261 }
1262 case kRiscvFloat32RoundDown: {
1263 __ Floor_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1264 kScratchDoubleReg);
1265 break;
1266 }
1267 case kRiscvFloat64RoundTruncate: {
1268 __ Trunc_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1269 kScratchDoubleReg);
1270 break;
1271 }
1272 case kRiscvFloat32RoundTruncate: {
1273 __ Trunc_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1274 kScratchDoubleReg);
1275 break;
1276 }
1277 case kRiscvFloat64RoundUp: {
1278 __ Ceil_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1279 kScratchDoubleReg);
1280 break;
1281 }
1282 case kRiscvFloat32RoundUp: {
1283 __ Ceil_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1284 kScratchDoubleReg);
1285 break;
1286 }
1287 case kRiscvFloat64RoundTiesEven: {
1288 __ Round_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1289 kScratchDoubleReg);
1290 break;
1291 }
1292 case kRiscvFloat32RoundTiesEven: {
1293 __ Round_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1294 kScratchDoubleReg);
1295 break;
1296 }
1297 case kRiscvFloat32Max: {
1298 __ Float32Max(i.OutputSingleRegister(), i.InputSingleRegister(0),
1299 i.InputSingleRegister(1));
1300 break;
1301 }
1302 case kRiscvFloat64Max: {
1303 __ Float64Max(i.OutputSingleRegister(), i.InputSingleRegister(0),
1304 i.InputSingleRegister(1));
1305 break;
1306 }
1307 case kRiscvFloat32Min: {
1308 __ Float32Min(i.OutputSingleRegister(), i.InputSingleRegister(0),
1309 i.InputSingleRegister(1));
1310 break;
1311 }
1312 case kRiscvFloat64Min: {
1313 __ Float64Min(i.OutputSingleRegister(), i.InputSingleRegister(0),
1314 i.InputSingleRegister(1));
1315 break;
1316 }
1317 case kRiscvFloat64SilenceNaN:
1318 __ FPUCanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1319 break;
1320 case kRiscvCvtSD:
1321 __ fcvt_s_d(i.OutputSingleRegister(), i.InputDoubleRegister(0));
1322 break;
1323 case kRiscvCvtDS:
1324 __ fcvt_d_s(i.OutputDoubleRegister(), i.InputSingleRegister(0));
1325 break;
1326 case kRiscvCvtDW: {
1327 __ fcvt_d_w(i.OutputDoubleRegister(), i.InputRegister(0));
1328 break;
1329 }
1330 case kRiscvCvtSW: {
1331 __ fcvt_s_w(i.OutputDoubleRegister(), i.InputRegister(0));
1332 break;
1333 }
1334 case kRiscvCvtSUw: {
1335 __ Cvt_s_uw(i.OutputDoubleRegister(), i.InputRegister(0));
1336 break;
1337 }
1338 case kRiscvCvtSL: {
1339 __ fcvt_s_l(i.OutputDoubleRegister(), i.InputRegister(0));
1340 break;
1341 }
1342 case kRiscvCvtDL: {
1343 __ fcvt_d_l(i.OutputDoubleRegister(), i.InputRegister(0));
1344 break;
1345 }
1346 case kRiscvCvtDUw: {
1347 __ Cvt_d_uw(i.OutputDoubleRegister(), i.InputRegister(0));
1348 break;
1349 }
1350 case kRiscvCvtDUl: {
1351 __ Cvt_d_ul(i.OutputDoubleRegister(), i.InputRegister(0));
1352 break;
1353 }
1354 case kRiscvCvtSUl: {
1355 __ Cvt_s_ul(i.OutputDoubleRegister(), i.InputRegister(0));
1356 break;
1357 }
1358 case kRiscvFloorWD: {
1359 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1360 __ Floor_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1361 break;
1362 }
1363 case kRiscvCeilWD: {
1364 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1365 __ Ceil_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1366 break;
1367 }
1368 case kRiscvRoundWD: {
1369 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1370 __ Round_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1371 break;
1372 }
1373 case kRiscvTruncWD: {
1374 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1375 __ Trunc_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1376 break;
1377 }
1378 case kRiscvFloorWS: {
1379 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1380 __ Floor_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1381 break;
1382 }
1383 case kRiscvCeilWS: {
1384 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1385 __ Ceil_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1386 break;
1387 }
1388 case kRiscvRoundWS: {
1389 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1390 __ Round_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1391 break;
1392 }
1393 case kRiscvTruncWS: {
1394 Label done;
1395 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1396 bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
1397 __ Trunc_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1398
1399 // On RISCV, if the input value exceeds INT32_MAX, the result of fcvt
1400 // is INT32_MAX. Note that, since INT32_MAX means the lower 31-bits are
1401 // all 1s, INT32_MAX cannot be represented precisely as a float, so an
1402 // fcvt result of INT32_MAX always indicate overflow.
1403 //
1404 // In wasm_compiler, to detect overflow in converting a FP value, fval, to
1405 // integer, V8 checks whether I2F(F2I(fval)) equals fval. However, if fval
1406 // == INT32_MAX+1, the value of I2F(F2I(fval)) happens to be fval. So,
1407 // INT32_MAX is not a good value to indicate overflow. Instead, we will
1408 // use INT32_MIN as the converted result of an out-of-range FP value,
1409 // exploiting the fact that INT32_MAX+1 is INT32_MIN.
1410 //
1411 // If the result of conversion overflow, the result will be set to
1412 // INT32_MIN. Here we detect overflow by testing whether output + 1 <
1413 // output (i.e., kScratchReg < output)
1414 if (set_overflow_to_min_i32) {
1415 __ Add32(kScratchReg, i.OutputRegister(), 1);
1416 __ BranchShort(&done, lt, i.OutputRegister(), Operand(kScratchReg));
1417 __ Move(i.OutputRegister(), kScratchReg);
1418 __ bind(&done);
1419 }
1420 break;
1421 }
1422 case kRiscvTruncLS: {
1423 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1424 __ Trunc_l_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1425 break;
1426 }
1427 case kRiscvTruncLD: {
1428 Label done;
1429 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1430 bool set_overflow_to_min_i64 = MiscField::decode(instr->opcode());
1431 __ Trunc_l_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1432 if (set_overflow_to_min_i64) {
1433 __ Add64(kScratchReg, i.OutputRegister(), 1);
1434 __ BranchShort(&done, lt, i.OutputRegister(), Operand(kScratchReg));
1435 __ Move(i.OutputRegister(), kScratchReg);
1436 __ bind(&done);
1437 }
1438 break;
1439 }
1440 case kRiscvTruncUwD: {
1441 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1442 __ Trunc_uw_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1443 break;
1444 }
1445 case kRiscvTruncUwS: {
1446 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1447 bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
1448 __ Trunc_uw_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1449
1450 // On RISCV, if the input value exceeds UINT32_MAX, the result of fcvt
1451 // is UINT32_MAX. Note that, since UINT32_MAX means all 32-bits are 1s,
1452 // UINT32_MAX cannot be represented precisely as float, so an fcvt result
1453 // of UINT32_MAX always indicates overflow.
1454 //
1455 // In wasm_compiler.cc, to detect overflow in converting a FP value, fval,
1456 // to integer, V8 checks whether I2F(F2I(fval)) equals fval. However, if
1457 // fval == UINT32_MAX+1, the value of I2F(F2I(fval)) happens to be fval.
1458 // So, UINT32_MAX is not a good value to indicate overflow. Instead, we
1459 // will use 0 as the converted result of an out-of-range FP value,
1460 // exploiting the fact that UINT32_MAX+1 is 0.
1461 if (set_overflow_to_min_u32) {
1462 __ Add32(kScratchReg, i.OutputRegister(), 1);
1463 // Set ouput to zero if result overflows (i.e., UINT32_MAX)
1464 __ LoadZeroIfConditionZero(i.OutputRegister(), kScratchReg);
1465 }
1466 break;
1467 }
1468 case kRiscvTruncUlS: {
1469 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1470 __ Trunc_ul_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1471 break;
1472 }
1473 case kRiscvTruncUlD: {
1474 Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1475 __ Trunc_ul_d(i.OutputRegister(0), i.InputDoubleRegister(0), result);
1476 break;
1477 }
1478 case kRiscvBitcastDL:
1479 __ fmv_x_d(i.OutputRegister(), i.InputDoubleRegister(0));
1480 break;
1481 case kRiscvBitcastLD:
1482 __ fmv_d_x(i.OutputDoubleRegister(), i.InputRegister(0));
1483 break;
1484 case kRiscvBitcastInt32ToFloat32:
1485 __ fmv_w_x(i.OutputDoubleRegister(), i.InputRegister(0));
1486 break;
1487 case kRiscvBitcastFloat32ToInt32:
1488 __ fmv_x_w(i.OutputRegister(), i.InputDoubleRegister(0));
1489 break;
1490 case kRiscvFloat64ExtractLowWord32:
1491 __ ExtractLowWordFromF64(i.OutputRegister(), i.InputDoubleRegister(0));
1492 break;
1493 case kRiscvFloat64ExtractHighWord32:
1494 __ ExtractHighWordFromF64(i.OutputRegister(), i.InputDoubleRegister(0));
1495 break;
1496 case kRiscvFloat64InsertLowWord32:
1497 __ InsertLowWordF64(i.OutputDoubleRegister(), i.InputRegister(1));
1498 break;
1499 case kRiscvFloat64InsertHighWord32:
1500 __ InsertHighWordF64(i.OutputDoubleRegister(), i.InputRegister(1));
1501 break;
1502 // ... more basic instructions ...
1503
1504 case kRiscvSignExtendByte:
1505 __ SignExtendByte(i.OutputRegister(), i.InputRegister(0));
1506 break;
1507 case kRiscvSignExtendShort:
1508 __ SignExtendShort(i.OutputRegister(), i.InputRegister(0));
1509 break;
1510 case kRiscvLbu:
1511 __ Lbu(i.OutputRegister(), i.MemoryOperand());
1512 break;
1513 case kRiscvLb:
1514 __ Lb(i.OutputRegister(), i.MemoryOperand());
1515 break;
1516 case kRiscvSb:
1517 __ Sb(i.InputOrZeroRegister(2), i.MemoryOperand());
1518 break;
1519 case kRiscvLhu:
1520 __ Lhu(i.OutputRegister(), i.MemoryOperand());
1521 break;
1522 case kRiscvUlhu:
1523 __ Ulhu(i.OutputRegister(), i.MemoryOperand());
1524 break;
1525 case kRiscvLh:
1526 __ Lh(i.OutputRegister(), i.MemoryOperand());
1527 break;
1528 case kRiscvUlh:
1529 __ Ulh(i.OutputRegister(), i.MemoryOperand());
1530 break;
1531 case kRiscvSh:
1532 __ Sh(i.InputOrZeroRegister(2), i.MemoryOperand());
1533 break;
1534 case kRiscvUsh:
1535 __ Ush(i.InputOrZeroRegister(2), i.MemoryOperand());
1536 break;
1537 case kRiscvLw:
1538 __ Lw(i.OutputRegister(), i.MemoryOperand());
1539 break;
1540 case kRiscvUlw:
1541 __ Ulw(i.OutputRegister(), i.MemoryOperand());
1542 break;
1543 case kRiscvLwu:
1544 __ Lwu(i.OutputRegister(), i.MemoryOperand());
1545 break;
1546 case kRiscvUlwu:
1547 __ Ulwu(i.OutputRegister(), i.MemoryOperand());
1548 break;
1549 case kRiscvLd:
1550 __ Ld(i.OutputRegister(), i.MemoryOperand());
1551 break;
1552 case kRiscvUld:
1553 __ Uld(i.OutputRegister(), i.MemoryOperand());
1554 break;
1555 case kRiscvSw:
1556 __ Sw(i.InputOrZeroRegister(2), i.MemoryOperand());
1557 break;
1558 case kRiscvUsw:
1559 __ Usw(i.InputOrZeroRegister(2), i.MemoryOperand());
1560 break;
1561 case kRiscvSd:
1562 __ Sd(i.InputOrZeroRegister(2), i.MemoryOperand());
1563 break;
1564 case kRiscvUsd:
1565 __ Usd(i.InputOrZeroRegister(2), i.MemoryOperand());
1566 break;
1567 case kRiscvLoadFloat: {
1568 __ LoadFloat(i.OutputSingleRegister(), i.MemoryOperand());
1569 break;
1570 }
1571 case kRiscvULoadFloat: {
1572 __ ULoadFloat(i.OutputSingleRegister(), i.MemoryOperand(), kScratchReg);
1573 break;
1574 }
1575 case kRiscvStoreFloat: {
1576 size_t index = 0;
1577 MemOperand operand = i.MemoryOperand(&index);
1578 FPURegister ft = i.InputOrZeroSingleRegister(index);
1579 if (ft == kDoubleRegZero && !__ IsSingleZeroRegSet()) {
1580 __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1581 }
1582 __ StoreFloat(ft, operand);
1583 break;
1584 }
1585 case kRiscvUStoreFloat: {
1586 size_t index = 0;
1587 MemOperand operand = i.MemoryOperand(&index);
1588 FPURegister ft = i.InputOrZeroSingleRegister(index);
1589 if (ft == kDoubleRegZero && !__ IsSingleZeroRegSet()) {
1590 __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1591 }
1592 __ UStoreFloat(ft, operand, kScratchReg);
1593 break;
1594 }
1595 case kRiscvLoadDouble:
1596 __ LoadDouble(i.OutputDoubleRegister(), i.MemoryOperand());
1597 break;
1598 case kRiscvULoadDouble:
1599 __ ULoadDouble(i.OutputDoubleRegister(), i.MemoryOperand(), kScratchReg);
1600 break;
1601 case kRiscvStoreDouble: {
1602 FPURegister ft = i.InputOrZeroDoubleRegister(2);
1603 if (ft == kDoubleRegZero && !__ IsDoubleZeroRegSet()) {
1604 __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1605 }
1606 __ StoreDouble(ft, i.MemoryOperand());
1607 break;
1608 }
1609 case kRiscvUStoreDouble: {
1610 FPURegister ft = i.InputOrZeroDoubleRegister(2);
1611 if (ft == kDoubleRegZero && !__ IsDoubleZeroRegSet()) {
1612 __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1613 }
1614 __ UStoreDouble(ft, i.MemoryOperand(), kScratchReg);
1615 break;
1616 }
1617 case kRiscvSync: {
1618 __ sync();
1619 break;
1620 }
1621 case kRiscvPush:
1622 if (instr->InputAt(0)->IsFPRegister()) {
1623 __ StoreDouble(i.InputDoubleRegister(0), MemOperand(sp, -kDoubleSize));
1624 __ Sub32(sp, sp, Operand(kDoubleSize));
1625 frame_access_state()->IncreaseSPDelta(kDoubleSize / kSystemPointerSize);
1626 } else {
1627 __ Push(i.InputOrZeroRegister(0));
1628 frame_access_state()->IncreaseSPDelta(1);
1629 }
1630 break;
1631 case kRiscvPeek: {
1632 int reverse_slot = i.InputInt32(0);
1633 int offset =
1634 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1635 if (instr->OutputAt(0)->IsFPRegister()) {
1636 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1637 if (op->representation() == MachineRepresentation::kFloat64) {
1638 __ LoadDouble(i.OutputDoubleRegister(), MemOperand(fp, offset));
1639 } else {
1640 DCHECK_EQ(op->representation(), MachineRepresentation::kFloat32);
1641 __ LoadFloat(
1642 i.OutputSingleRegister(0),
1643 MemOperand(fp, offset + kLessSignificantWordInDoublewordOffset));
1644 }
1645 } else {
1646 __ Ld(i.OutputRegister(0), MemOperand(fp, offset));
1647 }
1648 break;
1649 }
1650 case kRiscvStackClaim: {
1651 __ Sub64(sp, sp, Operand(i.InputInt32(0)));
1652 frame_access_state()->IncreaseSPDelta(i.InputInt32(0) /
1653 kSystemPointerSize);
1654 break;
1655 }
1656 case kRiscvStoreToStackSlot: {
1657 if (instr->InputAt(0)->IsFPRegister()) {
1658 if (instr->InputAt(0)->IsSimd128Register()) {
1659 UNREACHABLE();
1660 } else {
1661 __ StoreDouble(i.InputDoubleRegister(0),
1662 MemOperand(sp, i.InputInt32(1)));
1663 }
1664 } else {
1665 __ Sd(i.InputOrZeroRegister(0), MemOperand(sp, i.InputInt32(1)));
1666 }
1667 break;
1668 }
1669 case kRiscvByteSwap64: {
1670 __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 8, kScratchReg);
1671 break;
1672 }
1673 case kRiscvByteSwap32: {
1674 __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 4, kScratchReg);
1675 break;
1676 }
1677 case kAtomicLoadInt8:
1678 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1679 ASSEMBLE_ATOMIC_LOAD_INTEGER(Lb);
1680 break;
1681 case kAtomicLoadUint8:
1682 ASSEMBLE_ATOMIC_LOAD_INTEGER(Lbu);
1683 break;
1684 case kAtomicLoadInt16:
1685 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1686 ASSEMBLE_ATOMIC_LOAD_INTEGER(Lh);
1687 break;
1688 case kAtomicLoadUint16:
1689 ASSEMBLE_ATOMIC_LOAD_INTEGER(Lhu);
1690 break;
1691 case kAtomicLoadWord32:
1692 ASSEMBLE_ATOMIC_LOAD_INTEGER(Lw);
1693 break;
1694 case kRiscvWord64AtomicLoadUint64:
1695 ASSEMBLE_ATOMIC_LOAD_INTEGER(Ld);
1696 break;
1697 case kAtomicStoreWord8:
1698 ASSEMBLE_ATOMIC_STORE_INTEGER(Sb);
1699 break;
1700 case kAtomicStoreWord16:
1701 ASSEMBLE_ATOMIC_STORE_INTEGER(Sh);
1702 break;
1703 case kAtomicStoreWord32:
1704 ASSEMBLE_ATOMIC_STORE_INTEGER(Sw);
1705 break;
1706 case kRiscvWord64AtomicStoreWord64:
1707 ASSEMBLE_ATOMIC_STORE_INTEGER(Sd);
1708 break;
1709 case kAtomicExchangeInt8:
1710 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1711 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32);
1712 break;
1713 case kAtomicExchangeUint8:
1714 switch (AtomicWidthField::decode(opcode)) {
1715 case AtomicWidth::kWord32:
1716 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32);
1717 break;
1718 case AtomicWidth::kWord64:
1719 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64);
1720 break;
1721 }
1722 break;
1723 case kAtomicExchangeInt16:
1724 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1725 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32);
1726 break;
1727 case kAtomicExchangeUint16:
1728 switch (AtomicWidthField::decode(opcode)) {
1729 case AtomicWidth::kWord32:
1730 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32);
1731 break;
1732 case AtomicWidth::kWord64:
1733 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64);
1734 break;
1735 }
1736 break;
1737 case kAtomicExchangeWord32:
1738 switch (AtomicWidthField::decode(opcode)) {
1739 case AtomicWidth::kWord32:
1740 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Ll, Sc);
1741 break;
1742 case AtomicWidth::kWord64:
1743 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64);
1744 break;
1745 }
1746 break;
1747 case kRiscvWord64AtomicExchangeUint64:
1748 ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Lld, Scd);
1749 break;
1750 case kAtomicCompareExchangeInt8:
1751 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1752 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32);
1753 break;
1754 case kAtomicCompareExchangeUint8:
1755 switch (AtomicWidthField::decode(opcode)) {
1756 case AtomicWidth::kWord32:
1757 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32);
1758 break;
1759 case AtomicWidth::kWord64:
1760 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64);
1761 break;
1762 }
1763 break;
1764 case kAtomicCompareExchangeInt16:
1765 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1766 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32);
1767 break;
1768 case kAtomicCompareExchangeUint16:
1769 switch (AtomicWidthField::decode(opcode)) {
1770 case AtomicWidth::kWord32:
1771 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32);
1772 break;
1773 case AtomicWidth::kWord64:
1774 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64);
1775 break;
1776 }
1777 break;
1778 case kAtomicCompareExchangeWord32:
1779 switch (AtomicWidthField::decode(opcode)) {
1780 case AtomicWidth::kWord32:
1781 __ Sll32(i.InputRegister(2), i.InputRegister(2), 0);
1782 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Ll, Sc);
1783 break;
1784 case AtomicWidth::kWord64:
1785 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64);
1786 break;
1787 }
1788 break;
1789 case kRiscvWord64AtomicCompareExchangeUint64:
1790 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Lld, Scd);
1791 break;
1792 #define ATOMIC_BINOP_CASE(op, inst32, inst64) \
1793 case kAtomic##op##Int8: \
1794 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \
1795 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 8, inst32, 32); \
1796 break; \
1797 case kAtomic##op##Uint8: \
1798 switch (AtomicWidthField::decode(opcode)) { \
1799 case AtomicWidth::kWord32: \
1800 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 8, inst32, 32); \
1801 break; \
1802 case AtomicWidth::kWord64: \
1803 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 8, inst64, 64); \
1804 break; \
1805 } \
1806 break; \
1807 case kAtomic##op##Int16: \
1808 DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \
1809 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 16, inst32, 32); \
1810 break; \
1811 case kAtomic##op##Uint16: \
1812 switch (AtomicWidthField::decode(opcode)) { \
1813 case AtomicWidth::kWord32: \
1814 ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 16, inst32, 32); \
1815 break; \
1816 case AtomicWidth::kWord64: \
1817 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 16, inst64, 64); \
1818 break; \
1819 } \
1820 break; \
1821 case kAtomic##op##Word32: \
1822 switch (AtomicWidthField::decode(opcode)) { \
1823 case AtomicWidth::kWord32: \
1824 ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \
1825 break; \
1826 case AtomicWidth::kWord64: \
1827 ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 32, inst64, 64); \
1828 break; \
1829 } \
1830 break; \
1831 case kRiscvWord64Atomic##op##Uint64: \
1832 ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \
1833 break;
1834 ATOMIC_BINOP_CASE(Add, Add32, Add64)
1835 ATOMIC_BINOP_CASE(Sub, Sub32, Sub64)
1836 ATOMIC_BINOP_CASE(And, And, And)
1837 ATOMIC_BINOP_CASE(Or, Or, Or)
1838 ATOMIC_BINOP_CASE(Xor, Xor, Xor)
1839 #undef ATOMIC_BINOP_CASE
1840 case kRiscvAssertEqual:
1841 __ Assert(eq, static_cast<AbortReason>(i.InputOperand(2).immediate()),
1842 i.InputRegister(0), Operand(i.InputRegister(1)));
1843 break;
1844 case kRiscvStoreCompressTagged: {
1845 size_t index = 0;
1846 MemOperand operand = i.MemoryOperand(&index);
1847 __ StoreTaggedField(i.InputOrZeroRegister(index), operand);
1848 break;
1849 }
1850 case kRiscvLoadDecompressTaggedSigned: {
1851 CHECK(instr->HasOutput());
1852 Register result = i.OutputRegister();
1853 MemOperand operand = i.MemoryOperand();
1854 __ DecompressTaggedSigned(result, operand);
1855 break;
1856 }
1857 case kRiscvLoadDecompressTaggedPointer: {
1858 CHECK(instr->HasOutput());
1859 Register result = i.OutputRegister();
1860 MemOperand operand = i.MemoryOperand();
1861 __ DecompressTaggedPointer(result, operand);
1862 break;
1863 }
1864 case kRiscvLoadDecompressAnyTagged: {
1865 CHECK(instr->HasOutput());
1866 Register result = i.OutputRegister();
1867 MemOperand operand = i.MemoryOperand();
1868 __ DecompressAnyTagged(result, operand);
1869 break;
1870 }
1871 case kRiscvRvvSt: {
1872 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1873 Register dst = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm()
1874 : kScratchReg;
1875 if (i.MemoryOperand().offset() != 0) {
1876 __ Add64(dst, i.MemoryOperand().rm(), i.MemoryOperand().offset());
1877 }
1878 __ vs(i.InputSimd128Register(2), dst, 0, VSew::E8);
1879 break;
1880 }
1881 case kRiscvRvvLd: {
1882 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1883 Register src = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm()
1884 : kScratchReg;
1885 if (i.MemoryOperand().offset() != 0) {
1886 __ Add64(src, i.MemoryOperand().rm(), i.MemoryOperand().offset());
1887 }
1888 __ vl(i.OutputSimd128Register(), src, 0, VSew::E8);
1889 break;
1890 }
1891 case kRiscvS128Const: {
1892 Simd128Register dst = i.OutputSimd128Register();
1893 uint8_t imm[16];
1894 *reinterpret_cast<uint64_t*>(imm) =
1895 make_uint64(i.InputUint32(1), i.InputUint32(0));
1896 *(reinterpret_cast<uint64_t*>(imm) + 1) =
1897 make_uint64(i.InputUint32(3), i.InputUint32(2));
1898 __ WasmRvvS128const(dst, imm);
1899 break;
1900 }
1901 case kRiscvI64x2Add: {
1902 (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1);
1903 __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1904 i.InputSimd128Register(1));
1905 break;
1906 }
1907 case kRiscvI32x4Add: {
1908 (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1);
1909 __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1910 i.InputSimd128Register(1));
1911 break;
1912 }
1913 case kRiscvI16x8Add: {
1914 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1915 __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1916 i.InputSimd128Register(1));
1917 break;
1918 }
1919 case kRiscvI16x8AddSatS: {
1920 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1921 __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1922 i.InputSimd128Register(1));
1923 break;
1924 }
1925 case kRiscvI16x8AddSatU: {
1926 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1927 __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1928 i.InputSimd128Register(1));
1929 break;
1930 }
1931 case kRiscvI8x16Add: {
1932 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1933 __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1934 i.InputSimd128Register(1));
1935 break;
1936 }
1937 case kRiscvI8x16AddSatS: {
1938 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1939 __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1940 i.InputSimd128Register(1));
1941 break;
1942 }
1943 case kRiscvI8x16AddSatU: {
1944 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1945 __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1946 i.InputSimd128Register(1));
1947 break;
1948 }
1949 case kRiscvI64x2Sub: {
1950 (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1);
1951 __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1952 i.InputSimd128Register(1));
1953 break;
1954 }
1955 case kRiscvI32x4Sub: {
1956 (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1);
1957 __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1958 i.InputSimd128Register(1));
1959 break;
1960 }
1961 case kRiscvI16x8Sub: {
1962 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1963 __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1964 i.InputSimd128Register(1));
1965 break;
1966 }
1967 case kRiscvI16x8SubSatS: {
1968 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1969 __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1970 i.InputSimd128Register(1));
1971 break;
1972 }
1973 case kRiscvI16x8SubSatU: {
1974 (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1975 __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1976 i.InputSimd128Register(1));
1977 break;
1978 }
1979 case kRiscvI8x16Sub: {
1980 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1981 __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1982 i.InputSimd128Register(1));
1983 break;
1984 }
1985 case kRiscvI8x16SubSatS: {
1986 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1987 __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1988 i.InputSimd128Register(1));
1989 break;
1990 }
1991 case kRiscvI8x16SubSatU: {
1992 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1993 __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1994 i.InputSimd128Register(1));
1995 break;
1996 }
1997 case kRiscvS128And: {
1998 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1999 __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2000 i.InputSimd128Register(1));
2001 break;
2002 }
2003 case kRiscvS128Or: {
2004 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2005 __ vor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2006 i.InputSimd128Register(1));
2007 break;
2008 }
2009 case kRiscvS128Xor: {
2010 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2011 __ vxor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2012 i.InputSimd128Register(1));
2013 break;
2014 }
2015 case kRiscvS128Not: {
2016 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2017 __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2018 break;
2019 }
2020 case kRiscvS128AndNot: {
2021 (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2022 __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(1));
2023 __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2024 i.OutputSimd128Register());
2025 break;
2026 }
2027 case kRiscvI32x4ExtractLane: {
2028 __ WasmRvvExtractLane(i.OutputRegister(), i.InputSimd128Register(0),
2029 i.InputInt8(1), E32, m1);
2030 break;
2031 }
2032 case kRiscvI8x16Splat: {
2033 (__ VU).set(kScratchReg, E8, m1);
2034 __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2035 break;
2036 }
2037 case kRiscvI16x8Splat: {
2038 (__ VU).set(kScratchReg, E16, m1);
2039 __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2040 break;
2041 }
2042 case kRiscvI32x4Splat: {
2043 (__ VU).set(kScratchReg, E32, m1);
2044 __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2045 break;
2046 }
2047 case kRiscvI64x2Splat: {
2048 (__ VU).set(kScratchReg, E64, m1);
2049 __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2050 break;
2051 }
2052 case kRiscvF32x4Splat: {
2053 (__ VU).set(kScratchReg, E32, m1);
2054 __ fmv_x_w(kScratchReg, i.InputSingleRegister(0));
2055 __ vmv_vx(i.OutputSimd128Register(), kScratchReg);
2056 break;
2057 }
2058 case kRiscvF64x2Splat: {
2059 (__ VU).set(kScratchReg, E64, m1);
2060 __ fmv_x_d(kScratchReg, i.InputDoubleRegister(0));
2061 __ vmv_vx(i.OutputSimd128Register(), kScratchReg);
2062 break;
2063 }
2064 case kRiscvI32x4Abs: {
2065 __ VU.set(kScratchReg, E32, m1);
2066 __ vmv_vx(kSimd128RegZero, zero_reg);
2067 __ vmv_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2068 __ vmslt_vv(v0, i.InputSimd128Register(0), kSimd128RegZero);
2069 __ vsub_vv(i.OutputSimd128Register(), kSimd128RegZero,
2070 i.InputSimd128Register(0), Mask);
2071 break;
2072 }
2073 case kRiscvI8x16Eq: {
2074 __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2075 i.InputSimd128Register(1), E8, m1);
2076 break;
2077 }
2078 case kRiscvI16x8Eq: {
2079 __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2080 i.InputSimd128Register(1), E16, m1);
2081 break;
2082 }
2083 case kRiscvI32x4Eq: {
2084 __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2085 i.InputSimd128Register(1), E32, m1);
2086 break;
2087 }
2088 case kRiscvI64x2Eq: {
2089 __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2090 i.InputSimd128Register(1), E64, m1);
2091 break;
2092 }
2093 case kRiscvI8x16Ne: {
2094 __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2095 i.InputSimd128Register(1), E8, m1);
2096 break;
2097 }
2098 case kRiscvI16x8Ne: {
2099 __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2100 i.InputSimd128Register(1), E16, m1);
2101 break;
2102 }
2103 case kRiscvI32x4Ne: {
2104 __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2105 i.InputSimd128Register(1), E32, m1);
2106 break;
2107 }
2108 case kRiscvI64x2Ne: {
2109 __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2110 i.InputSimd128Register(1), E64, m1);
2111 break;
2112 }
2113 case kRiscvI8x16GeS: {
2114 __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2115 i.InputSimd128Register(1), E8, m1);
2116 break;
2117 }
2118 case kRiscvI16x8GeS: {
2119 __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2120 i.InputSimd128Register(1), E16, m1);
2121 break;
2122 }
2123 case kRiscvI32x4GeS: {
2124 __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2125 i.InputSimd128Register(1), E32, m1);
2126 break;
2127 }
2128 case kRiscvI64x2GeS: {
2129 __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2130 i.InputSimd128Register(1), E64, m1);
2131 break;
2132 }
2133 case kRiscvI8x16GeU: {
2134 __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2135 i.InputSimd128Register(1), E8, m1);
2136 break;
2137 }
2138 case kRiscvI16x8GeU: {
2139 __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2140 i.InputSimd128Register(1), E16, m1);
2141 break;
2142 }
2143 case kRiscvI32x4GeU: {
2144 __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2145 i.InputSimd128Register(1), E32, m1);
2146 break;
2147 }
2148 case kRiscvI8x16GtS: {
2149 __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2150 i.InputSimd128Register(1), E8, m1);
2151 break;
2152 }
2153 case kRiscvI16x8GtS: {
2154 __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2155 i.InputSimd128Register(1), E16, m1);
2156 break;
2157 }
2158 case kRiscvI32x4GtS: {
2159 __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2160 i.InputSimd128Register(1), E32, m1);
2161 break;
2162 }
2163 case kRiscvI64x2GtS: {
2164 __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2165 i.InputSimd128Register(1), E64, m1);
2166 break;
2167 }
2168 case kRiscvI8x16GtU: {
2169 __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2170 i.InputSimd128Register(1), E8, m1);
2171 break;
2172 }
2173 case kRiscvI16x8GtU: {
2174 __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2175 i.InputSimd128Register(1), E16, m1);
2176 break;
2177 }
2178 case kRiscvI32x4GtU: {
2179 __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2180 i.InputSimd128Register(1), E32, m1);
2181 break;
2182 }
2183 case kRiscvI8x16Shl: {
2184 __ VU.set(kScratchReg, E8, m1);
2185 if (instr->InputAt(1)->IsRegister()) {
2186 __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2187 i.InputRegister(1));
2188 } else {
2189 __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2190 i.InputInt3(1));
2191 }
2192 break;
2193 }
2194 case kRiscvI16x8Shl: {
2195 __ VU.set(kScratchReg, E16, m1);
2196 if (instr->InputAt(1)->IsRegister()) {
2197 __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2198 i.InputRegister(1));
2199 } else {
2200 __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2201 i.InputInt4(1));
2202 }
2203 break;
2204 }
2205 case kRiscvI32x4Shl: {
2206 __ VU.set(kScratchReg, E32, m1);
2207 if (instr->InputAt(1)->IsRegister()) {
2208 __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2209 i.InputRegister(1));
2210 } else {
2211 __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2212 i.InputInt5(1));
2213 }
2214 break;
2215 }
2216 case kRiscvI64x2Shl: {
2217 __ VU.set(kScratchReg, E64, m1);
2218 if (instr->InputAt(1)->IsRegister()) {
2219 __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2220 i.InputRegister(1));
2221 } else {
2222 if (is_int5(i.InputInt6(1))) {
2223 __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2224 i.InputInt6(1));
2225 } else {
2226 __ li(kScratchReg, i.InputInt6(1));
2227 __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2228 kScratchReg);
2229 }
2230 }
2231 break;
2232 }
2233 case kRiscvI8x16ReplaceLane: {
2234 Simd128Register src = i.InputSimd128Register(0);
2235 Simd128Register dst = i.OutputSimd128Register();
2236 __ VU.set(kScratchReg, E32, m1);
2237 __ li(kScratchReg, 0x1 << i.InputInt8(1));
2238 __ vmv_sx(v0, kScratchReg);
2239 __ vmerge_vx(dst, i.InputRegister(2), src);
2240 break;
2241 }
2242 case kRiscvI16x8ReplaceLane: {
2243 Simd128Register src = i.InputSimd128Register(0);
2244 Simd128Register dst = i.OutputSimd128Register();
2245 __ VU.set(kScratchReg, E16, m1);
2246 __ li(kScratchReg, 0x1 << i.InputInt8(1));
2247 __ vmv_sx(v0, kScratchReg);
2248 __ vmerge_vx(dst, i.InputRegister(2), src);
2249 break;
2250 }
2251 case kRiscvI64x2ReplaceLane: {
2252 Simd128Register src = i.InputSimd128Register(0);
2253 Simd128Register dst = i.OutputSimd128Register();
2254 __ VU.set(kScratchReg, E64, m1);
2255 __ li(kScratchReg, 0x1 << i.InputInt8(1));
2256 __ vmv_sx(v0, kScratchReg);
2257 __ vmerge_vx(dst, i.InputRegister(2), src);
2258 break;
2259 }
2260 case kRiscvI32x4ReplaceLane: {
2261 Simd128Register src = i.InputSimd128Register(0);
2262 Simd128Register dst = i.OutputSimd128Register();
2263 __ VU.set(kScratchReg, E32, m1);
2264 __ li(kScratchReg, 0x1 << i.InputInt8(1));
2265 __ vmv_sx(v0, kScratchReg);
2266 __ vmerge_vx(dst, i.InputRegister(2), src);
2267 break;
2268 }
2269 case kRiscvI8x16BitMask: {
2270 Register dst = i.OutputRegister();
2271 Simd128Register src = i.InputSimd128Register(0);
2272 __ VU.set(kScratchReg, E8, m1);
2273 __ vmv_vx(kSimd128RegZero, zero_reg);
2274 __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2275 __ VU.set(kScratchReg, E32, m1);
2276 __ vmv_xs(dst, kSimd128ScratchReg);
2277 break;
2278 }
2279 case kRiscvI16x8BitMask: {
2280 Register dst = i.OutputRegister();
2281 Simd128Register src = i.InputSimd128Register(0);
2282 __ VU.set(kScratchReg, E16, m1);
2283 __ vmv_vx(kSimd128RegZero, zero_reg);
2284 __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2285 __ VU.set(kScratchReg, E32, m1);
2286 __ vmv_xs(dst, kSimd128ScratchReg);
2287 break;
2288 }
2289 case kRiscvI32x4BitMask: {
2290 Register dst = i.OutputRegister();
2291 Simd128Register src = i.InputSimd128Register(0);
2292 __ VU.set(kScratchReg, E32, m1);
2293 __ vmv_vx(kSimd128RegZero, zero_reg);
2294 __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2295 __ vmv_xs(dst, kSimd128ScratchReg);
2296 break;
2297 }
2298 case kRiscvI64x2BitMask: {
2299 Register dst = i.OutputRegister();
2300 Simd128Register src = i.InputSimd128Register(0);
2301 __ VU.set(kScratchReg, E64, m1);
2302 __ vmv_vx(kSimd128RegZero, zero_reg);
2303 __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2304 __ VU.set(kScratchReg, E32, m1);
2305 __ vmv_xs(dst, kSimd128ScratchReg);
2306 break;
2307 }
2308 case kRiscvV128AnyTrue: {
2309 __ VU.set(kScratchReg, E8, m1);
2310 Register dst = i.OutputRegister();
2311 Label t;
2312 __ vmv_sx(kSimd128ScratchReg, zero_reg);
2313 __ vredmaxu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2314 kSimd128ScratchReg);
2315 __ vmv_xs(dst, kSimd128ScratchReg);
2316 __ beq(dst, zero_reg, &t);
2317 __ li(dst, 1);
2318 __ bind(&t);
2319 break;
2320 }
2321 case kRiscvI64x2AllTrue: {
2322 __ VU.set(kScratchReg, E64, m1);
2323 Register dst = i.OutputRegister();
2324 Label all_true;
2325 __ li(kScratchReg, -1);
2326 __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2327 __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2328 kSimd128ScratchReg);
2329 __ vmv_xs(dst, kSimd128ScratchReg);
2330 __ beqz(dst, &all_true);
2331 __ li(dst, 1);
2332 __ bind(&all_true);
2333 break;
2334 }
2335 case kRiscvI32x4AllTrue: {
2336 __ VU.set(kScratchReg, E32, m1);
2337 Register dst = i.OutputRegister();
2338 Label all_true;
2339 __ li(kScratchReg, -1);
2340 __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2341 __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2342 kSimd128ScratchReg);
2343 __ vmv_xs(dst, kSimd128ScratchReg);
2344 __ beqz(dst, &all_true);
2345 __ li(dst, 1);
2346 __ bind(&all_true);
2347 break;
2348 }
2349 case kRiscvI16x8AllTrue: {
2350 __ VU.set(kScratchReg, E16, m1);
2351 Register dst = i.OutputRegister();
2352 Label all_true;
2353 __ li(kScratchReg, -1);
2354 __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2355 __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2356 kSimd128ScratchReg);
2357 __ vmv_xs(dst, kSimd128ScratchReg);
2358 __ beqz(dst, &all_true);
2359 __ li(dst, 1);
2360 __ bind(&all_true);
2361 break;
2362 }
2363 case kRiscvI8x16AllTrue: {
2364 __ VU.set(kScratchReg, E8, m1);
2365 Register dst = i.OutputRegister();
2366 Label all_true;
2367 __ li(kScratchReg, -1);
2368 __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2369 __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2370 kSimd128ScratchReg);
2371 __ vmv_xs(dst, kSimd128ScratchReg);
2372 __ beqz(dst, &all_true);
2373 __ li(dst, 1);
2374 __ bind(&all_true);
2375 break;
2376 }
2377 case kRiscvI8x16Shuffle: {
2378 VRegister dst = i.OutputSimd128Register(),
2379 src0 = i.InputSimd128Register(0),
2380 src1 = i.InputSimd128Register(1);
2381
2382 int64_t imm1 = make_uint64(i.InputInt32(3), i.InputInt32(2));
2383 int64_t imm2 = make_uint64(i.InputInt32(5), i.InputInt32(4));
2384 __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2385 __ li(kScratchReg, 1);
2386 __ vmv_vx(v0, kScratchReg);
2387 __ li(kScratchReg, imm1);
2388 __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg);
2389 __ li(kScratchReg, imm2);
2390 __ vsll_vi(v0, v0, 1);
2391 __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg);
2392
2393 __ VU.set(kScratchReg, E8, m1);
2394 if (dst == src0) {
2395 __ vmv_vv(kSimd128ScratchReg2, src0);
2396 src0 = kSimd128ScratchReg2;
2397 } else if (dst == src1) {
2398 __ vmv_vv(kSimd128ScratchReg2, src1);
2399 src1 = kSimd128ScratchReg2;
2400 }
2401 __ vrgather_vv(dst, src0, kSimd128ScratchReg);
2402 __ vadd_vi(kSimd128ScratchReg, kSimd128ScratchReg, -16);
2403 __ vrgather_vv(kSimd128ScratchReg, src1, kSimd128ScratchReg);
2404 __ vor_vv(dst, dst, kSimd128ScratchReg);
2405 break;
2406 }
2407 case kRiscvF32x4Abs: {
2408 __ VU.set(kScratchReg, VSew::E32, Vlmul::m1);
2409 __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2410 break;
2411 }
2412 case kRiscvF64x2Abs: {
2413 __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2414 __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2415 break;
2416 }
2417 case kRiscvF32x4Neg: {
2418 __ VU.set(kScratchReg, VSew::E32, Vlmul::m1);
2419 __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2420 break;
2421 }
2422 case kRiscvF64x2Neg: {
2423 __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2424 __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2425 break;
2426 }
2427 case kRiscvF32x4DemoteF64x2Zero: {
2428 __ VU.set(kScratchReg, E32, m1);
2429 __ vfncvt_f_f_w(i.OutputSimd128Register(), i.InputSimd128Register(0));
2430 __ vmv_vi(v0, 12);
2431 __ vmerge_vx(i.OutputSimd128Register(), zero_reg,
2432 i.OutputSimd128Register());
2433 break;
2434 }
2435 case kRiscvF32x4Add: {
2436 __ VU.set(kScratchReg, E32, m1);
2437 __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2438 i.InputSimd128Register(1));
2439 break;
2440 }
2441 case kRiscvF32x4Sub: {
2442 __ VU.set(kScratchReg, E32, m1);
2443 __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2444 i.InputSimd128Register(1));
2445 break;
2446 }
2447 case kRiscvF64x2Add: {
2448 __ VU.set(kScratchReg, E64, m1);
2449 __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2450 i.InputSimd128Register(1));
2451 break;
2452 }
2453 case kRiscvF64x2Sub: {
2454 __ VU.set(kScratchReg, E64, m1);
2455 __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2456 i.InputSimd128Register(1));
2457 break;
2458 }
2459 case kRiscvF32x4Ceil: {
2460 __ Ceil_f(i.OutputSimd128Register(), i.InputSimd128Register(0),
2461 kScratchReg, kSimd128ScratchReg);
2462 break;
2463 }
2464 case kRiscvF64x2Ceil: {
2465 __ Ceil_d(i.OutputSimd128Register(), i.InputSimd128Register(0),
2466 kScratchReg, kSimd128ScratchReg);
2467 break;
2468 }
2469 case kRiscvF32x4Floor: {
2470 __ Floor_f(i.OutputSimd128Register(), i.InputSimd128Register(0),
2471 kScratchReg, kSimd128ScratchReg);
2472 break;
2473 }
2474 case kRiscvF64x2Floor: {
2475 __ Floor_d(i.OutputSimd128Register(), i.InputSimd128Register(0),
2476 kScratchReg, kSimd128ScratchReg);
2477 break;
2478 }
2479 case kRiscvS128Select: {
2480 __ VU.set(kScratchReg, E8, m1);
2481 __ vand_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2482 i.InputSimd128Register(0));
2483 __ vnot_vv(kSimd128ScratchReg2, i.InputSimd128Register(0));
2484 __ vand_vv(kSimd128ScratchReg2, i.InputSimd128Register(2),
2485 kSimd128ScratchReg2);
2486 __ vor_vv(i.OutputSimd128Register(), kSimd128ScratchReg,
2487 kSimd128ScratchReg2);
2488 break;
2489 }
2490 case kRiscvF32x4UConvertI32x4: {
2491 __ VU.set(kScratchReg, E32, m1);
2492 __ VU.set(RoundingMode::RTZ);
2493 __ vfcvt_f_xu_v(i.OutputSimd128Register(), i.InputSimd128Register(0));
2494 break;
2495 }
2496 case kRiscvF32x4SConvertI32x4: {
2497 __ VU.set(kScratchReg, E32, m1);
2498 __ VU.set(RoundingMode::RTZ);
2499 __ vfcvt_f_x_v(i.OutputSimd128Register(), i.InputSimd128Register(0));
2500 break;
2501 }
2502 case kRiscvF32x4Div: {
2503 __ VU.set(kScratchReg, E32, m1);
2504 __ VU.set(RoundingMode::RTZ);
2505 __ vfdiv_vv(i.OutputSimd128Register(), i.InputSimd128Register(1),
2506 i.InputSimd128Register(0));
2507 break;
2508 }
2509 case kRiscvF32x4Mul: {
2510 __ VU.set(kScratchReg, E32, m1);
2511 __ VU.set(RoundingMode::RTZ);
2512 __ vfmul_vv(i.OutputSimd128Register(), i.InputSimd128Register(1),
2513 i.InputSimd128Register(0));
2514 break;
2515 }
2516 case kRiscvF32x4Eq: {
2517 __ VU.set(kScratchReg, E32, m1);
2518 __ vmfeq_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2519 __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2520 __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2521 break;
2522 }
2523 case kRiscvF32x4Ne: {
2524 __ VU.set(kScratchReg, E32, m1);
2525 __ vmfne_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2526 __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2527 __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2528 break;
2529 }
2530 case kRiscvF32x4Lt: {
2531 __ VU.set(kScratchReg, E32, m1);
2532 __ vmflt_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2533 __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2534 __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2535 break;
2536 }
2537 case kRiscvF32x4Le: {
2538 __ VU.set(kScratchReg, E32, m1);
2539 __ vmfle_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2540 __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2541 __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2542 break;
2543 }
2544 case kRiscvF32x4Max: {
2545 __ VU.set(kScratchReg, E32, m1);
2546 const int32_t kNaN = 0x7FC00000;
2547 __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0));
2548 __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2549 i.InputSimd128Register(1));
2550 __ vand_vv(v0, v0, kSimd128ScratchReg);
2551 __ li(kScratchReg, kNaN);
2552 __ vmv_vx(kSimd128ScratchReg, kScratchReg);
2553 __ vfmax_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2554 i.InputSimd128Register(0), Mask);
2555 __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg);
2556 break;
2557 }
2558 case kRiscvF32x4Min: {
2559 __ VU.set(kScratchReg, E32, m1);
2560 const int32_t kNaN = 0x7FC00000;
2561 __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0));
2562 __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2563 i.InputSimd128Register(1));
2564 __ vand_vv(v0, v0, kSimd128ScratchReg);
2565 __ li(kScratchReg, kNaN);
2566 __ vmv_vx(kSimd128ScratchReg, kScratchReg);
2567 __ vfmin_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2568 i.InputSimd128Register(0), Mask);
2569 __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg);
2570 break;
2571 }
2572 default:
2573 #ifdef DEBUG
2574 switch (arch_opcode) {
2575 #define Print(name) \
2576 case k##name: \
2577 printf("k%s", #name); \
2578 break;
2579 TARGET_ARCH_OPCODE_LIST(Print);
2580 #undef Print
2581 default:
2582 break;
2583 }
2584 #endif
2585 UNIMPLEMENTED();
2586 }
2587 return kSuccess;
2588 }
2589
2590 #define UNSUPPORTED_COND(opcode, condition) \
2591 StdoutStream{} << "Unsupported " << #opcode << " condition: \"" << condition \
2592 << "\""; \
2593 UNIMPLEMENTED();
2594
IsInludeEqual(Condition cc)2595 bool IsInludeEqual(Condition cc) {
2596 switch (cc) {
2597 case equal:
2598 case greater_equal:
2599 case less_equal:
2600 case Uless_equal:
2601 case Ugreater_equal:
2602 return true;
2603 default:
2604 return false;
2605 }
2606 }
2607
AssembleBranchToLabels(CodeGenerator * gen,TurboAssembler * tasm,Instruction * instr,FlagsCondition condition,Label * tlabel,Label * flabel,bool fallthru)2608 void AssembleBranchToLabels(CodeGenerator* gen, TurboAssembler* tasm,
2609 Instruction* instr, FlagsCondition condition,
2610 Label* tlabel, Label* flabel, bool fallthru) {
2611 #undef __
2612 #define __ tasm->
2613 RiscvOperandConverter i(gen, instr);
2614
2615 Condition cc = kNoCondition;
2616 // RISC-V does not have condition code flags, so compare and branch are
2617 // implemented differently than on the other arch's. The compare operations
2618 // emit riscv64 pseudo-instructions, which are handled here by branch
2619 // instructions that do the actual comparison. Essential that the input
2620 // registers to compare pseudo-op are not modified before this branch op, as
2621 // they are tested here.
2622
2623 if (instr->arch_opcode() == kRiscvTst) {
2624 cc = FlagsConditionToConditionTst(condition);
2625 __ Branch(tlabel, cc, kScratchReg, Operand(zero_reg));
2626 } else if (instr->arch_opcode() == kRiscvAdd64 ||
2627 instr->arch_opcode() == kRiscvSub64) {
2628 cc = FlagsConditionToConditionOvf(condition);
2629 __ Sra64(kScratchReg, i.OutputRegister(), 32);
2630 __ Sra64(kScratchReg2, i.OutputRegister(), 31);
2631 __ Branch(tlabel, cc, kScratchReg2, Operand(kScratchReg));
2632 } else if (instr->arch_opcode() == kRiscvAddOvf64 ||
2633 instr->arch_opcode() == kRiscvSubOvf64) {
2634 switch (condition) {
2635 // Overflow occurs if overflow register is negative
2636 case kOverflow:
2637 __ Branch(tlabel, lt, kScratchReg, Operand(zero_reg));
2638 break;
2639 case kNotOverflow:
2640 __ Branch(tlabel, ge, kScratchReg, Operand(zero_reg));
2641 break;
2642 default:
2643 UNSUPPORTED_COND(instr->arch_opcode(), condition);
2644 }
2645 } else if (instr->arch_opcode() == kRiscvMulOvf32) {
2646 // Overflow occurs if overflow register is not zero
2647 switch (condition) {
2648 case kOverflow:
2649 __ Branch(tlabel, ne, kScratchReg, Operand(zero_reg));
2650 break;
2651 case kNotOverflow:
2652 __ Branch(tlabel, eq, kScratchReg, Operand(zero_reg));
2653 break;
2654 default:
2655 UNSUPPORTED_COND(kRiscvMulOvf32, condition);
2656 }
2657 } else if (instr->arch_opcode() == kRiscvCmp) {
2658 cc = FlagsConditionToConditionCmp(condition);
2659 __ Branch(tlabel, cc, i.InputRegister(0), i.InputOperand(1));
2660 } else if (instr->arch_opcode() == kRiscvCmpZero) {
2661 cc = FlagsConditionToConditionCmp(condition);
2662 if (i.InputOrZeroRegister(0) == zero_reg && IsInludeEqual(cc)) {
2663 __ Branch(tlabel);
2664 } else if (i.InputOrZeroRegister(0) != zero_reg) {
2665 __ Branch(tlabel, cc, i.InputRegister(0), Operand(zero_reg));
2666 }
2667 } else if (instr->arch_opcode() == kArchStackPointerGreaterThan) {
2668 cc = FlagsConditionToConditionCmp(condition);
2669 Register lhs_register = sp;
2670 uint32_t offset;
2671 if (gen->ShouldApplyOffsetToStackCheck(instr, &offset)) {
2672 lhs_register = i.TempRegister(0);
2673 __ Sub64(lhs_register, sp, offset);
2674 }
2675 __ Branch(tlabel, cc, lhs_register, Operand(i.InputRegister(0)));
2676 } else if (instr->arch_opcode() == kRiscvCmpS ||
2677 instr->arch_opcode() == kRiscvCmpD) {
2678 bool predicate;
2679 FlagsConditionToConditionCmpFPU(&predicate, condition);
2680 // floating-point compare result is set in kScratchReg
2681 if (predicate) {
2682 __ BranchTrueF(kScratchReg, tlabel);
2683 } else {
2684 __ BranchFalseF(kScratchReg, tlabel);
2685 }
2686 } else {
2687 PrintF("AssembleArchBranch Unimplemented arch_opcode: %d\n",
2688 instr->arch_opcode());
2689 UNIMPLEMENTED();
2690 }
2691 if (!fallthru) __ Branch(flabel); // no fallthru to flabel.
2692 #undef __
2693 #define __ tasm()->
2694 }
2695
2696 // Assembles branches after an instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2697 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2698 Label* tlabel = branch->true_label;
2699 Label* flabel = branch->false_label;
2700
2701 AssembleBranchToLabels(this, tasm(), instr, branch->condition, tlabel, flabel,
2702 branch->fallthru);
2703 }
2704
2705 #undef UNSUPPORTED_COND
2706
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)2707 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2708 BranchInfo* branch) {
2709 AssembleArchBranch(instr, branch);
2710 }
2711
AssembleArchJump(RpoNumber target)2712 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2713 if (!IsNextInAssemblyOrder(target)) __ Branch(GetLabel(target));
2714 }
2715
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2716 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2717 FlagsCondition condition) {
2718 class OutOfLineTrap final : public OutOfLineCode {
2719 public:
2720 OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
2721 : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
2722 void Generate() final {
2723 RiscvOperandConverter i(gen_, instr_);
2724 TrapId trap_id =
2725 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
2726 GenerateCallToTrap(trap_id);
2727 }
2728
2729 private:
2730 void GenerateCallToTrap(TrapId trap_id) {
2731 if (trap_id == TrapId::kInvalid) {
2732 // We cannot test calls to the runtime in cctest/test-run-wasm.
2733 // Therefore we emit a call to C here instead of a call to the runtime.
2734 // We use the context register as the scratch register, because we do
2735 // not have a context here.
2736 __ PrepareCallCFunction(0, 0, cp);
2737 __ CallCFunction(
2738 ExternalReference::wasm_call_trap_callback_for_testing(), 0);
2739 __ LeaveFrame(StackFrame::WASM);
2740 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
2741 int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
2742 pop_count += (pop_count & 1); // align
2743 __ Drop(pop_count);
2744 __ Ret();
2745 } else {
2746 gen_->AssembleSourcePosition(instr_);
2747 // A direct call to a wasm runtime stub defined in this module.
2748 // Just encode the stub index. This will be patched when the code
2749 // is added to the native module and copied into wasm code space.
2750 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
2751 ReferenceMap* reference_map =
2752 gen_->zone()->New<ReferenceMap>(gen_->zone());
2753 gen_->RecordSafepoint(reference_map);
2754 if (FLAG_debug_code) {
2755 __ stop();
2756 }
2757 }
2758 }
2759 Instruction* instr_;
2760 CodeGenerator* gen_;
2761 };
2762 auto ool = zone()->New<OutOfLineTrap>(this, instr);
2763 Label* tlabel = ool->entry();
2764 AssembleBranchToLabels(this, tasm(), instr, condition, tlabel, nullptr, true);
2765 }
2766
2767 // Assembles boolean materializations after an instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2768 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2769 FlagsCondition condition) {
2770 RiscvOperandConverter i(this, instr);
2771
2772 // Materialize a full 32-bit 1 or 0 value. The result register is always the
2773 // last output of the instruction.
2774 DCHECK_NE(0u, instr->OutputCount());
2775 Register result = i.OutputRegister(instr->OutputCount() - 1);
2776 Condition cc = kNoCondition;
2777 // RISC-V does not have condition code flags, so compare and branch are
2778 // implemented differently than on the other arch's. The compare operations
2779 // emit riscv64 pseudo-instructions, which are checked and handled here.
2780
2781 if (instr->arch_opcode() == kRiscvTst) {
2782 cc = FlagsConditionToConditionTst(condition);
2783 if (cc == eq) {
2784 __ Sltu(result, kScratchReg, 1);
2785 } else {
2786 __ Sltu(result, zero_reg, kScratchReg);
2787 }
2788 return;
2789 } else if (instr->arch_opcode() == kRiscvAdd64 ||
2790 instr->arch_opcode() == kRiscvSub64) {
2791 cc = FlagsConditionToConditionOvf(condition);
2792 // Check for overflow creates 1 or 0 for result.
2793 __ Srl64(kScratchReg, i.OutputRegister(), 63);
2794 __ Srl32(kScratchReg2, i.OutputRegister(), 31);
2795 __ Xor(result, kScratchReg, kScratchReg2);
2796 if (cc == eq) // Toggle result for not overflow.
2797 __ Xor(result, result, 1);
2798 return;
2799 } else if (instr->arch_opcode() == kRiscvAddOvf64 ||
2800 instr->arch_opcode() == kRiscvSubOvf64) {
2801 // Overflow occurs if overflow register is negative
2802 __ Slt(result, kScratchReg, zero_reg);
2803 } else if (instr->arch_opcode() == kRiscvMulOvf32) {
2804 // Overflow occurs if overflow register is not zero
2805 __ Sgtu(result, kScratchReg, zero_reg);
2806 } else if (instr->arch_opcode() == kRiscvCmp) {
2807 cc = FlagsConditionToConditionCmp(condition);
2808 switch (cc) {
2809 case eq:
2810 case ne: {
2811 Register left = i.InputOrZeroRegister(0);
2812 Operand right = i.InputOperand(1);
2813 if (instr->InputAt(1)->IsImmediate()) {
2814 if (is_int12(-right.immediate())) {
2815 if (right.immediate() == 0) {
2816 if (cc == eq) {
2817 __ Sltu(result, left, 1);
2818 } else {
2819 __ Sltu(result, zero_reg, left);
2820 }
2821 } else {
2822 __ Add64(result, left, Operand(-right.immediate()));
2823 if (cc == eq) {
2824 __ Sltu(result, result, 1);
2825 } else {
2826 __ Sltu(result, zero_reg, result);
2827 }
2828 }
2829 } else {
2830 if (is_uint12(right.immediate())) {
2831 __ Xor(result, left, right);
2832 } else {
2833 __ li(kScratchReg, right);
2834 __ Xor(result, left, kScratchReg);
2835 }
2836 if (cc == eq) {
2837 __ Sltu(result, result, 1);
2838 } else {
2839 __ Sltu(result, zero_reg, result);
2840 }
2841 }
2842 } else {
2843 __ Xor(result, left, right);
2844 if (cc == eq) {
2845 __ Sltu(result, result, 1);
2846 } else {
2847 __ Sltu(result, zero_reg, result);
2848 }
2849 }
2850 } break;
2851 case lt:
2852 case ge: {
2853 Register left = i.InputOrZeroRegister(0);
2854 Operand right = i.InputOperand(1);
2855 __ Slt(result, left, right);
2856 if (cc == ge) {
2857 __ Xor(result, result, 1);
2858 }
2859 } break;
2860 case gt:
2861 case le: {
2862 Register left = i.InputOrZeroRegister(1);
2863 Operand right = i.InputOperand(0);
2864 __ Slt(result, left, right);
2865 if (cc == le) {
2866 __ Xor(result, result, 1);
2867 }
2868 } break;
2869 case Uless:
2870 case Ugreater_equal: {
2871 Register left = i.InputOrZeroRegister(0);
2872 Operand right = i.InputOperand(1);
2873 __ Sltu(result, left, right);
2874 if (cc == Ugreater_equal) {
2875 __ Xor(result, result, 1);
2876 }
2877 } break;
2878 case Ugreater:
2879 case Uless_equal: {
2880 Register left = i.InputRegister(1);
2881 Operand right = i.InputOperand(0);
2882 __ Sltu(result, left, right);
2883 if (cc == Uless_equal) {
2884 __ Xor(result, result, 1);
2885 }
2886 } break;
2887 default:
2888 UNREACHABLE();
2889 }
2890 return;
2891 } else if (instr->arch_opcode() == kRiscvCmpZero) {
2892 cc = FlagsConditionToConditionCmp(condition);
2893 switch (cc) {
2894 case eq: {
2895 Register left = i.InputOrZeroRegister(0);
2896 __ Sltu(result, left, 1);
2897 break;
2898 }
2899 case ne: {
2900 Register left = i.InputOrZeroRegister(0);
2901 __ Sltu(result, zero_reg, left);
2902 break;
2903 }
2904 case lt:
2905 case ge: {
2906 Register left = i.InputOrZeroRegister(0);
2907 Operand right = Operand(zero_reg);
2908 __ Slt(result, left, right);
2909 if (cc == ge) {
2910 __ Xor(result, result, 1);
2911 }
2912 } break;
2913 case gt:
2914 case le: {
2915 Operand left = i.InputOperand(0);
2916 __ Slt(result, zero_reg, left);
2917 if (cc == le) {
2918 __ Xor(result, result, 1);
2919 }
2920 } break;
2921 case Uless:
2922 case Ugreater_equal: {
2923 Register left = i.InputOrZeroRegister(0);
2924 Operand right = Operand(zero_reg);
2925 __ Sltu(result, left, right);
2926 if (cc == Ugreater_equal) {
2927 __ Xor(result, result, 1);
2928 }
2929 } break;
2930 case Ugreater:
2931 case Uless_equal: {
2932 Register left = zero_reg;
2933 Operand right = i.InputOperand(0);
2934 __ Sltu(result, left, right);
2935 if (cc == Uless_equal) {
2936 __ Xor(result, result, 1);
2937 }
2938 } break;
2939 default:
2940 UNREACHABLE();
2941 }
2942 return;
2943 } else if (instr->arch_opcode() == kArchStackPointerGreaterThan) {
2944 cc = FlagsConditionToConditionCmp(condition);
2945 Register lhs_register = sp;
2946 uint32_t offset;
2947 if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
2948 lhs_register = i.TempRegister(0);
2949 __ Sub64(lhs_register, sp, offset);
2950 }
2951 __ Sgtu(result, lhs_register, Operand(i.InputRegister(0)));
2952 return;
2953 } else if (instr->arch_opcode() == kRiscvCmpD ||
2954 instr->arch_opcode() == kRiscvCmpS) {
2955 FPURegister left = i.InputOrZeroDoubleRegister(0);
2956 FPURegister right = i.InputOrZeroDoubleRegister(1);
2957 if ((instr->arch_opcode() == kRiscvCmpD) &&
2958 (left == kDoubleRegZero || right == kDoubleRegZero) &&
2959 !__ IsDoubleZeroRegSet()) {
2960 __ LoadFPRImmediate(kDoubleRegZero, 0.0);
2961 } else if ((instr->arch_opcode() == kRiscvCmpS) &&
2962 (left == kDoubleRegZero || right == kDoubleRegZero) &&
2963 !__ IsSingleZeroRegSet()) {
2964 __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
2965 }
2966 bool predicate;
2967 FlagsConditionToConditionCmpFPU(&predicate, condition);
2968 // RISCV compare returns 0 or 1, do nothing when predicate; otherwise
2969 // toggle kScratchReg (i.e., 0 -> 1, 1 -> 0)
2970 if (predicate) {
2971 __ Move(result, kScratchReg);
2972 } else {
2973 __ Xor(result, kScratchReg, 1);
2974 }
2975 return;
2976 } else {
2977 PrintF("AssembleArchBranch Unimplemented arch_opcode is : %d\n",
2978 instr->arch_opcode());
2979 TRACE_UNIMPL();
2980 UNIMPLEMENTED();
2981 }
2982 }
2983
AssembleArchBinarySearchSwitch(Instruction * instr)2984 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2985 RiscvOperandConverter i(this, instr);
2986 Register input = i.InputRegister(0);
2987 std::vector<std::pair<int32_t, Label*>> cases;
2988 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2989 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2990 }
2991 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2992 cases.data() + cases.size());
2993 }
2994
AssembleArchTableSwitch(Instruction * instr)2995 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2996 RiscvOperandConverter i(this, instr);
2997 Register input = i.InputRegister(0);
2998 size_t const case_count = instr->InputCount() - 2;
2999
3000 __ Branch(GetLabel(i.InputRpo(1)), Ugreater_equal, input,
3001 Operand(case_count));
3002 __ GenerateSwitchTable(input, case_count, [&i, this](size_t index) {
3003 return GetLabel(i.InputRpo(index + 2));
3004 });
3005 }
3006
FinishFrame(Frame * frame)3007 void CodeGenerator::FinishFrame(Frame* frame) {
3008 auto call_descriptor = linkage()->GetIncomingDescriptor();
3009
3010 const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3011 if (saves_fpu != 0) {
3012 int count = base::bits::CountPopulation(saves_fpu);
3013 DCHECK_EQ(kNumCalleeSavedFPU, count);
3014 frame->AllocateSavedCalleeRegisterSlots(count *
3015 (kDoubleSize / kSystemPointerSize));
3016 }
3017
3018 const RegList saves = call_descriptor->CalleeSavedRegisters();
3019 if (saves != 0) {
3020 int count = base::bits::CountPopulation(saves);
3021 frame->AllocateSavedCalleeRegisterSlots(count);
3022 }
3023 }
3024
AssembleConstructFrame()3025 void CodeGenerator::AssembleConstructFrame() {
3026 auto call_descriptor = linkage()->GetIncomingDescriptor();
3027
3028 if (frame_access_state()->has_frame()) {
3029 if (call_descriptor->IsCFunctionCall()) {
3030 if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
3031 __ StubPrologue(StackFrame::C_WASM_ENTRY);
3032 // Reserve stack space for saving the c_entry_fp later.
3033 __ Sub64(sp, sp, Operand(kSystemPointerSize));
3034 } else {
3035 __ Push(ra, fp);
3036 __ Move(fp, sp);
3037 }
3038 } else if (call_descriptor->IsJSFunctionCall()) {
3039 __ Prologue();
3040 } else {
3041 __ StubPrologue(info()->GetOutputStackFrameType());
3042 if (call_descriptor->IsWasmFunctionCall()) {
3043 __ Push(kWasmInstanceRegister);
3044 } else if (call_descriptor->IsWasmImportWrapper() ||
3045 call_descriptor->IsWasmCapiFunction()) {
3046 // Wasm import wrappers are passed a tuple in the place of the instance.
3047 // Unpack the tuple into the instance and the target callable.
3048 // This must be done here in the codegen because it cannot be expressed
3049 // properly in the graph.
3050 __ LoadTaggedPointerField(
3051 kJSFunctionRegister,
3052 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue2Offset));
3053 __ LoadTaggedPointerField(
3054 kWasmInstanceRegister,
3055 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue1Offset));
3056 __ Push(kWasmInstanceRegister);
3057 if (call_descriptor->IsWasmCapiFunction()) {
3058 // Reserve space for saving the PC later.
3059 __ Sub64(sp, sp, Operand(kSystemPointerSize));
3060 }
3061 }
3062 }
3063 }
3064
3065 int required_slots =
3066 frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
3067
3068 if (info()->is_osr()) {
3069 // TurboFan OSR-compiled functions cannot be entered directly.
3070 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
3071
3072 // Unoptimized code jumps directly to this entrypoint while the unoptimized
3073 // frame is still on the stack. Optimized code uses OSR values directly from
3074 // the unoptimized frame. Thus, all that needs to be done is to allocate the
3075 // remaining stack slots.
3076 __ RecordComment("-- OSR entrypoint --");
3077 osr_pc_offset_ = __ pc_offset();
3078 required_slots -= osr_helper()->UnoptimizedFrameSlots();
3079 }
3080
3081 const RegList saves = call_descriptor->CalleeSavedRegisters();
3082 const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3083
3084 if (required_slots > 0) {
3085 DCHECK(frame_access_state()->has_frame());
3086 if (info()->IsWasm() && required_slots > 128) {
3087 // For WebAssembly functions with big frames we have to do the stack
3088 // overflow check before we construct the frame. Otherwise we may not
3089 // have enough space on the stack to call the runtime for the stack
3090 // overflow.
3091 Label done;
3092
3093 // If the frame is bigger than the stack, we throw the stack overflow
3094 // exception unconditionally. Thereby we can avoid the integer overflow
3095 // check in the condition code.
3096 if ((required_slots * kSystemPointerSize) < (FLAG_stack_size * 1024)) {
3097 __ Ld(
3098 kScratchReg,
3099 FieldMemOperand(kWasmInstanceRegister,
3100 WasmInstanceObject::kRealStackLimitAddressOffset));
3101 __ Ld(kScratchReg, MemOperand(kScratchReg));
3102 __ Add64(kScratchReg, kScratchReg,
3103 Operand(required_slots * kSystemPointerSize));
3104 __ BranchShort(&done, uge, sp, Operand(kScratchReg));
3105 }
3106
3107 __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
3108 // We come from WebAssembly, there are no references for the GC.
3109 ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
3110 RecordSafepoint(reference_map);
3111 if (FLAG_debug_code) {
3112 __ stop();
3113 }
3114
3115 __ bind(&done);
3116 }
3117 }
3118
3119 const int returns = frame()->GetReturnSlotCount();
3120
3121 // Skip callee-saved and return slots, which are pushed below.
3122 required_slots -= base::bits::CountPopulation(saves);
3123 required_slots -= base::bits::CountPopulation(saves_fpu);
3124 required_slots -= returns;
3125 if (required_slots > 0) {
3126 __ Sub64(sp, sp, Operand(required_slots * kSystemPointerSize));
3127 }
3128
3129 if (saves_fpu != 0) {
3130 // Save callee-saved FPU registers.
3131 __ MultiPushFPU(saves_fpu);
3132 DCHECK_EQ(kNumCalleeSavedFPU, base::bits::CountPopulation(saves_fpu));
3133 }
3134
3135 if (saves != 0) {
3136 // Save callee-saved registers.
3137 __ MultiPush(saves);
3138 }
3139
3140 if (returns != 0) {
3141 // Create space for returns.
3142 __ Sub64(sp, sp, Operand(returns * kSystemPointerSize));
3143 }
3144 }
3145
AssembleReturn(InstructionOperand * additional_pop_count)3146 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
3147 auto call_descriptor = linkage()->GetIncomingDescriptor();
3148
3149 const int returns = frame()->GetReturnSlotCount();
3150 if (returns != 0) {
3151 __ Add64(sp, sp, Operand(returns * kSystemPointerSize));
3152 }
3153
3154 // Restore GP registers.
3155 const RegList saves = call_descriptor->CalleeSavedRegisters();
3156 if (saves != 0) {
3157 __ MultiPop(saves);
3158 }
3159
3160 // Restore FPU registers.
3161 const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3162 if (saves_fpu != 0) {
3163 __ MultiPopFPU(saves_fpu);
3164 }
3165
3166 RiscvOperandConverter g(this, nullptr);
3167
3168 const int parameter_slots =
3169 static_cast<int>(call_descriptor->ParameterSlotCount());
3170
3171 // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
3172 // Check RawMachineAssembler::PopAndReturn.
3173 if (parameter_slots != 0) {
3174 if (additional_pop_count->IsImmediate()) {
3175 DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
3176 } else if (FLAG_debug_code) {
3177 __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue,
3178 g.ToRegister(additional_pop_count),
3179 Operand(static_cast<int64_t>(0)));
3180 }
3181 }
3182
3183 // Functions with JS linkage have at least one parameter (the receiver).
3184 // If {parameter_slots} == 0, it means it is a builtin with
3185 // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
3186 // itself.
3187 const bool drop_jsargs = frame_access_state()->has_frame() &&
3188 call_descriptor->IsJSFunctionCall() &&
3189 parameter_slots != 0;
3190
3191 if (call_descriptor->IsCFunctionCall()) {
3192 AssembleDeconstructFrame();
3193 } else if (frame_access_state()->has_frame()) {
3194 // Canonicalize JSFunction return sites for now unless they have an variable
3195 // number of stack slot pops.
3196 if (additional_pop_count->IsImmediate() &&
3197 g.ToConstant(additional_pop_count).ToInt32() == 0) {
3198 if (return_label_.is_bound()) {
3199 __ Branch(&return_label_);
3200 return;
3201 } else {
3202 __ bind(&return_label_);
3203 }
3204 }
3205 if (drop_jsargs) {
3206 // Get the actual argument count
3207 __ Ld(t0, MemOperand(fp, StandardFrameConstants::kArgCOffset));
3208 }
3209 AssembleDeconstructFrame();
3210 }
3211 if (drop_jsargs) {
3212 // We must pop all arguments from the stack (including the receiver). This
3213 // number of arguments is given by max(1 + argc_reg, parameter_slots).
3214 __ Add64(t0, t0, Operand(1)); // Also pop the receiver.
3215 if (parameter_slots > 1) {
3216 Label done;
3217 __ li(kScratchReg, parameter_slots);
3218 __ BranchShort(&done, ge, t0, Operand(kScratchReg));
3219 __ Move(t0, kScratchReg);
3220 __ bind(&done);
3221 }
3222 __ Sll64(t0, t0, kSystemPointerSizeLog2);
3223 __ Add64(sp, sp, t0);
3224 } else if (additional_pop_count->IsImmediate()) {
3225 // it should be a kInt32 or a kInt64
3226 DCHECK_LE(g.ToConstant(additional_pop_count).type(), Constant::kInt64);
3227 int additional_count = g.ToConstant(additional_pop_count).ToInt32();
3228 __ Drop(parameter_slots + additional_count);
3229 } else {
3230 Register pop_reg = g.ToRegister(additional_pop_count);
3231 __ Drop(parameter_slots);
3232 __ Sll64(pop_reg, pop_reg, kSystemPointerSizeLog2);
3233 __ Add64(sp, sp, pop_reg);
3234 }
3235 __ Ret();
3236 }
3237
FinishCode()3238 void CodeGenerator::FinishCode() { __ ForceConstantPoolEmissionWithoutJump(); }
3239
PrepareForDeoptimizationExits(ZoneDeque<DeoptimizationExit * > * exits)3240 void CodeGenerator::PrepareForDeoptimizationExits(
3241 ZoneDeque<DeoptimizationExit*>* exits) {
3242 __ ForceConstantPoolEmissionWithoutJump();
3243 int total_size = 0;
3244 for (DeoptimizationExit* exit : deoptimization_exits_) {
3245 total_size += (exit->kind() == DeoptimizeKind::kLazy)
3246 ? Deoptimizer::kLazyDeoptExitSize
3247 : Deoptimizer::kNonLazyDeoptExitSize;
3248 }
3249
3250 __ CheckTrampolinePoolQuick(total_size);
3251 DCHECK(Deoptimizer::kSupportsFixedDeoptExitSizes);
3252 }
3253
AssembleMove(InstructionOperand * source,InstructionOperand * destination)3254 void CodeGenerator::AssembleMove(InstructionOperand* source,
3255 InstructionOperand* destination) {
3256 RiscvOperandConverter g(this, nullptr);
3257 // Dispatch on the source and destination operand kinds. Not all
3258 // combinations are possible.
3259 if (source->IsRegister()) {
3260 DCHECK(destination->IsRegister() || destination->IsStackSlot());
3261 Register src = g.ToRegister(source);
3262 if (destination->IsRegister()) {
3263 __ Move(g.ToRegister(destination), src);
3264 } else {
3265 __ Sd(src, g.ToMemOperand(destination));
3266 }
3267 } else if (source->IsStackSlot()) {
3268 DCHECK(destination->IsRegister() || destination->IsStackSlot());
3269 MemOperand src = g.ToMemOperand(source);
3270 if (destination->IsRegister()) {
3271 __ Ld(g.ToRegister(destination), src);
3272 } else {
3273 Register temp = kScratchReg;
3274 __ Ld(temp, src);
3275 __ Sd(temp, g.ToMemOperand(destination));
3276 }
3277 } else if (source->IsConstant()) {
3278 Constant src = g.ToConstant(source);
3279 if (destination->IsRegister() || destination->IsStackSlot()) {
3280 Register dst =
3281 destination->IsRegister() ? g.ToRegister(destination) : kScratchReg;
3282 switch (src.type()) {
3283 case Constant::kInt32:
3284 if (src.ToInt32() == 0 && destination->IsStackSlot()) {
3285 dst = zero_reg;
3286 } else {
3287 __ li(dst, Operand(src.ToInt32()));
3288 }
3289 break;
3290 case Constant::kFloat32:
3291 __ li(dst, Operand::EmbeddedNumber(src.ToFloat32()));
3292 break;
3293 case Constant::kInt64:
3294 if (RelocInfo::IsWasmReference(src.rmode())) {
3295 __ li(dst, Operand(src.ToInt64(), src.rmode()));
3296 } else {
3297 if (src.ToInt64() == 0 && destination->IsStackSlot()) {
3298 dst = zero_reg;
3299 } else {
3300 __ li(dst, Operand(src.ToInt64()));
3301 }
3302 }
3303 break;
3304 case Constant::kFloat64:
3305 __ li(dst, Operand::EmbeddedNumber(src.ToFloat64().value()));
3306 break;
3307 case Constant::kExternalReference:
3308 __ li(dst, src.ToExternalReference());
3309 break;
3310 case Constant::kDelayedStringConstant:
3311 __ li(dst, src.ToDelayedStringConstant());
3312 break;
3313 case Constant::kHeapObject: {
3314 Handle<HeapObject> src_object = src.ToHeapObject();
3315 RootIndex index;
3316 if (IsMaterializableFromRoot(src_object, &index)) {
3317 __ LoadRoot(dst, index);
3318 } else {
3319 __ li(dst, src_object);
3320 }
3321 break;
3322 }
3323 case Constant::kCompressedHeapObject: {
3324 Handle<HeapObject> src_object = src.ToHeapObject();
3325 RootIndex index;
3326 if (IsMaterializableFromRoot(src_object, &index)) {
3327 __ LoadRoot(dst, index);
3328 } else {
3329 __ li(dst, src_object, RelocInfo::COMPRESSED_EMBEDDED_OBJECT);
3330 }
3331 break;
3332 }
3333 case Constant::kRpoNumber:
3334 UNREACHABLE(); // TODO(titzer): loading RPO numbers
3335 }
3336 if (destination->IsStackSlot()) __ Sd(dst, g.ToMemOperand(destination));
3337 } else if (src.type() == Constant::kFloat32) {
3338 if (destination->IsFPStackSlot()) {
3339 MemOperand dst = g.ToMemOperand(destination);
3340 if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
3341 __ Sw(zero_reg, dst);
3342 } else {
3343 __ li(kScratchReg, Operand(bit_cast<int32_t>(src.ToFloat32())));
3344 __ Sw(kScratchReg, dst);
3345 }
3346 } else {
3347 DCHECK(destination->IsFPRegister());
3348 FloatRegister dst = g.ToSingleRegister(destination);
3349 __ LoadFPRImmediate(dst, src.ToFloat32());
3350 }
3351 } else {
3352 DCHECK_EQ(Constant::kFloat64, src.type());
3353 DoubleRegister dst = destination->IsFPRegister()
3354 ? g.ToDoubleRegister(destination)
3355 : kScratchDoubleReg;
3356 __ LoadFPRImmediate(dst, src.ToFloat64().value());
3357 if (destination->IsFPStackSlot()) {
3358 __ StoreDouble(dst, g.ToMemOperand(destination));
3359 }
3360 }
3361 } else if (source->IsFPRegister()) {
3362 MachineRepresentation rep = LocationOperand::cast(source)->representation();
3363 if (rep == MachineRepresentation::kSimd128) {
3364 VRegister src = g.ToSimd128Register(source);
3365 if (destination->IsSimd128Register()) {
3366 VRegister dst = g.ToSimd128Register(destination);
3367 __ vmv_vv(dst, src);
3368 } else {
3369 DCHECK(destination->IsSimd128StackSlot());
3370 Register dst = g.ToMemOperand(destination).offset() == 0
3371 ? g.ToMemOperand(destination).rm()
3372 : kScratchReg;
3373 if (g.ToMemOperand(destination).offset() != 0) {
3374 __ Add64(dst, g.ToMemOperand(destination).rm(),
3375 g.ToMemOperand(destination).offset());
3376 }
3377 __ vs(src, dst, 0, E8);
3378 }
3379 } else {
3380 FPURegister src = g.ToDoubleRegister(source);
3381 if (destination->IsFPRegister()) {
3382 FPURegister dst = g.ToDoubleRegister(destination);
3383 __ Move(dst, src);
3384 } else {
3385 DCHECK(destination->IsFPStackSlot());
3386 if (rep == MachineRepresentation::kFloat32) {
3387 __ StoreFloat(src, g.ToMemOperand(destination));
3388 } else {
3389 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3390 __ StoreDouble(src, g.ToMemOperand(destination));
3391 }
3392 }
3393 }
3394 } else if (source->IsFPStackSlot()) {
3395 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
3396 MemOperand src = g.ToMemOperand(source);
3397 MachineRepresentation rep = LocationOperand::cast(source)->representation();
3398 if (rep == MachineRepresentation::kSimd128) {
3399 Register src_reg = src.offset() == 0 ? src.rm() : kScratchReg;
3400 if (src.offset() != 0) {
3401 __ Add64(src_reg, src.rm(), src.offset());
3402 }
3403 if (destination->IsSimd128Register()) {
3404 __ vl(g.ToSimd128Register(destination), src_reg, 0, E8);
3405 } else {
3406 DCHECK(destination->IsSimd128StackSlot());
3407 VRegister temp = kSimd128ScratchReg;
3408 Register dst = g.ToMemOperand(destination).offset() == 0
3409 ? g.ToMemOperand(destination).rm()
3410 : kScratchReg;
3411 if (g.ToMemOperand(destination).offset() != 0) {
3412 __ Add64(dst, g.ToMemOperand(destination).rm(),
3413 g.ToMemOperand(destination).offset());
3414 }
3415 __ vl(temp, src_reg, 0, E8);
3416 __ vs(temp, dst, 0, E8);
3417 }
3418 } else {
3419 if (destination->IsFPRegister()) {
3420 if (rep == MachineRepresentation::kFloat32) {
3421 __ LoadFloat(g.ToDoubleRegister(destination), src);
3422 } else {
3423 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3424 __ LoadDouble(g.ToDoubleRegister(destination), src);
3425 }
3426 } else {
3427 DCHECK(destination->IsFPStackSlot());
3428 FPURegister temp = kScratchDoubleReg;
3429 if (rep == MachineRepresentation::kFloat32) {
3430 __ LoadFloat(temp, src);
3431 __ StoreFloat(temp, g.ToMemOperand(destination));
3432 } else {
3433 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3434 __ LoadDouble(temp, src);
3435 __ StoreDouble(temp, g.ToMemOperand(destination));
3436 }
3437 }
3438 }
3439 } else {
3440 UNREACHABLE();
3441 }
3442 }
3443
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)3444 void CodeGenerator::AssembleSwap(InstructionOperand* source,
3445 InstructionOperand* destination) {
3446 RiscvOperandConverter g(this, nullptr);
3447 // Dispatch on the source and destination operand kinds. Not all
3448 // combinations are possible.
3449 if (source->IsRegister()) {
3450 // Register-register.
3451 Register temp = kScratchReg;
3452 Register src = g.ToRegister(source);
3453 if (destination->IsRegister()) {
3454 Register dst = g.ToRegister(destination);
3455 __ Move(temp, src);
3456 __ Move(src, dst);
3457 __ Move(dst, temp);
3458 } else {
3459 DCHECK(destination->IsStackSlot());
3460 MemOperand dst = g.ToMemOperand(destination);
3461 __ Move(temp, src);
3462 __ Ld(src, dst);
3463 __ Sd(temp, dst);
3464 }
3465 } else if (source->IsStackSlot()) {
3466 DCHECK(destination->IsStackSlot());
3467 Register temp_0 = kScratchReg;
3468 Register temp_1 = kScratchReg2;
3469 MemOperand src = g.ToMemOperand(source);
3470 MemOperand dst = g.ToMemOperand(destination);
3471 __ Ld(temp_0, src);
3472 __ Ld(temp_1, dst);
3473 __ Sd(temp_0, dst);
3474 __ Sd(temp_1, src);
3475 } else if (source->IsFPRegister()) {
3476 MachineRepresentation rep = LocationOperand::cast(source)->representation();
3477 if (rep == MachineRepresentation::kSimd128) {
3478 UNIMPLEMENTED();
3479 } else {
3480 FPURegister temp = kScratchDoubleReg;
3481 FPURegister src = g.ToDoubleRegister(source);
3482 if (destination->IsFPRegister()) {
3483 FPURegister dst = g.ToDoubleRegister(destination);
3484 __ Move(temp, src);
3485 __ Move(src, dst);
3486 __ Move(dst, temp);
3487 } else {
3488 DCHECK(destination->IsFPStackSlot());
3489 MemOperand dst = g.ToMemOperand(destination);
3490 if (rep == MachineRepresentation::kFloat32) {
3491 __ MoveFloat(temp, src);
3492 __ LoadFloat(src, dst);
3493 __ StoreFloat(temp, dst);
3494 } else {
3495 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3496 __ MoveDouble(temp, src);
3497 __ LoadDouble(src, dst);
3498 __ StoreDouble(temp, dst);
3499 }
3500 }
3501 }
3502 } else if (source->IsFPStackSlot()) {
3503 DCHECK(destination->IsFPStackSlot());
3504 Register temp_0 = kScratchReg;
3505 MemOperand src0 = g.ToMemOperand(source);
3506 MemOperand src1(src0.rm(), src0.offset() + kIntSize);
3507 MemOperand dst0 = g.ToMemOperand(destination);
3508 MemOperand dst1(dst0.rm(), dst0.offset() + kIntSize);
3509 MachineRepresentation rep = LocationOperand::cast(source)->representation();
3510 if (rep == MachineRepresentation::kSimd128) {
3511 UNIMPLEMENTED();
3512 } else {
3513 FPURegister temp_1 = kScratchDoubleReg;
3514 if (rep == MachineRepresentation::kFloat32) {
3515 __ LoadFloat(temp_1, dst0); // Save destination in temp_1.
3516 __ Lw(temp_0, src0); // Then use temp_0 to copy source to destination.
3517 __ Sw(temp_0, dst0);
3518 __ StoreFloat(temp_1, src0);
3519 } else {
3520 DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3521 __ LoadDouble(temp_1, dst0); // Save destination in temp_1.
3522 __ Lw(temp_0, src0); // Then use temp_0 to copy source to destination.
3523 __ Sw(temp_0, dst0);
3524 __ Lw(temp_0, src1);
3525 __ Sw(temp_0, dst1);
3526 __ StoreDouble(temp_1, src0);
3527 }
3528 }
3529 } else {
3530 // No other combinations are possible.
3531 UNREACHABLE();
3532 }
3533 }
3534
AssembleJumpTable(Label ** targets,size_t target_count)3535 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3536 // On 64-bit RISC-V we emit the jump tables inline.
3537 UNREACHABLE();
3538 }
3539
3540 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
3541 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
3542 #undef ASSEMBLE_ATOMIC_BINOP
3543 #undef ASSEMBLE_ATOMIC_BINOP_EXT
3544 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
3545 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT
3546 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
3547 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT
3548 #undef ASSEMBLE_IEEE754_BINOP
3549 #undef ASSEMBLE_IEEE754_UNOP
3550
3551 #undef TRACE_MSG
3552 #undef TRACE_UNIMPL
3553 #undef __
3554
3555 } // namespace compiler
3556 } // namespace internal
3557 } // namespace v8
3558