1 // Copyright 2021 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/codegen/assembler-inl.h"
6 #include "src/codegen/callable.h"
7 #include "src/codegen/macro-assembler.h"
8 #include "src/codegen/optimized-compilation-info.h"
9 #include "src/codegen/riscv64/constants-riscv64.h"
10 #include "src/compiler/backend/code-generator-impl.h"
11 #include "src/compiler/backend/code-generator.h"
12 #include "src/compiler/backend/gap-resolver.h"
13 #include "src/compiler/node-matchers.h"
14 #include "src/compiler/osr.h"
15 #include "src/heap/memory-chunk.h"
16 #include "src/wasm/wasm-code-manager.h"
17 
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21 
22 #define __ tasm()->
23 
24 // TODO(plind): consider renaming these macros.
25 #define TRACE_MSG(msg)                                                      \
26   PrintF("code_gen: \'%s\' in function %s at line %d\n", msg, __FUNCTION__, \
27          __LINE__)
28 
29 #define TRACE_UNIMPL()                                            \
30   PrintF("UNIMPLEMENTED code_generator_riscv64: %s at line %d\n", \
31          __FUNCTION__, __LINE__)
32 
33 // Adds RISC-V-specific methods to convert InstructionOperands.
34 class RiscvOperandConverter final : public InstructionOperandConverter {
35  public:
RiscvOperandConverter(CodeGenerator * gen,Instruction * instr)36   RiscvOperandConverter(CodeGenerator* gen, Instruction* instr)
37       : InstructionOperandConverter(gen, instr) {}
38 
OutputSingleRegister(size_t index=0)39   FloatRegister OutputSingleRegister(size_t index = 0) {
40     return ToSingleRegister(instr_->OutputAt(index));
41   }
42 
InputSingleRegister(size_t index)43   FloatRegister InputSingleRegister(size_t index) {
44     return ToSingleRegister(instr_->InputAt(index));
45   }
46 
ToSingleRegister(InstructionOperand * op)47   FloatRegister ToSingleRegister(InstructionOperand* op) {
48     // Single (Float) and Double register namespace is same on RISC-V,
49     // both are typedefs of FPURegister.
50     return ToDoubleRegister(op);
51   }
52 
InputOrZeroRegister(size_t index)53   Register InputOrZeroRegister(size_t index) {
54     if (instr_->InputAt(index)->IsImmediate()) {
55       Constant constant = ToConstant(instr_->InputAt(index));
56       switch (constant.type()) {
57         case Constant::kInt32:
58         case Constant::kInt64:
59           DCHECK_EQ(0, InputInt32(index));
60           break;
61         case Constant::kFloat32:
62           DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
63           break;
64         case Constant::kFloat64:
65           DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
66           break;
67         default:
68           UNREACHABLE();
69       }
70       return zero_reg;
71     }
72     return InputRegister(index);
73   }
74 
InputOrZeroDoubleRegister(size_t index)75   DoubleRegister InputOrZeroDoubleRegister(size_t index) {
76     if (instr_->InputAt(index)->IsImmediate()) return kDoubleRegZero;
77 
78     return InputDoubleRegister(index);
79   }
80 
InputOrZeroSingleRegister(size_t index)81   DoubleRegister InputOrZeroSingleRegister(size_t index) {
82     if (instr_->InputAt(index)->IsImmediate()) return kDoubleRegZero;
83 
84     return InputSingleRegister(index);
85   }
86 
InputImmediate(size_t index)87   Operand InputImmediate(size_t index) {
88     Constant constant = ToConstant(instr_->InputAt(index));
89     switch (constant.type()) {
90       case Constant::kInt32:
91         return Operand(constant.ToInt32());
92       case Constant::kInt64:
93         return Operand(constant.ToInt64());
94       case Constant::kFloat32:
95         return Operand::EmbeddedNumber(constant.ToFloat32());
96       case Constant::kFloat64:
97         return Operand::EmbeddedNumber(constant.ToFloat64().value());
98       case Constant::kExternalReference:
99       case Constant::kCompressedHeapObject:
100       case Constant::kHeapObject:
101         // TODO(plind): Maybe we should handle ExtRef & HeapObj here?
102         //    maybe not done on arm due to const pool ??
103         break;
104       case Constant::kDelayedStringConstant:
105         return Operand::EmbeddedStringConstant(
106             constant.ToDelayedStringConstant());
107       case Constant::kRpoNumber:
108         UNREACHABLE();  // TODO(titzer): RPO immediates
109     }
110     UNREACHABLE();
111   }
112 
InputOperand(size_t index)113   Operand InputOperand(size_t index) {
114     InstructionOperand* op = instr_->InputAt(index);
115     if (op->IsRegister()) {
116       return Operand(ToRegister(op));
117     }
118     return InputImmediate(index);
119   }
120 
MemoryOperand(size_t * first_index)121   MemOperand MemoryOperand(size_t* first_index) {
122     const size_t index = *first_index;
123     switch (AddressingModeField::decode(instr_->opcode())) {
124       case kMode_None:
125         break;
126       case kMode_MRI:
127         *first_index += 2;
128         return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
129       case kMode_MRR:
130         // TODO(plind): r6 address mode, to be implemented ...
131         UNREACHABLE();
132     }
133     UNREACHABLE();
134   }
135 
MemoryOperand(size_t index=0)136   MemOperand MemoryOperand(size_t index = 0) { return MemoryOperand(&index); }
137 
ToMemOperand(InstructionOperand * op) const138   MemOperand ToMemOperand(InstructionOperand* op) const {
139     DCHECK_NOT_NULL(op);
140     DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
141     return SlotToMemOperand(AllocatedOperand::cast(op)->index());
142   }
143 
SlotToMemOperand(int slot) const144   MemOperand SlotToMemOperand(int slot) const {
145     FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
146     return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
147   }
148 };
149 
HasRegisterInput(Instruction * instr,size_t index)150 static inline bool HasRegisterInput(Instruction* instr, size_t index) {
151   return instr->InputAt(index)->IsRegister();
152 }
153 namespace {
154 
155 class OutOfLineRecordWrite final : public OutOfLineCode {
156  public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Register index,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,StubCallMode stub_mode)157   OutOfLineRecordWrite(CodeGenerator* gen, Register object, Register index,
158                        Register value, Register scratch0, Register scratch1,
159                        RecordWriteMode mode, StubCallMode stub_mode)
160       : OutOfLineCode(gen),
161         object_(object),
162         index_(index),
163         value_(value),
164         scratch0_(scratch0),
165         scratch1_(scratch1),
166         mode_(mode),
167         stub_mode_(stub_mode),
168         must_save_lr_(!gen->frame_access_state()->has_frame()),
169         zone_(gen->zone()) {
170     DCHECK(!AreAliased(object, index, scratch0, scratch1));
171     DCHECK(!AreAliased(value, index, scratch0, scratch1));
172   }
173 
Generate()174   void Generate() final {
175     if (COMPRESS_POINTERS_BOOL) {
176       __ DecompressTaggedPointer(value_, value_);
177     }
178     __ CheckPageFlag(value_, scratch0_,
179                      MemoryChunk::kPointersToHereAreInterestingMask, eq,
180                      exit());
181     __ Add64(scratch1_, object_, index_);
182     RememberedSetAction const remembered_set_action =
183         mode_ > RecordWriteMode::kValueIsMap ? RememberedSetAction::kEmit
184                                              : RememberedSetAction::kOmit;
185     SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
186                                             ? SaveFPRegsMode::kSave
187                                             : SaveFPRegsMode::kIgnore;
188     if (must_save_lr_) {
189       // We need to save and restore ra if the frame was elided.
190       __ Push(ra);
191     }
192     if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
193       __ CallEphemeronKeyBarrier(object_, scratch1_, save_fp_mode);
194     } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
195       // A direct call to a wasm runtime stub defined in this module.
196       // Just encode the stub index. This will be patched when the code
197       // is added to the native module and copied into wasm code space.
198       __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
199                                           remembered_set_action, save_fp_mode,
200                                           StubCallMode::kCallWasmRuntimeStub);
201     } else {
202       __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
203                                           remembered_set_action, save_fp_mode);
204     }
205     if (must_save_lr_) {
206       __ Pop(ra);
207     }
208   }
209 
210  private:
211   Register const object_;
212   Register const index_;
213   Register const value_;
214   Register const scratch0_;
215   Register const scratch1_;
216   RecordWriteMode const mode_;
217   StubCallMode const stub_mode_;
218   bool must_save_lr_;
219   Zone* zone_;
220 };
221 
FlagsConditionToConditionCmp(FlagsCondition condition)222 Condition FlagsConditionToConditionCmp(FlagsCondition condition) {
223   switch (condition) {
224     case kEqual:
225       return eq;
226     case kNotEqual:
227       return ne;
228     case kSignedLessThan:
229       return lt;
230     case kSignedGreaterThanOrEqual:
231       return ge;
232     case kSignedLessThanOrEqual:
233       return le;
234     case kSignedGreaterThan:
235       return gt;
236     case kUnsignedLessThan:
237       return Uless;
238     case kUnsignedGreaterThanOrEqual:
239       return Ugreater_equal;
240     case kUnsignedLessThanOrEqual:
241       return Uless_equal;
242     case kUnsignedGreaterThan:
243       return Ugreater;
244     case kUnorderedEqual:
245     case kUnorderedNotEqual:
246       break;
247     default:
248       break;
249   }
250   UNREACHABLE();
251 }
252 
FlagsConditionToConditionTst(FlagsCondition condition)253 Condition FlagsConditionToConditionTst(FlagsCondition condition) {
254   switch (condition) {
255     case kNotEqual:
256       return ne;
257     case kEqual:
258       return eq;
259     default:
260       break;
261   }
262   UNREACHABLE();
263 }
264 
FlagsConditionToConditionOvf(FlagsCondition condition)265 Condition FlagsConditionToConditionOvf(FlagsCondition condition) {
266   switch (condition) {
267     case kOverflow:
268       return ne;
269     case kNotOverflow:
270       return eq;
271     default:
272       break;
273   }
274   UNREACHABLE();
275 }
276 
FlagsConditionToConditionCmpFPU(bool * predicate,FlagsCondition condition)277 FPUCondition FlagsConditionToConditionCmpFPU(bool* predicate,
278                                              FlagsCondition condition) {
279   switch (condition) {
280     case kEqual:
281       *predicate = true;
282       return EQ;
283     case kNotEqual:
284       *predicate = false;
285       return EQ;
286     case kUnsignedLessThan:
287       *predicate = true;
288       return LT;
289     case kUnsignedGreaterThanOrEqual:
290       *predicate = false;
291       return LT;
292     case kUnsignedLessThanOrEqual:
293       *predicate = true;
294       return LE;
295     case kUnsignedGreaterThan:
296       *predicate = false;
297       return LE;
298     case kUnorderedEqual:
299     case kUnorderedNotEqual:
300       *predicate = true;
301       break;
302     default:
303       *predicate = true;
304       break;
305   }
306   UNREACHABLE();
307 }
308 
309 }  // namespace
310 
311 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr)          \
312   do {                                                   \
313     __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
314     __ sync();                                           \
315   } while (0)
316 
317 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr)               \
318   do {                                                         \
319     __ sync();                                                 \
320     __ asm_instr(i.InputOrZeroRegister(2), i.MemoryOperand()); \
321     __ sync();                                                 \
322   } while (0)
323 
324 #define ASSEMBLE_ATOMIC_BINOP(load_linked, store_conditional, bin_instr)       \
325   do {                                                                         \
326     Label binop;                                                               \
327     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
328     __ sync();                                                                 \
329     __ bind(&binop);                                                           \
330     __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0));     \
331     __ bin_instr(i.TempRegister(1), i.OutputRegister(0),                       \
332                  Operand(i.InputRegister(2)));                                 \
333     __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
334     __ BranchShort(&binop, ne, i.TempRegister(1), Operand(zero_reg));          \
335     __ sync();                                                                 \
336   } while (0)
337 
338 #define ASSEMBLE_ATOMIC_BINOP_EXT(load_linked, store_conditional, sign_extend, \
339                                   size, bin_instr, representation)             \
340   do {                                                                         \
341     Label binop;                                                               \
342     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
343     if (representation == 32) {                                                \
344       __ And(i.TempRegister(3), i.TempRegister(0), 0x3);                       \
345     } else {                                                                   \
346       DCHECK_EQ(representation, 64);                                           \
347       __ And(i.TempRegister(3), i.TempRegister(0), 0x7);                       \
348     }                                                                          \
349     __ Sub64(i.TempRegister(0), i.TempRegister(0),                             \
350              Operand(i.TempRegister(3)));                                      \
351     __ Sll32(i.TempRegister(3), i.TempRegister(3), 3);                         \
352     __ sync();                                                                 \
353     __ bind(&binop);                                                           \
354     __ load_linked(i.TempRegister(1), MemOperand(i.TempRegister(0), 0));       \
355     __ ExtractBits(i.OutputRegister(0), i.TempRegister(1), i.TempRegister(3),  \
356                    size, sign_extend);                                         \
357     __ bin_instr(i.TempRegister(2), i.OutputRegister(0),                       \
358                  Operand(i.InputRegister(2)));                                 \
359     __ InsertBits(i.TempRegister(1), i.TempRegister(2), i.TempRegister(3),     \
360                   size);                                                       \
361     __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
362     __ BranchShort(&binop, ne, i.TempRegister(1), Operand(zero_reg));          \
363     __ sync();                                                                 \
364   } while (0)
365 
366 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_linked, store_conditional)       \
367   do {                                                                         \
368     Label exchange;                                                            \
369     __ sync();                                                                 \
370     __ bind(&exchange);                                                        \
371     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
372     __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0));     \
373     __ Move(i.TempRegister(1), i.InputRegister(2));                            \
374     __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \
375     __ BranchShort(&exchange, ne, i.TempRegister(1), Operand(zero_reg));       \
376     __ sync();                                                                 \
377   } while (0)
378 
379 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(                                  \
380     load_linked, store_conditional, sign_extend, size, representation)         \
381   do {                                                                         \
382     Label exchange;                                                            \
383     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
384     if (representation == 32) {                                                \
385       __ And(i.TempRegister(1), i.TempRegister(0), 0x3);                       \
386     } else {                                                                   \
387       DCHECK_EQ(representation, 64);                                           \
388       __ And(i.TempRegister(1), i.TempRegister(0), 0x7);                       \
389     }                                                                          \
390     __ Sub64(i.TempRegister(0), i.TempRegister(0),                             \
391              Operand(i.TempRegister(1)));                                      \
392     __ Sll32(i.TempRegister(1), i.TempRegister(1), 3);                         \
393     __ sync();                                                                 \
394     __ bind(&exchange);                                                        \
395     __ load_linked(i.TempRegister(2), MemOperand(i.TempRegister(0), 0));       \
396     __ ExtractBits(i.OutputRegister(0), i.TempRegister(2), i.TempRegister(1),  \
397                    size, sign_extend);                                         \
398     __ InsertBits(i.TempRegister(2), i.InputRegister(2), i.TempRegister(1),    \
399                   size);                                                       \
400     __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
401     __ BranchShort(&exchange, ne, i.TempRegister(2), Operand(zero_reg));       \
402     __ sync();                                                                 \
403   } while (0)
404 
405 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_linked,                  \
406                                                  store_conditional)            \
407   do {                                                                         \
408     Label compareExchange;                                                     \
409     Label exit;                                                                \
410     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
411     __ sync();                                                                 \
412     __ bind(&compareExchange);                                                 \
413     __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0));     \
414     __ BranchShort(&exit, ne, i.InputRegister(2),                              \
415                    Operand(i.OutputRegister(0)));                              \
416     __ Move(i.TempRegister(2), i.InputRegister(3));                            \
417     __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
418     __ BranchShort(&compareExchange, ne, i.TempRegister(2),                    \
419                    Operand(zero_reg));                                         \
420     __ bind(&exit);                                                            \
421     __ sync();                                                                 \
422   } while (0)
423 
424 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(                          \
425     load_linked, store_conditional, sign_extend, size, representation)         \
426   do {                                                                         \
427     Label compareExchange;                                                     \
428     Label exit;                                                                \
429     __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));       \
430     if (representation == 32) {                                                \
431       __ And(i.TempRegister(1), i.TempRegister(0), 0x3);                       \
432     } else {                                                                   \
433       DCHECK_EQ(representation, 64);                                           \
434       __ And(i.TempRegister(1), i.TempRegister(0), 0x7);                       \
435     }                                                                          \
436     __ Sub64(i.TempRegister(0), i.TempRegister(0),                             \
437              Operand(i.TempRegister(1)));                                      \
438     __ Sll32(i.TempRegister(1), i.TempRegister(1), 3);                         \
439     __ sync();                                                                 \
440     __ bind(&compareExchange);                                                 \
441     __ load_linked(i.TempRegister(2), MemOperand(i.TempRegister(0), 0));       \
442     __ ExtractBits(i.OutputRegister(0), i.TempRegister(2), i.TempRegister(1),  \
443                    size, sign_extend);                                         \
444     __ ExtractBits(i.InputRegister(2), i.InputRegister(2), 0, size,            \
445                    sign_extend);                                               \
446     __ BranchShort(&exit, ne, i.InputRegister(2),                              \
447                    Operand(i.OutputRegister(0)));                              \
448     __ InsertBits(i.TempRegister(2), i.InputRegister(3), i.TempRegister(1),    \
449                   size);                                                       \
450     __ store_conditional(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \
451     __ BranchShort(&compareExchange, ne, i.TempRegister(2),                    \
452                    Operand(zero_reg));                                         \
453     __ bind(&exit);                                                            \
454     __ sync();                                                                 \
455   } while (0)
456 
457 #define ASSEMBLE_IEEE754_BINOP(name)                                        \
458   do {                                                                      \
459     FrameScope scope(tasm(), StackFrame::MANUAL);                           \
460     __ PrepareCallCFunction(0, 2, kScratchReg);                             \
461     __ MovToFloatParameters(i.InputDoubleRegister(0),                       \
462                             i.InputDoubleRegister(1));                      \
463     __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
464     /* Move the result in the double result register. */                    \
465     __ MovFromFloatResult(i.OutputDoubleRegister());                        \
466   } while (0)
467 
468 #define ASSEMBLE_IEEE754_UNOP(name)                                         \
469   do {                                                                      \
470     FrameScope scope(tasm(), StackFrame::MANUAL);                           \
471     __ PrepareCallCFunction(0, 1, kScratchReg);                             \
472     __ MovToFloatParameter(i.InputDoubleRegister(0));                       \
473     __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
474     /* Move the result in the double result register. */                    \
475     __ MovFromFloatResult(i.OutputDoubleRegister());                        \
476   } while (0)
477 
478 #define ASSEMBLE_F64X2_ARITHMETIC_BINOP(op)                     \
479   do {                                                          \
480     __ op(i.OutputSimd128Register(), i.InputSimd128Register(0), \
481           i.InputSimd128Register(1));                           \
482   } while (0)
483 
AssembleDeconstructFrame()484 void CodeGenerator::AssembleDeconstructFrame() {
485   __ Move(sp, fp);
486   __ Pop(ra, fp);
487 }
488 
AssemblePrepareTailCall()489 void CodeGenerator::AssemblePrepareTailCall() {
490   if (frame_access_state()->has_frame()) {
491     __ Ld(ra, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
492     __ Ld(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
493   }
494   frame_access_state()->SetFrameAccessToSP();
495 }
496 
AssembleArchSelect(Instruction * instr,FlagsCondition condition)497 void CodeGenerator::AssembleArchSelect(Instruction* instr,
498                                        FlagsCondition condition) {
499   UNIMPLEMENTED();
500 }
501 
502 namespace {
503 
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)504 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
505                                    FrameAccessState* state,
506                                    int new_slot_above_sp,
507                                    bool allow_shrinkage = true) {
508   int current_sp_offset = state->GetSPToFPSlotCount() +
509                           StandardFrameConstants::kFixedSlotCountAboveFp;
510   int stack_slot_delta = new_slot_above_sp - current_sp_offset;
511   if (stack_slot_delta > 0) {
512     tasm->Sub64(sp, sp, stack_slot_delta * kSystemPointerSize);
513     state->IncreaseSPDelta(stack_slot_delta);
514   } else if (allow_shrinkage && stack_slot_delta < 0) {
515     tasm->Add64(sp, sp, -stack_slot_delta * kSystemPointerSize);
516     state->IncreaseSPDelta(stack_slot_delta);
517   }
518 }
519 
520 }  // namespace
521 
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_slot_offset)522 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
523                                               int first_unused_slot_offset) {
524   AdjustStackPointerForTailCall(tasm(), frame_access_state(),
525                                 first_unused_slot_offset, false);
526 }
527 
AssembleTailCallAfterGap(Instruction * instr,int first_unused_slot_offset)528 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
529                                              int first_unused_slot_offset) {
530   AdjustStackPointerForTailCall(tasm(), frame_access_state(),
531                                 first_unused_slot_offset);
532 }
533 
534 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()535 void CodeGenerator::AssembleCodeStartRegisterCheck() {
536   __ ComputeCodeStartAddress(kScratchReg);
537   __ Assert(eq, AbortReason::kWrongFunctionCodeStart,
538             kJavaScriptCallCodeStartRegister, Operand(kScratchReg));
539 }
540 
541 // Check if the code object is marked for deoptimization. If it is, then it
542 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
543 // to:
544 //    1. read from memory the word that contains that bit, which can be found in
545 //       the flags in the referenced {CodeDataContainer} object;
546 //    2. test kMarkedForDeoptimizationBit in those flags; and
547 //    3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()548 void CodeGenerator::BailoutIfDeoptimized() {
549   int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
550   __ LoadTaggedPointerField(
551       kScratchReg, MemOperand(kJavaScriptCallCodeStartRegister, offset));
552   __ Lw(kScratchReg,
553         FieldMemOperand(kScratchReg,
554                         CodeDataContainer::kKindSpecificFlagsOffset));
555   __ And(kScratchReg, kScratchReg,
556          Operand(1 << Code::kMarkedForDeoptimizationBit));
557   __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
558           RelocInfo::CODE_TARGET, ne, kScratchReg, Operand(zero_reg));
559 }
560 
561 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)562 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
563     Instruction* instr) {
564   RiscvOperandConverter i(this, instr);
565   InstructionCode opcode = instr->opcode();
566   ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
567   switch (arch_opcode) {
568     case kArchCallCodeObject: {
569       if (instr->InputAt(0)->IsImmediate()) {
570         __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
571       } else {
572         Register reg = i.InputRegister(0);
573         DCHECK_IMPLIES(
574             instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
575             reg == kJavaScriptCallCodeStartRegister);
576         __ CallCodeObject(reg);
577       }
578       RecordCallPosition(instr);
579       frame_access_state()->ClearSPDelta();
580       break;
581     }
582     case kArchCallBuiltinPointer: {
583       DCHECK(!instr->InputAt(0)->IsImmediate());
584       Register builtin_index = i.InputRegister(0);
585       __ CallBuiltinByIndex(builtin_index);
586       RecordCallPosition(instr);
587       frame_access_state()->ClearSPDelta();
588       break;
589     }
590     case kArchCallWasmFunction: {
591       if (instr->InputAt(0)->IsImmediate()) {
592         Constant constant = i.ToConstant(instr->InputAt(0));
593         Address wasm_code = static_cast<Address>(constant.ToInt64());
594         __ Call(wasm_code, constant.rmode());
595       } else {
596         __ Add64(t6, i.InputOrZeroRegister(0), 0);
597         __ Call(t6);
598       }
599       RecordCallPosition(instr);
600       frame_access_state()->ClearSPDelta();
601       break;
602     }
603     case kArchTailCallCodeObject: {
604       if (instr->InputAt(0)->IsImmediate()) {
605         __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
606       } else {
607         Register reg = i.InputOrZeroRegister(0);
608         DCHECK_IMPLIES(
609             instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
610             reg == kJavaScriptCallCodeStartRegister);
611         __ JumpCodeObject(reg);
612       }
613       frame_access_state()->ClearSPDelta();
614       frame_access_state()->SetFrameAccessToDefault();
615       break;
616     }
617     case kArchTailCallWasm: {
618       if (instr->InputAt(0)->IsImmediate()) {
619         Constant constant = i.ToConstant(instr->InputAt(0));
620         Address wasm_code = static_cast<Address>(constant.ToInt64());
621         __ Jump(wasm_code, constant.rmode());
622       } else {
623         __ Add64(kScratchReg, i.InputOrZeroRegister(0), 0);
624         __ Jump(kScratchReg);
625       }
626       frame_access_state()->ClearSPDelta();
627       frame_access_state()->SetFrameAccessToDefault();
628       break;
629     }
630     case kArchTailCallAddress: {
631       CHECK(!instr->InputAt(0)->IsImmediate());
632       Register reg = i.InputOrZeroRegister(0);
633       DCHECK_IMPLIES(
634           instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
635           reg == kJavaScriptCallCodeStartRegister);
636       __ Jump(reg);
637       frame_access_state()->ClearSPDelta();
638       frame_access_state()->SetFrameAccessToDefault();
639       break;
640     }
641     case kArchCallJSFunction: {
642       Register func = i.InputOrZeroRegister(0);
643       if (FLAG_debug_code) {
644         // Check the function's context matches the context argument.
645         __ LoadTaggedPointerField(
646             kScratchReg, FieldMemOperand(func, JSFunction::kContextOffset));
647         __ Assert(eq, AbortReason::kWrongFunctionContext, cp,
648                   Operand(kScratchReg));
649       }
650       static_assert(kJavaScriptCallCodeStartRegister == a2, "ABI mismatch");
651       __ LoadTaggedPointerField(a2,
652                                 FieldMemOperand(func, JSFunction::kCodeOffset));
653       __ CallCodeObject(a2);
654       RecordCallPosition(instr);
655       frame_access_state()->ClearSPDelta();
656       break;
657     }
658     case kArchPrepareCallCFunction: {
659       int const num_parameters = MiscField::decode(instr->opcode());
660       __ PrepareCallCFunction(num_parameters, kScratchReg);
661       // Frame alignment requires using FP-relative frame addressing.
662       frame_access_state()->SetFrameAccessToFP();
663       break;
664     }
665     case kArchSaveCallerRegisters: {
666       fp_mode_ =
667           static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
668       DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
669              fp_mode_ == SaveFPRegsMode::kSave);
670       // kReturnRegister0 should have been saved before entering the stub.
671       int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
672       DCHECK(IsAligned(bytes, kSystemPointerSize));
673       DCHECK_EQ(0, frame_access_state()->sp_delta());
674       frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
675       DCHECK(!caller_registers_saved_);
676       caller_registers_saved_ = true;
677       break;
678     }
679     case kArchRestoreCallerRegisters: {
680       DCHECK(fp_mode_ ==
681              static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
682       DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
683              fp_mode_ == SaveFPRegsMode::kSave);
684       // Don't overwrite the returned value.
685       int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
686       frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
687       DCHECK_EQ(0, frame_access_state()->sp_delta());
688       DCHECK(caller_registers_saved_);
689       caller_registers_saved_ = false;
690       break;
691     }
692     case kArchPrepareTailCall:
693       AssemblePrepareTailCall();
694       break;
695     case kArchCallCFunction: {
696       int const num_parameters = MiscField::decode(instr->opcode());
697       Label after_call;
698       bool isWasmCapiFunction =
699           linkage()->GetIncomingDescriptor()->IsWasmCapiFunction();
700       if (isWasmCapiFunction) {
701         // Put the return address in a stack slot.
702         __ LoadAddress(kScratchReg, &after_call, RelocInfo::EXTERNAL_REFERENCE);
703         __ Sd(kScratchReg,
704               MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
705       }
706       if (instr->InputAt(0)->IsImmediate()) {
707         ExternalReference ref = i.InputExternalReference(0);
708         __ CallCFunction(ref, num_parameters);
709       } else {
710         Register func = i.InputOrZeroRegister(0);
711         __ CallCFunction(func, num_parameters);
712       }
713       __ bind(&after_call);
714       if (isWasmCapiFunction) {
715         RecordSafepoint(instr->reference_map());
716       }
717 
718       frame_access_state()->SetFrameAccessToDefault();
719       // Ideally, we should decrement SP delta to match the change of stack
720       // pointer in CallCFunction. However, for certain architectures (e.g.
721       // ARM), there may be more strict alignment requirement, causing old SP
722       // to be saved on the stack. In those cases, we can not calculate the SP
723       // delta statically.
724       frame_access_state()->ClearSPDelta();
725       if (caller_registers_saved_) {
726         // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
727         // Here, we assume the sequence to be:
728         //   kArchSaveCallerRegisters;
729         //   kArchCallCFunction;
730         //   kArchRestoreCallerRegisters;
731         int bytes =
732             __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
733         frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
734       }
735       break;
736     }
737     case kArchJmp:
738       AssembleArchJump(i.InputRpo(0));
739       break;
740     case kArchBinarySearchSwitch:
741       AssembleArchBinarySearchSwitch(instr);
742       break;
743     case kArchTableSwitch:
744       AssembleArchTableSwitch(instr);
745       break;
746     case kArchAbortCSADcheck:
747       DCHECK(i.InputRegister(0) == a0);
748       {
749         // We don't actually want to generate a pile of code for this, so just
750         // claim there is a stack frame, without generating one.
751         FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
752         __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck),
753                 RelocInfo::CODE_TARGET);
754       }
755       __ stop();
756       break;
757     case kArchDebugBreak:
758       __ DebugBreak();
759       break;
760     case kArchComment:
761       __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
762       break;
763     case kArchNop:
764     case kArchThrowTerminator:
765       // don't emit code for nops.
766       break;
767     case kArchDeoptimize: {
768       DeoptimizationExit* exit =
769           BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
770       __ Branch(exit->label());
771       break;
772     }
773     case kArchRet:
774       AssembleReturn(instr->InputAt(0));
775       break;
776     case kArchStackPointerGreaterThan:
777       // Pseudo-instruction used for cmp/branch. No opcode emitted here.
778       break;
779     case kArchStackCheckOffset:
780       __ Move(i.OutputRegister(), Smi::FromInt(GetStackCheckOffset()));
781       break;
782     case kArchFramePointer:
783       __ Move(i.OutputRegister(), fp);
784       break;
785     case kArchParentFramePointer:
786       if (frame_access_state()->has_frame()) {
787         __ Ld(i.OutputRegister(), MemOperand(fp, 0));
788       } else {
789         __ Move(i.OutputRegister(), fp);
790       }
791       break;
792     case kArchTruncateDoubleToI:
793       __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
794                            i.InputDoubleRegister(0), DetermineStubCallMode());
795       break;
796     case kArchStoreWithWriteBarrier: {
797       RecordWriteMode mode =
798           static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
799       Register object = i.InputRegister(0);
800       Register index = i.InputRegister(1);
801       Register value = i.InputRegister(2);
802       Register scratch0 = i.TempRegister(0);
803       Register scratch1 = i.TempRegister(1);
804       auto ool = zone()->New<OutOfLineRecordWrite>(this, object, index, value,
805                                                    scratch0, scratch1, mode,
806                                                    DetermineStubCallMode());
807       __ Add64(kScratchReg, object, index);
808       __ StoreTaggedField(value, MemOperand(kScratchReg));
809       if (mode > RecordWriteMode::kValueIsPointer) {
810         __ JumpIfSmi(value, ool->exit());
811       }
812       __ CheckPageFlag(object, scratch0,
813                        MemoryChunk::kPointersFromHereAreInterestingMask, ne,
814                        ool->entry());
815       __ bind(ool->exit());
816       break;
817     }
818     case kArchStackSlot: {
819       FrameOffset offset =
820           frame_access_state()->GetFrameOffset(i.InputInt32(0));
821       Register base_reg = offset.from_stack_pointer() ? sp : fp;
822       __ Add64(i.OutputRegister(), base_reg, Operand(offset.offset()));
823       int alignment = i.InputInt32(1);
824       DCHECK(alignment == 0 || alignment == 4 || alignment == 8 ||
825              alignment == 16);
826       if (FLAG_debug_code && alignment > 0) {
827         // Verify that the output_register is properly aligned
828         __ And(kScratchReg, i.OutputRegister(),
829                Operand(kSystemPointerSize - 1));
830         __ Assert(eq, AbortReason::kAllocationIsNotDoubleAligned, kScratchReg,
831                   Operand(zero_reg));
832       }
833       if (alignment == 2 * kSystemPointerSize) {
834         Label done;
835         __ Add64(kScratchReg, base_reg, Operand(offset.offset()));
836         __ And(kScratchReg, kScratchReg, Operand(alignment - 1));
837         __ BranchShort(&done, eq, kScratchReg, Operand(zero_reg));
838         __ Add64(i.OutputRegister(), i.OutputRegister(), kSystemPointerSize);
839         __ bind(&done);
840       } else if (alignment > 2 * kSystemPointerSize) {
841         Label done;
842         __ Add64(kScratchReg, base_reg, Operand(offset.offset()));
843         __ And(kScratchReg, kScratchReg, Operand(alignment - 1));
844         __ BranchShort(&done, eq, kScratchReg, Operand(zero_reg));
845         __ li(kScratchReg2, alignment);
846         __ Sub64(kScratchReg2, kScratchReg2, Operand(kScratchReg));
847         __ Add64(i.OutputRegister(), i.OutputRegister(), kScratchReg2);
848         __ bind(&done);
849       }
850 
851       break;
852     }
853     case kIeee754Float64Acos:
854       ASSEMBLE_IEEE754_UNOP(acos);
855       break;
856     case kIeee754Float64Acosh:
857       ASSEMBLE_IEEE754_UNOP(acosh);
858       break;
859     case kIeee754Float64Asin:
860       ASSEMBLE_IEEE754_UNOP(asin);
861       break;
862     case kIeee754Float64Asinh:
863       ASSEMBLE_IEEE754_UNOP(asinh);
864       break;
865     case kIeee754Float64Atan:
866       ASSEMBLE_IEEE754_UNOP(atan);
867       break;
868     case kIeee754Float64Atanh:
869       ASSEMBLE_IEEE754_UNOP(atanh);
870       break;
871     case kIeee754Float64Atan2:
872       ASSEMBLE_IEEE754_BINOP(atan2);
873       break;
874     case kIeee754Float64Cos:
875       ASSEMBLE_IEEE754_UNOP(cos);
876       break;
877     case kIeee754Float64Cosh:
878       ASSEMBLE_IEEE754_UNOP(cosh);
879       break;
880     case kIeee754Float64Cbrt:
881       ASSEMBLE_IEEE754_UNOP(cbrt);
882       break;
883     case kIeee754Float64Exp:
884       ASSEMBLE_IEEE754_UNOP(exp);
885       break;
886     case kIeee754Float64Expm1:
887       ASSEMBLE_IEEE754_UNOP(expm1);
888       break;
889     case kIeee754Float64Log:
890       ASSEMBLE_IEEE754_UNOP(log);
891       break;
892     case kIeee754Float64Log1p:
893       ASSEMBLE_IEEE754_UNOP(log1p);
894       break;
895     case kIeee754Float64Log2:
896       ASSEMBLE_IEEE754_UNOP(log2);
897       break;
898     case kIeee754Float64Log10:
899       ASSEMBLE_IEEE754_UNOP(log10);
900       break;
901     case kIeee754Float64Pow:
902       ASSEMBLE_IEEE754_BINOP(pow);
903       break;
904     case kIeee754Float64Sin:
905       ASSEMBLE_IEEE754_UNOP(sin);
906       break;
907     case kIeee754Float64Sinh:
908       ASSEMBLE_IEEE754_UNOP(sinh);
909       break;
910     case kIeee754Float64Tan:
911       ASSEMBLE_IEEE754_UNOP(tan);
912       break;
913     case kIeee754Float64Tanh:
914       ASSEMBLE_IEEE754_UNOP(tanh);
915       break;
916     case kRiscvAdd32:
917       __ Add32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
918       break;
919     case kRiscvAdd64:
920       __ Add64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
921       break;
922     case kRiscvAddOvf64:
923       __ AddOverflow64(i.OutputRegister(), i.InputOrZeroRegister(0),
924                        i.InputOperand(1), kScratchReg);
925       break;
926     case kRiscvSub32:
927       __ Sub32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
928       break;
929     case kRiscvSub64:
930       __ Sub64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
931       break;
932     case kRiscvSubOvf64:
933       __ SubOverflow64(i.OutputRegister(), i.InputOrZeroRegister(0),
934                        i.InputOperand(1), kScratchReg);
935       break;
936     case kRiscvMul32:
937       __ Mul32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
938       break;
939     case kRiscvMulOvf32:
940       __ MulOverflow32(i.OutputRegister(), i.InputOrZeroRegister(0),
941                        i.InputOperand(1), kScratchReg);
942       break;
943     case kRiscvMulHigh32:
944       __ Mulh32(i.OutputRegister(), i.InputOrZeroRegister(0),
945                 i.InputOperand(1));
946       break;
947     case kRiscvMulHighU32:
948       __ Mulhu32(i.OutputRegister(), i.InputOrZeroRegister(0),
949                  i.InputOperand(1), kScratchReg, kScratchReg2);
950       break;
951     case kRiscvMulHigh64:
952       __ Mulh64(i.OutputRegister(), i.InputOrZeroRegister(0),
953                 i.InputOperand(1));
954       break;
955     case kRiscvDiv32: {
956       __ Div32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
957       // Set ouput to zero if divisor == 0
958       __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
959       break;
960     }
961     case kRiscvDivU32: {
962       __ Divu32(i.OutputRegister(), i.InputOrZeroRegister(0),
963                 i.InputOperand(1));
964       // Set ouput to zero if divisor == 0
965       __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
966       break;
967     }
968     case kRiscvMod32:
969       __ Mod32(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
970       break;
971     case kRiscvModU32:
972       __ Modu32(i.OutputRegister(), i.InputOrZeroRegister(0),
973                 i.InputOperand(1));
974       break;
975     case kRiscvMul64:
976       __ Mul64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
977       break;
978     case kRiscvDiv64: {
979       __ Div64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
980       // Set ouput to zero if divisor == 0
981       __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
982       break;
983     }
984     case kRiscvDivU64: {
985       __ Divu64(i.OutputRegister(), i.InputOrZeroRegister(0),
986                 i.InputOperand(1));
987       // Set ouput to zero if divisor == 0
988       __ LoadZeroIfConditionZero(i.OutputRegister(), i.InputRegister(1));
989       break;
990     }
991     case kRiscvMod64:
992       __ Mod64(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
993       break;
994     case kRiscvModU64:
995       __ Modu64(i.OutputRegister(), i.InputOrZeroRegister(0),
996                 i.InputOperand(1));
997       break;
998     case kRiscvAnd:
999       __ And(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1000       break;
1001     case kRiscvAnd32:
1002       __ And(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1003       __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1004       break;
1005     case kRiscvOr:
1006       __ Or(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1007       break;
1008     case kRiscvOr32:
1009       __ Or(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1010       __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1011       break;
1012     case kRiscvNor:
1013       if (instr->InputAt(1)->IsRegister()) {
1014         __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1015       } else {
1016         DCHECK_EQ(0, i.InputOperand(1).immediate());
1017         __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), zero_reg);
1018       }
1019       break;
1020     case kRiscvNor32:
1021       if (instr->InputAt(1)->IsRegister()) {
1022         __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1023         __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1024       } else {
1025         DCHECK_EQ(0, i.InputOperand(1).immediate());
1026         __ Nor(i.OutputRegister(), i.InputOrZeroRegister(0), zero_reg);
1027         __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1028       }
1029       break;
1030     case kRiscvXor:
1031       __ Xor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1032       break;
1033     case kRiscvXor32:
1034       __ Xor(i.OutputRegister(), i.InputOrZeroRegister(0), i.InputOperand(1));
1035       __ Sll32(i.OutputRegister(), i.OutputRegister(), 0x0);
1036       break;
1037     case kRiscvClz32:
1038       __ Clz32(i.OutputRegister(), i.InputOrZeroRegister(0));
1039       break;
1040     case kRiscvClz64:
1041       __ Clz64(i.OutputRegister(), i.InputOrZeroRegister(0));
1042       break;
1043     case kRiscvCtz32: {
1044       Register src = i.InputRegister(0);
1045       Register dst = i.OutputRegister();
1046       __ Ctz32(dst, src);
1047     } break;
1048     case kRiscvCtz64: {
1049       Register src = i.InputRegister(0);
1050       Register dst = i.OutputRegister();
1051       __ Ctz64(dst, src);
1052     } break;
1053     case kRiscvPopcnt32: {
1054       Register src = i.InputRegister(0);
1055       Register dst = i.OutputRegister();
1056       __ Popcnt32(dst, src, kScratchReg);
1057     } break;
1058     case kRiscvPopcnt64: {
1059       Register src = i.InputRegister(0);
1060       Register dst = i.OutputRegister();
1061       __ Popcnt64(dst, src, kScratchReg);
1062     } break;
1063     case kRiscvShl32:
1064       if (instr->InputAt(1)->IsRegister()) {
1065         __ Sll32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1066       } else {
1067         int64_t imm = i.InputOperand(1).immediate();
1068         __ Sll32(i.OutputRegister(), i.InputRegister(0),
1069                  static_cast<uint16_t>(imm));
1070       }
1071       break;
1072     case kRiscvShr32:
1073       if (instr->InputAt(1)->IsRegister()) {
1074         __ Srl32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1075       } else {
1076         int64_t imm = i.InputOperand(1).immediate();
1077         __ Srl32(i.OutputRegister(), i.InputRegister(0),
1078                  static_cast<uint16_t>(imm));
1079       }
1080       break;
1081     case kRiscvSar32:
1082       if (instr->InputAt(1)->IsRegister()) {
1083         __ Sra32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1084       } else {
1085         int64_t imm = i.InputOperand(1).immediate();
1086         __ Sra32(i.OutputRegister(), i.InputRegister(0),
1087                  static_cast<uint16_t>(imm));
1088       }
1089       break;
1090     case kRiscvZeroExtendWord: {
1091       __ ZeroExtendWord(i.OutputRegister(), i.InputRegister(0));
1092       break;
1093     }
1094     case kRiscvSignExtendWord: {
1095       __ SignExtendWord(i.OutputRegister(), i.InputRegister(0));
1096       break;
1097     }
1098     case kRiscvShl64:
1099       __ Sll64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1100       break;
1101     case kRiscvShr64:
1102       __ Srl64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1103       break;
1104     case kRiscvSar64:
1105       __ Sra64(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1106       break;
1107     case kRiscvRor32:
1108       __ Ror(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1109       break;
1110     case kRiscvRor64:
1111       __ Dror(i.OutputRegister(), i.InputRegister(0), i.InputOperand(1));
1112       break;
1113     case kRiscvTst:
1114       __ And(kScratchReg, i.InputRegister(0), i.InputOperand(1));
1115       // Pseudo-instruction used for cmp/branch. No opcode emitted here.
1116       break;
1117     case kRiscvCmp:
1118       // Pseudo-instruction used for cmp/branch. No opcode emitted here.
1119       break;
1120     case kRiscvCmpZero:
1121       // Pseudo-instruction used for cmpzero/branch. No opcode emitted here.
1122       break;
1123     case kRiscvMov:
1124       // TODO(plind): Should we combine mov/li like this, or use separate instr?
1125       //    - Also see x64 ASSEMBLE_BINOP & RegisterOrOperandType
1126       if (HasRegisterInput(instr, 0)) {
1127         __ Move(i.OutputRegister(), i.InputRegister(0));
1128       } else {
1129         __ li(i.OutputRegister(), i.InputOperand(0));
1130       }
1131       break;
1132 
1133     case kRiscvCmpS: {
1134       FPURegister left = i.InputOrZeroSingleRegister(0);
1135       FPURegister right = i.InputOrZeroSingleRegister(1);
1136       bool predicate;
1137       FPUCondition cc =
1138           FlagsConditionToConditionCmpFPU(&predicate, instr->flags_condition());
1139 
1140       if ((left == kDoubleRegZero || right == kDoubleRegZero) &&
1141           !__ IsSingleZeroRegSet()) {
1142         __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1143       }
1144       // compare result set to kScratchReg
1145       __ CompareF32(kScratchReg, cc, left, right);
1146     } break;
1147     case kRiscvAddS:
1148       // TODO(plind): add special case: combine mult & add.
1149       __ fadd_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1150                 i.InputDoubleRegister(1));
1151       break;
1152     case kRiscvSubS:
1153       __ fsub_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1154                 i.InputDoubleRegister(1));
1155       break;
1156     case kRiscvMulS:
1157       // TODO(plind): add special case: right op is -1.0, see arm port.
1158       __ fmul_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1159                 i.InputDoubleRegister(1));
1160       break;
1161     case kRiscvDivS:
1162       __ fdiv_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1163                 i.InputDoubleRegister(1));
1164       break;
1165     case kRiscvModS: {
1166       // TODO(bmeurer): We should really get rid of this special instruction,
1167       // and generate a CallAddress instruction instead.
1168       FrameScope scope(tasm(), StackFrame::MANUAL);
1169       __ PrepareCallCFunction(0, 2, kScratchReg);
1170       __ MovToFloatParameters(i.InputDoubleRegister(0),
1171                               i.InputDoubleRegister(1));
1172       // TODO(balazs.kilvady): implement mod_two_floats_operation(isolate())
1173       __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1174       // Move the result in the double result register.
1175       __ MovFromFloatResult(i.OutputSingleRegister());
1176       break;
1177     }
1178     case kRiscvAbsS:
1179       __ fabs_s(i.OutputSingleRegister(), i.InputSingleRegister(0));
1180       break;
1181     case kRiscvNegS:
1182       __ Neg_s(i.OutputSingleRegister(), i.InputSingleRegister(0));
1183       break;
1184     case kRiscvSqrtS: {
1185       __ fsqrt_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1186       break;
1187     }
1188     case kRiscvMaxS:
1189       __ fmax_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1190                 i.InputDoubleRegister(1));
1191       break;
1192     case kRiscvMinS:
1193       __ fmin_s(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1194                 i.InputDoubleRegister(1));
1195       break;
1196     case kRiscvCmpD: {
1197       FPURegister left = i.InputOrZeroDoubleRegister(0);
1198       FPURegister right = i.InputOrZeroDoubleRegister(1);
1199       bool predicate;
1200       FPUCondition cc =
1201           FlagsConditionToConditionCmpFPU(&predicate, instr->flags_condition());
1202       if ((left == kDoubleRegZero || right == kDoubleRegZero) &&
1203           !__ IsDoubleZeroRegSet()) {
1204         __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1205       }
1206       // compare result set to kScratchReg
1207       __ CompareF64(kScratchReg, cc, left, right);
1208     } break;
1209     case kRiscvAddD:
1210       // TODO(plind): add special case: combine mult & add.
1211       __ fadd_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1212                 i.InputDoubleRegister(1));
1213       break;
1214     case kRiscvSubD:
1215       __ fsub_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1216                 i.InputDoubleRegister(1));
1217       break;
1218     case kRiscvMulD:
1219       // TODO(plind): add special case: right op is -1.0, see arm port.
1220       __ fmul_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1221                 i.InputDoubleRegister(1));
1222       break;
1223     case kRiscvDivD:
1224       __ fdiv_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1225                 i.InputDoubleRegister(1));
1226       break;
1227     case kRiscvModD: {
1228       // TODO(bmeurer): We should really get rid of this special instruction,
1229       // and generate a CallAddress instruction instead.
1230       FrameScope scope(tasm(), StackFrame::MANUAL);
1231       __ PrepareCallCFunction(0, 2, kScratchReg);
1232       __ MovToFloatParameters(i.InputDoubleRegister(0),
1233                               i.InputDoubleRegister(1));
1234       __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1235       // Move the result in the double result register.
1236       __ MovFromFloatResult(i.OutputDoubleRegister());
1237       break;
1238     }
1239     case kRiscvAbsD:
1240       __ fabs_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1241       break;
1242     case kRiscvNegD:
1243       __ Neg_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1244       break;
1245     case kRiscvSqrtD: {
1246       __ fsqrt_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1247       break;
1248     }
1249     case kRiscvMaxD:
1250       __ fmax_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1251                 i.InputDoubleRegister(1));
1252       break;
1253     case kRiscvMinD:
1254       __ fmin_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1255                 i.InputDoubleRegister(1));
1256       break;
1257     case kRiscvFloat64RoundDown: {
1258       __ Floor_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1259                    kScratchDoubleReg);
1260       break;
1261     }
1262     case kRiscvFloat32RoundDown: {
1263       __ Floor_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1264                    kScratchDoubleReg);
1265       break;
1266     }
1267     case kRiscvFloat64RoundTruncate: {
1268       __ Trunc_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1269                    kScratchDoubleReg);
1270       break;
1271     }
1272     case kRiscvFloat32RoundTruncate: {
1273       __ Trunc_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1274                    kScratchDoubleReg);
1275       break;
1276     }
1277     case kRiscvFloat64RoundUp: {
1278       __ Ceil_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1279                   kScratchDoubleReg);
1280       break;
1281     }
1282     case kRiscvFloat32RoundUp: {
1283       __ Ceil_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1284                   kScratchDoubleReg);
1285       break;
1286     }
1287     case kRiscvFloat64RoundTiesEven: {
1288       __ Round_d_d(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1289                    kScratchDoubleReg);
1290       break;
1291     }
1292     case kRiscvFloat32RoundTiesEven: {
1293       __ Round_s_s(i.OutputSingleRegister(), i.InputSingleRegister(0),
1294                    kScratchDoubleReg);
1295       break;
1296     }
1297     case kRiscvFloat32Max: {
1298       __ Float32Max(i.OutputSingleRegister(), i.InputSingleRegister(0),
1299                     i.InputSingleRegister(1));
1300       break;
1301     }
1302     case kRiscvFloat64Max: {
1303       __ Float64Max(i.OutputSingleRegister(), i.InputSingleRegister(0),
1304                     i.InputSingleRegister(1));
1305       break;
1306     }
1307     case kRiscvFloat32Min: {
1308       __ Float32Min(i.OutputSingleRegister(), i.InputSingleRegister(0),
1309                     i.InputSingleRegister(1));
1310       break;
1311     }
1312     case kRiscvFloat64Min: {
1313       __ Float64Min(i.OutputSingleRegister(), i.InputSingleRegister(0),
1314                     i.InputSingleRegister(1));
1315       break;
1316     }
1317     case kRiscvFloat64SilenceNaN:
1318       __ FPUCanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1319       break;
1320     case kRiscvCvtSD:
1321       __ fcvt_s_d(i.OutputSingleRegister(), i.InputDoubleRegister(0));
1322       break;
1323     case kRiscvCvtDS:
1324       __ fcvt_d_s(i.OutputDoubleRegister(), i.InputSingleRegister(0));
1325       break;
1326     case kRiscvCvtDW: {
1327       __ fcvt_d_w(i.OutputDoubleRegister(), i.InputRegister(0));
1328       break;
1329     }
1330     case kRiscvCvtSW: {
1331       __ fcvt_s_w(i.OutputDoubleRegister(), i.InputRegister(0));
1332       break;
1333     }
1334     case kRiscvCvtSUw: {
1335       __ Cvt_s_uw(i.OutputDoubleRegister(), i.InputRegister(0));
1336       break;
1337     }
1338     case kRiscvCvtSL: {
1339       __ fcvt_s_l(i.OutputDoubleRegister(), i.InputRegister(0));
1340       break;
1341     }
1342     case kRiscvCvtDL: {
1343       __ fcvt_d_l(i.OutputDoubleRegister(), i.InputRegister(0));
1344       break;
1345     }
1346     case kRiscvCvtDUw: {
1347       __ Cvt_d_uw(i.OutputDoubleRegister(), i.InputRegister(0));
1348       break;
1349     }
1350     case kRiscvCvtDUl: {
1351       __ Cvt_d_ul(i.OutputDoubleRegister(), i.InputRegister(0));
1352       break;
1353     }
1354     case kRiscvCvtSUl: {
1355       __ Cvt_s_ul(i.OutputDoubleRegister(), i.InputRegister(0));
1356       break;
1357     }
1358     case kRiscvFloorWD: {
1359       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1360       __ Floor_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1361       break;
1362     }
1363     case kRiscvCeilWD: {
1364       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1365       __ Ceil_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1366       break;
1367     }
1368     case kRiscvRoundWD: {
1369       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1370       __ Round_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1371       break;
1372     }
1373     case kRiscvTruncWD: {
1374       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1375       __ Trunc_w_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1376       break;
1377     }
1378     case kRiscvFloorWS: {
1379       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1380       __ Floor_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1381       break;
1382     }
1383     case kRiscvCeilWS: {
1384       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1385       __ Ceil_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1386       break;
1387     }
1388     case kRiscvRoundWS: {
1389       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1390       __ Round_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1391       break;
1392     }
1393     case kRiscvTruncWS: {
1394       Label done;
1395       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1396       bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
1397       __ Trunc_w_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1398 
1399       // On RISCV, if the input value exceeds INT32_MAX, the result of fcvt
1400       // is INT32_MAX. Note that, since INT32_MAX means the lower 31-bits are
1401       // all 1s, INT32_MAX cannot be represented precisely as a float, so an
1402       // fcvt result of INT32_MAX always indicate overflow.
1403       //
1404       // In wasm_compiler, to detect overflow in converting a FP value, fval, to
1405       // integer, V8 checks whether I2F(F2I(fval)) equals fval. However, if fval
1406       // == INT32_MAX+1, the value of I2F(F2I(fval)) happens to be fval. So,
1407       // INT32_MAX is not a good value to indicate overflow. Instead, we will
1408       // use INT32_MIN as the converted result of an out-of-range FP value,
1409       // exploiting the fact that INT32_MAX+1 is INT32_MIN.
1410       //
1411       // If the result of conversion overflow, the result will be set to
1412       // INT32_MIN. Here we detect overflow by testing whether output + 1 <
1413       // output (i.e., kScratchReg  < output)
1414       if (set_overflow_to_min_i32) {
1415         __ Add32(kScratchReg, i.OutputRegister(), 1);
1416         __ BranchShort(&done, lt, i.OutputRegister(), Operand(kScratchReg));
1417         __ Move(i.OutputRegister(), kScratchReg);
1418         __ bind(&done);
1419       }
1420       break;
1421     }
1422     case kRiscvTruncLS: {
1423       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1424       __ Trunc_l_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1425       break;
1426     }
1427     case kRiscvTruncLD: {
1428       Label done;
1429       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1430       bool set_overflow_to_min_i64 = MiscField::decode(instr->opcode());
1431       __ Trunc_l_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1432       if (set_overflow_to_min_i64) {
1433         __ Add64(kScratchReg, i.OutputRegister(), 1);
1434         __ BranchShort(&done, lt, i.OutputRegister(), Operand(kScratchReg));
1435         __ Move(i.OutputRegister(), kScratchReg);
1436         __ bind(&done);
1437       }
1438       break;
1439     }
1440     case kRiscvTruncUwD: {
1441       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1442       __ Trunc_uw_d(i.OutputRegister(), i.InputDoubleRegister(0), result);
1443       break;
1444     }
1445     case kRiscvTruncUwS: {
1446       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1447       bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
1448       __ Trunc_uw_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1449 
1450       // On RISCV, if the input value exceeds UINT32_MAX, the result of fcvt
1451       // is UINT32_MAX. Note that, since UINT32_MAX means all 32-bits are 1s,
1452       // UINT32_MAX cannot be represented precisely as float, so an fcvt result
1453       // of UINT32_MAX always indicates overflow.
1454       //
1455       // In wasm_compiler.cc, to detect overflow in converting a FP value, fval,
1456       // to integer, V8 checks whether I2F(F2I(fval)) equals fval. However, if
1457       // fval == UINT32_MAX+1, the value of I2F(F2I(fval)) happens to be fval.
1458       // So, UINT32_MAX is not a good value to indicate overflow. Instead, we
1459       // will use 0 as the converted result of an out-of-range FP value,
1460       // exploiting the fact that UINT32_MAX+1 is 0.
1461       if (set_overflow_to_min_u32) {
1462         __ Add32(kScratchReg, i.OutputRegister(), 1);
1463         // Set ouput to zero if result overflows (i.e., UINT32_MAX)
1464         __ LoadZeroIfConditionZero(i.OutputRegister(), kScratchReg);
1465       }
1466       break;
1467     }
1468     case kRiscvTruncUlS: {
1469       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1470       __ Trunc_ul_s(i.OutputRegister(), i.InputDoubleRegister(0), result);
1471       break;
1472     }
1473     case kRiscvTruncUlD: {
1474       Register result = instr->OutputCount() > 1 ? i.OutputRegister(1) : no_reg;
1475       __ Trunc_ul_d(i.OutputRegister(0), i.InputDoubleRegister(0), result);
1476       break;
1477     }
1478     case kRiscvBitcastDL:
1479       __ fmv_x_d(i.OutputRegister(), i.InputDoubleRegister(0));
1480       break;
1481     case kRiscvBitcastLD:
1482       __ fmv_d_x(i.OutputDoubleRegister(), i.InputRegister(0));
1483       break;
1484     case kRiscvBitcastInt32ToFloat32:
1485       __ fmv_w_x(i.OutputDoubleRegister(), i.InputRegister(0));
1486       break;
1487     case kRiscvBitcastFloat32ToInt32:
1488       __ fmv_x_w(i.OutputRegister(), i.InputDoubleRegister(0));
1489       break;
1490     case kRiscvFloat64ExtractLowWord32:
1491       __ ExtractLowWordFromF64(i.OutputRegister(), i.InputDoubleRegister(0));
1492       break;
1493     case kRiscvFloat64ExtractHighWord32:
1494       __ ExtractHighWordFromF64(i.OutputRegister(), i.InputDoubleRegister(0));
1495       break;
1496     case kRiscvFloat64InsertLowWord32:
1497       __ InsertLowWordF64(i.OutputDoubleRegister(), i.InputRegister(1));
1498       break;
1499     case kRiscvFloat64InsertHighWord32:
1500       __ InsertHighWordF64(i.OutputDoubleRegister(), i.InputRegister(1));
1501       break;
1502       // ... more basic instructions ...
1503 
1504     case kRiscvSignExtendByte:
1505       __ SignExtendByte(i.OutputRegister(), i.InputRegister(0));
1506       break;
1507     case kRiscvSignExtendShort:
1508       __ SignExtendShort(i.OutputRegister(), i.InputRegister(0));
1509       break;
1510     case kRiscvLbu:
1511       __ Lbu(i.OutputRegister(), i.MemoryOperand());
1512       break;
1513     case kRiscvLb:
1514       __ Lb(i.OutputRegister(), i.MemoryOperand());
1515       break;
1516     case kRiscvSb:
1517       __ Sb(i.InputOrZeroRegister(2), i.MemoryOperand());
1518       break;
1519     case kRiscvLhu:
1520       __ Lhu(i.OutputRegister(), i.MemoryOperand());
1521       break;
1522     case kRiscvUlhu:
1523       __ Ulhu(i.OutputRegister(), i.MemoryOperand());
1524       break;
1525     case kRiscvLh:
1526       __ Lh(i.OutputRegister(), i.MemoryOperand());
1527       break;
1528     case kRiscvUlh:
1529       __ Ulh(i.OutputRegister(), i.MemoryOperand());
1530       break;
1531     case kRiscvSh:
1532       __ Sh(i.InputOrZeroRegister(2), i.MemoryOperand());
1533       break;
1534     case kRiscvUsh:
1535       __ Ush(i.InputOrZeroRegister(2), i.MemoryOperand());
1536       break;
1537     case kRiscvLw:
1538       __ Lw(i.OutputRegister(), i.MemoryOperand());
1539       break;
1540     case kRiscvUlw:
1541       __ Ulw(i.OutputRegister(), i.MemoryOperand());
1542       break;
1543     case kRiscvLwu:
1544       __ Lwu(i.OutputRegister(), i.MemoryOperand());
1545       break;
1546     case kRiscvUlwu:
1547       __ Ulwu(i.OutputRegister(), i.MemoryOperand());
1548       break;
1549     case kRiscvLd:
1550       __ Ld(i.OutputRegister(), i.MemoryOperand());
1551       break;
1552     case kRiscvUld:
1553       __ Uld(i.OutputRegister(), i.MemoryOperand());
1554       break;
1555     case kRiscvSw:
1556       __ Sw(i.InputOrZeroRegister(2), i.MemoryOperand());
1557       break;
1558     case kRiscvUsw:
1559       __ Usw(i.InputOrZeroRegister(2), i.MemoryOperand());
1560       break;
1561     case kRiscvSd:
1562       __ Sd(i.InputOrZeroRegister(2), i.MemoryOperand());
1563       break;
1564     case kRiscvUsd:
1565       __ Usd(i.InputOrZeroRegister(2), i.MemoryOperand());
1566       break;
1567     case kRiscvLoadFloat: {
1568       __ LoadFloat(i.OutputSingleRegister(), i.MemoryOperand());
1569       break;
1570     }
1571     case kRiscvULoadFloat: {
1572       __ ULoadFloat(i.OutputSingleRegister(), i.MemoryOperand(), kScratchReg);
1573       break;
1574     }
1575     case kRiscvStoreFloat: {
1576       size_t index = 0;
1577       MemOperand operand = i.MemoryOperand(&index);
1578       FPURegister ft = i.InputOrZeroSingleRegister(index);
1579       if (ft == kDoubleRegZero && !__ IsSingleZeroRegSet()) {
1580         __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1581       }
1582       __ StoreFloat(ft, operand);
1583       break;
1584     }
1585     case kRiscvUStoreFloat: {
1586       size_t index = 0;
1587       MemOperand operand = i.MemoryOperand(&index);
1588       FPURegister ft = i.InputOrZeroSingleRegister(index);
1589       if (ft == kDoubleRegZero && !__ IsSingleZeroRegSet()) {
1590         __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
1591       }
1592       __ UStoreFloat(ft, operand, kScratchReg);
1593       break;
1594     }
1595     case kRiscvLoadDouble:
1596       __ LoadDouble(i.OutputDoubleRegister(), i.MemoryOperand());
1597       break;
1598     case kRiscvULoadDouble:
1599       __ ULoadDouble(i.OutputDoubleRegister(), i.MemoryOperand(), kScratchReg);
1600       break;
1601     case kRiscvStoreDouble: {
1602       FPURegister ft = i.InputOrZeroDoubleRegister(2);
1603       if (ft == kDoubleRegZero && !__ IsDoubleZeroRegSet()) {
1604         __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1605       }
1606       __ StoreDouble(ft, i.MemoryOperand());
1607       break;
1608     }
1609     case kRiscvUStoreDouble: {
1610       FPURegister ft = i.InputOrZeroDoubleRegister(2);
1611       if (ft == kDoubleRegZero && !__ IsDoubleZeroRegSet()) {
1612         __ LoadFPRImmediate(kDoubleRegZero, 0.0);
1613       }
1614       __ UStoreDouble(ft, i.MemoryOperand(), kScratchReg);
1615       break;
1616     }
1617     case kRiscvSync: {
1618       __ sync();
1619       break;
1620     }
1621     case kRiscvPush:
1622       if (instr->InputAt(0)->IsFPRegister()) {
1623         __ StoreDouble(i.InputDoubleRegister(0), MemOperand(sp, -kDoubleSize));
1624         __ Sub32(sp, sp, Operand(kDoubleSize));
1625         frame_access_state()->IncreaseSPDelta(kDoubleSize / kSystemPointerSize);
1626       } else {
1627         __ Push(i.InputOrZeroRegister(0));
1628         frame_access_state()->IncreaseSPDelta(1);
1629       }
1630       break;
1631     case kRiscvPeek: {
1632       int reverse_slot = i.InputInt32(0);
1633       int offset =
1634           FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1635       if (instr->OutputAt(0)->IsFPRegister()) {
1636         LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1637         if (op->representation() == MachineRepresentation::kFloat64) {
1638           __ LoadDouble(i.OutputDoubleRegister(), MemOperand(fp, offset));
1639         } else {
1640           DCHECK_EQ(op->representation(), MachineRepresentation::kFloat32);
1641           __ LoadFloat(
1642               i.OutputSingleRegister(0),
1643               MemOperand(fp, offset + kLessSignificantWordInDoublewordOffset));
1644         }
1645       } else {
1646         __ Ld(i.OutputRegister(0), MemOperand(fp, offset));
1647       }
1648       break;
1649     }
1650     case kRiscvStackClaim: {
1651       __ Sub64(sp, sp, Operand(i.InputInt32(0)));
1652       frame_access_state()->IncreaseSPDelta(i.InputInt32(0) /
1653                                             kSystemPointerSize);
1654       break;
1655     }
1656     case kRiscvStoreToStackSlot: {
1657       if (instr->InputAt(0)->IsFPRegister()) {
1658         if (instr->InputAt(0)->IsSimd128Register()) {
1659           UNREACHABLE();
1660         } else {
1661           __ StoreDouble(i.InputDoubleRegister(0),
1662                          MemOperand(sp, i.InputInt32(1)));
1663         }
1664       } else {
1665         __ Sd(i.InputOrZeroRegister(0), MemOperand(sp, i.InputInt32(1)));
1666       }
1667       break;
1668     }
1669     case kRiscvByteSwap64: {
1670       __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 8, kScratchReg);
1671       break;
1672     }
1673     case kRiscvByteSwap32: {
1674       __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 4, kScratchReg);
1675       break;
1676     }
1677     case kAtomicLoadInt8:
1678       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1679       ASSEMBLE_ATOMIC_LOAD_INTEGER(Lb);
1680       break;
1681     case kAtomicLoadUint8:
1682       ASSEMBLE_ATOMIC_LOAD_INTEGER(Lbu);
1683       break;
1684     case kAtomicLoadInt16:
1685       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1686       ASSEMBLE_ATOMIC_LOAD_INTEGER(Lh);
1687       break;
1688     case kAtomicLoadUint16:
1689       ASSEMBLE_ATOMIC_LOAD_INTEGER(Lhu);
1690       break;
1691     case kAtomicLoadWord32:
1692       ASSEMBLE_ATOMIC_LOAD_INTEGER(Lw);
1693       break;
1694     case kRiscvWord64AtomicLoadUint64:
1695       ASSEMBLE_ATOMIC_LOAD_INTEGER(Ld);
1696       break;
1697     case kAtomicStoreWord8:
1698       ASSEMBLE_ATOMIC_STORE_INTEGER(Sb);
1699       break;
1700     case kAtomicStoreWord16:
1701       ASSEMBLE_ATOMIC_STORE_INTEGER(Sh);
1702       break;
1703     case kAtomicStoreWord32:
1704       ASSEMBLE_ATOMIC_STORE_INTEGER(Sw);
1705       break;
1706     case kRiscvWord64AtomicStoreWord64:
1707       ASSEMBLE_ATOMIC_STORE_INTEGER(Sd);
1708       break;
1709     case kAtomicExchangeInt8:
1710       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1711       ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32);
1712       break;
1713     case kAtomicExchangeUint8:
1714       switch (AtomicWidthField::decode(opcode)) {
1715         case AtomicWidth::kWord32:
1716           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32);
1717           break;
1718         case AtomicWidth::kWord64:
1719           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64);
1720           break;
1721       }
1722       break;
1723     case kAtomicExchangeInt16:
1724       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1725       ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32);
1726       break;
1727     case kAtomicExchangeUint16:
1728       switch (AtomicWidthField::decode(opcode)) {
1729         case AtomicWidth::kWord32:
1730           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32);
1731           break;
1732         case AtomicWidth::kWord64:
1733           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64);
1734           break;
1735       }
1736       break;
1737     case kAtomicExchangeWord32:
1738       switch (AtomicWidthField::decode(opcode)) {
1739         case AtomicWidth::kWord32:
1740           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Ll, Sc);
1741           break;
1742         case AtomicWidth::kWord64:
1743           ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64);
1744           break;
1745       }
1746       break;
1747     case kRiscvWord64AtomicExchangeUint64:
1748       ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Lld, Scd);
1749       break;
1750     case kAtomicCompareExchangeInt8:
1751       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1752       ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32);
1753       break;
1754     case kAtomicCompareExchangeUint8:
1755       switch (AtomicWidthField::decode(opcode)) {
1756         case AtomicWidth::kWord32:
1757           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32);
1758           break;
1759         case AtomicWidth::kWord64:
1760           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64);
1761           break;
1762       }
1763       break;
1764     case kAtomicCompareExchangeInt16:
1765       DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32);
1766       ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32);
1767       break;
1768     case kAtomicCompareExchangeUint16:
1769       switch (AtomicWidthField::decode(opcode)) {
1770         case AtomicWidth::kWord32:
1771           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32);
1772           break;
1773         case AtomicWidth::kWord64:
1774           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64);
1775           break;
1776       }
1777       break;
1778     case kAtomicCompareExchangeWord32:
1779       switch (AtomicWidthField::decode(opcode)) {
1780         case AtomicWidth::kWord32:
1781           __ Sll32(i.InputRegister(2), i.InputRegister(2), 0);
1782           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Ll, Sc);
1783           break;
1784         case AtomicWidth::kWord64:
1785           ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64);
1786           break;
1787       }
1788       break;
1789     case kRiscvWord64AtomicCompareExchangeUint64:
1790       ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Lld, Scd);
1791       break;
1792 #define ATOMIC_BINOP_CASE(op, inst32, inst64)                          \
1793   case kAtomic##op##Int8:                                              \
1794     DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \
1795     ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 8, inst32, 32);            \
1796     break;                                                             \
1797   case kAtomic##op##Uint8:                                             \
1798     switch (AtomicWidthField::decode(opcode)) {                        \
1799       case AtomicWidth::kWord32:                                       \
1800         ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 8, inst32, 32);       \
1801         break;                                                         \
1802       case AtomicWidth::kWord64:                                       \
1803         ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 8, inst64, 64);     \
1804         break;                                                         \
1805     }                                                                  \
1806     break;                                                             \
1807   case kAtomic##op##Int16:                                             \
1808     DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \
1809     ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 16, inst32, 32);           \
1810     break;                                                             \
1811   case kAtomic##op##Uint16:                                            \
1812     switch (AtomicWidthField::decode(opcode)) {                        \
1813       case AtomicWidth::kWord32:                                       \
1814         ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 16, inst32, 32);      \
1815         break;                                                         \
1816       case AtomicWidth::kWord64:                                       \
1817         ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 16, inst64, 64);    \
1818         break;                                                         \
1819     }                                                                  \
1820     break;                                                             \
1821   case kAtomic##op##Word32:                                            \
1822     switch (AtomicWidthField::decode(opcode)) {                        \
1823       case AtomicWidth::kWord32:                                       \
1824         ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32);                         \
1825         break;                                                         \
1826       case AtomicWidth::kWord64:                                       \
1827         ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 32, inst64, 64);    \
1828         break;                                                         \
1829     }                                                                  \
1830     break;                                                             \
1831   case kRiscvWord64Atomic##op##Uint64:                                 \
1832     ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64);                           \
1833     break;
1834       ATOMIC_BINOP_CASE(Add, Add32, Add64)
1835       ATOMIC_BINOP_CASE(Sub, Sub32, Sub64)
1836       ATOMIC_BINOP_CASE(And, And, And)
1837       ATOMIC_BINOP_CASE(Or, Or, Or)
1838       ATOMIC_BINOP_CASE(Xor, Xor, Xor)
1839 #undef ATOMIC_BINOP_CASE
1840     case kRiscvAssertEqual:
1841       __ Assert(eq, static_cast<AbortReason>(i.InputOperand(2).immediate()),
1842                 i.InputRegister(0), Operand(i.InputRegister(1)));
1843       break;
1844     case kRiscvStoreCompressTagged: {
1845       size_t index = 0;
1846       MemOperand operand = i.MemoryOperand(&index);
1847       __ StoreTaggedField(i.InputOrZeroRegister(index), operand);
1848       break;
1849     }
1850     case kRiscvLoadDecompressTaggedSigned: {
1851       CHECK(instr->HasOutput());
1852       Register result = i.OutputRegister();
1853       MemOperand operand = i.MemoryOperand();
1854       __ DecompressTaggedSigned(result, operand);
1855       break;
1856     }
1857     case kRiscvLoadDecompressTaggedPointer: {
1858       CHECK(instr->HasOutput());
1859       Register result = i.OutputRegister();
1860       MemOperand operand = i.MemoryOperand();
1861       __ DecompressTaggedPointer(result, operand);
1862       break;
1863     }
1864     case kRiscvLoadDecompressAnyTagged: {
1865       CHECK(instr->HasOutput());
1866       Register result = i.OutputRegister();
1867       MemOperand operand = i.MemoryOperand();
1868       __ DecompressAnyTagged(result, operand);
1869       break;
1870     }
1871     case kRiscvRvvSt: {
1872       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1873       Register dst = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm()
1874                                                      : kScratchReg;
1875       if (i.MemoryOperand().offset() != 0) {
1876         __ Add64(dst, i.MemoryOperand().rm(), i.MemoryOperand().offset());
1877       }
1878       __ vs(i.InputSimd128Register(2), dst, 0, VSew::E8);
1879       break;
1880     }
1881     case kRiscvRvvLd: {
1882       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1883       Register src = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm()
1884                                                      : kScratchReg;
1885       if (i.MemoryOperand().offset() != 0) {
1886         __ Add64(src, i.MemoryOperand().rm(), i.MemoryOperand().offset());
1887       }
1888       __ vl(i.OutputSimd128Register(), src, 0, VSew::E8);
1889       break;
1890     }
1891     case kRiscvS128Const: {
1892       Simd128Register dst = i.OutputSimd128Register();
1893       uint8_t imm[16];
1894       *reinterpret_cast<uint64_t*>(imm) =
1895           make_uint64(i.InputUint32(1), i.InputUint32(0));
1896       *(reinterpret_cast<uint64_t*>(imm) + 1) =
1897           make_uint64(i.InputUint32(3), i.InputUint32(2));
1898       __ WasmRvvS128const(dst, imm);
1899       break;
1900     }
1901     case kRiscvI64x2Add: {
1902       (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1);
1903       __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1904                  i.InputSimd128Register(1));
1905       break;
1906     }
1907     case kRiscvI32x4Add: {
1908       (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1);
1909       __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1910                  i.InputSimd128Register(1));
1911       break;
1912     }
1913     case kRiscvI16x8Add: {
1914       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1915       __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1916                  i.InputSimd128Register(1));
1917       break;
1918     }
1919     case kRiscvI16x8AddSatS: {
1920       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1921       __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1922                   i.InputSimd128Register(1));
1923       break;
1924     }
1925     case kRiscvI16x8AddSatU: {
1926       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1927       __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1928                    i.InputSimd128Register(1));
1929       break;
1930     }
1931     case kRiscvI8x16Add: {
1932       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1933       __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1934                  i.InputSimd128Register(1));
1935       break;
1936     }
1937     case kRiscvI8x16AddSatS: {
1938       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1939       __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1940                   i.InputSimd128Register(1));
1941       break;
1942     }
1943     case kRiscvI8x16AddSatU: {
1944       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1945       __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1946                    i.InputSimd128Register(1));
1947       break;
1948     }
1949     case kRiscvI64x2Sub: {
1950       (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1);
1951       __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1952                  i.InputSimd128Register(1));
1953       break;
1954     }
1955     case kRiscvI32x4Sub: {
1956       (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1);
1957       __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1958                  i.InputSimd128Register(1));
1959       break;
1960     }
1961     case kRiscvI16x8Sub: {
1962       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1963       __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1964                  i.InputSimd128Register(1));
1965       break;
1966     }
1967     case kRiscvI16x8SubSatS: {
1968       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1969       __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1970                   i.InputSimd128Register(1));
1971       break;
1972     }
1973     case kRiscvI16x8SubSatU: {
1974       (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1);
1975       __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1976                    i.InputSimd128Register(1));
1977       break;
1978     }
1979     case kRiscvI8x16Sub: {
1980       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1981       __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1982                  i.InputSimd128Register(1));
1983       break;
1984     }
1985     case kRiscvI8x16SubSatS: {
1986       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1987       __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1988                   i.InputSimd128Register(1));
1989       break;
1990     }
1991     case kRiscvI8x16SubSatU: {
1992       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1993       __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
1994                    i.InputSimd128Register(1));
1995       break;
1996     }
1997     case kRiscvS128And: {
1998       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
1999       __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2000                  i.InputSimd128Register(1));
2001       break;
2002     }
2003     case kRiscvS128Or: {
2004       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2005       __ vor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2006                 i.InputSimd128Register(1));
2007       break;
2008     }
2009     case kRiscvS128Xor: {
2010       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2011       __ vxor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2012                  i.InputSimd128Register(1));
2013       break;
2014     }
2015     case kRiscvS128Not: {
2016       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2017       __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2018       break;
2019     }
2020     case kRiscvS128AndNot: {
2021       (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1);
2022       __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(1));
2023       __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2024                  i.OutputSimd128Register());
2025       break;
2026     }
2027     case kRiscvI32x4ExtractLane: {
2028       __ WasmRvvExtractLane(i.OutputRegister(), i.InputSimd128Register(0),
2029                             i.InputInt8(1), E32, m1);
2030       break;
2031     }
2032     case kRiscvI8x16Splat: {
2033       (__ VU).set(kScratchReg, E8, m1);
2034       __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2035       break;
2036     }
2037     case kRiscvI16x8Splat: {
2038       (__ VU).set(kScratchReg, E16, m1);
2039       __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2040       break;
2041     }
2042     case kRiscvI32x4Splat: {
2043       (__ VU).set(kScratchReg, E32, m1);
2044       __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2045       break;
2046     }
2047     case kRiscvI64x2Splat: {
2048       (__ VU).set(kScratchReg, E64, m1);
2049       __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0));
2050       break;
2051     }
2052     case kRiscvF32x4Splat: {
2053       (__ VU).set(kScratchReg, E32, m1);
2054       __ fmv_x_w(kScratchReg, i.InputSingleRegister(0));
2055       __ vmv_vx(i.OutputSimd128Register(), kScratchReg);
2056       break;
2057     }
2058     case kRiscvF64x2Splat: {
2059       (__ VU).set(kScratchReg, E64, m1);
2060       __ fmv_x_d(kScratchReg, i.InputDoubleRegister(0));
2061       __ vmv_vx(i.OutputSimd128Register(), kScratchReg);
2062       break;
2063     }
2064     case kRiscvI32x4Abs: {
2065       __ VU.set(kScratchReg, E32, m1);
2066       __ vmv_vx(kSimd128RegZero, zero_reg);
2067       __ vmv_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2068       __ vmslt_vv(v0, i.InputSimd128Register(0), kSimd128RegZero);
2069       __ vsub_vv(i.OutputSimd128Register(), kSimd128RegZero,
2070                  i.InputSimd128Register(0), Mask);
2071       break;
2072     }
2073     case kRiscvI8x16Eq: {
2074       __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2075                    i.InputSimd128Register(1), E8, m1);
2076       break;
2077     }
2078     case kRiscvI16x8Eq: {
2079       __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2080                    i.InputSimd128Register(1), E16, m1);
2081       break;
2082     }
2083     case kRiscvI32x4Eq: {
2084       __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2085                    i.InputSimd128Register(1), E32, m1);
2086       break;
2087     }
2088     case kRiscvI64x2Eq: {
2089       __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2090                    i.InputSimd128Register(1), E64, m1);
2091       break;
2092     }
2093     case kRiscvI8x16Ne: {
2094       __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2095                    i.InputSimd128Register(1), E8, m1);
2096       break;
2097     }
2098     case kRiscvI16x8Ne: {
2099       __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2100                    i.InputSimd128Register(1), E16, m1);
2101       break;
2102     }
2103     case kRiscvI32x4Ne: {
2104       __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2105                    i.InputSimd128Register(1), E32, m1);
2106       break;
2107     }
2108     case kRiscvI64x2Ne: {
2109       __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0),
2110                    i.InputSimd128Register(1), E64, m1);
2111       break;
2112     }
2113     case kRiscvI8x16GeS: {
2114       __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2115                     i.InputSimd128Register(1), E8, m1);
2116       break;
2117     }
2118     case kRiscvI16x8GeS: {
2119       __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2120                     i.InputSimd128Register(1), E16, m1);
2121       break;
2122     }
2123     case kRiscvI32x4GeS: {
2124       __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2125                     i.InputSimd128Register(1), E32, m1);
2126       break;
2127     }
2128     case kRiscvI64x2GeS: {
2129       __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2130                     i.InputSimd128Register(1), E64, m1);
2131       break;
2132     }
2133     case kRiscvI8x16GeU: {
2134       __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2135                     i.InputSimd128Register(1), E8, m1);
2136       break;
2137     }
2138     case kRiscvI16x8GeU: {
2139       __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2140                     i.InputSimd128Register(1), E16, m1);
2141       break;
2142     }
2143     case kRiscvI32x4GeU: {
2144       __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2145                     i.InputSimd128Register(1), E32, m1);
2146       break;
2147     }
2148     case kRiscvI8x16GtS: {
2149       __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2150                     i.InputSimd128Register(1), E8, m1);
2151       break;
2152     }
2153     case kRiscvI16x8GtS: {
2154       __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2155                     i.InputSimd128Register(1), E16, m1);
2156       break;
2157     }
2158     case kRiscvI32x4GtS: {
2159       __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2160                     i.InputSimd128Register(1), E32, m1);
2161       break;
2162     }
2163     case kRiscvI64x2GtS: {
2164       __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0),
2165                     i.InputSimd128Register(1), E64, m1);
2166       break;
2167     }
2168     case kRiscvI8x16GtU: {
2169       __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2170                     i.InputSimd128Register(1), E8, m1);
2171       break;
2172     }
2173     case kRiscvI16x8GtU: {
2174       __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2175                     i.InputSimd128Register(1), E16, m1);
2176       break;
2177     }
2178     case kRiscvI32x4GtU: {
2179       __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0),
2180                     i.InputSimd128Register(1), E32, m1);
2181       break;
2182     }
2183     case kRiscvI8x16Shl: {
2184       __ VU.set(kScratchReg, E8, m1);
2185       if (instr->InputAt(1)->IsRegister()) {
2186         __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2187                    i.InputRegister(1));
2188       } else {
2189         __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2190                    i.InputInt3(1));
2191       }
2192       break;
2193     }
2194     case kRiscvI16x8Shl: {
2195       __ VU.set(kScratchReg, E16, m1);
2196       if (instr->InputAt(1)->IsRegister()) {
2197         __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2198                    i.InputRegister(1));
2199       } else {
2200         __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2201                    i.InputInt4(1));
2202       }
2203       break;
2204     }
2205     case kRiscvI32x4Shl: {
2206       __ VU.set(kScratchReg, E32, m1);
2207       if (instr->InputAt(1)->IsRegister()) {
2208         __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2209                    i.InputRegister(1));
2210       } else {
2211         __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2212                    i.InputInt5(1));
2213       }
2214       break;
2215     }
2216     case kRiscvI64x2Shl: {
2217       __ VU.set(kScratchReg, E64, m1);
2218       if (instr->InputAt(1)->IsRegister()) {
2219         __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2220                    i.InputRegister(1));
2221       } else {
2222         if (is_int5(i.InputInt6(1))) {
2223           __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0),
2224                      i.InputInt6(1));
2225         } else {
2226           __ li(kScratchReg, i.InputInt6(1));
2227           __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0),
2228                      kScratchReg);
2229         }
2230       }
2231       break;
2232     }
2233     case kRiscvI8x16ReplaceLane: {
2234       Simd128Register src = i.InputSimd128Register(0);
2235       Simd128Register dst = i.OutputSimd128Register();
2236       __ VU.set(kScratchReg, E32, m1);
2237       __ li(kScratchReg, 0x1 << i.InputInt8(1));
2238       __ vmv_sx(v0, kScratchReg);
2239       __ vmerge_vx(dst, i.InputRegister(2), src);
2240       break;
2241     }
2242     case kRiscvI16x8ReplaceLane: {
2243       Simd128Register src = i.InputSimd128Register(0);
2244       Simd128Register dst = i.OutputSimd128Register();
2245       __ VU.set(kScratchReg, E16, m1);
2246       __ li(kScratchReg, 0x1 << i.InputInt8(1));
2247       __ vmv_sx(v0, kScratchReg);
2248       __ vmerge_vx(dst, i.InputRegister(2), src);
2249       break;
2250     }
2251     case kRiscvI64x2ReplaceLane: {
2252       Simd128Register src = i.InputSimd128Register(0);
2253       Simd128Register dst = i.OutputSimd128Register();
2254       __ VU.set(kScratchReg, E64, m1);
2255       __ li(kScratchReg, 0x1 << i.InputInt8(1));
2256       __ vmv_sx(v0, kScratchReg);
2257       __ vmerge_vx(dst, i.InputRegister(2), src);
2258       break;
2259     }
2260     case kRiscvI32x4ReplaceLane: {
2261       Simd128Register src = i.InputSimd128Register(0);
2262       Simd128Register dst = i.OutputSimd128Register();
2263       __ VU.set(kScratchReg, E32, m1);
2264       __ li(kScratchReg, 0x1 << i.InputInt8(1));
2265       __ vmv_sx(v0, kScratchReg);
2266       __ vmerge_vx(dst, i.InputRegister(2), src);
2267       break;
2268     }
2269     case kRiscvI8x16BitMask: {
2270       Register dst = i.OutputRegister();
2271       Simd128Register src = i.InputSimd128Register(0);
2272       __ VU.set(kScratchReg, E8, m1);
2273       __ vmv_vx(kSimd128RegZero, zero_reg);
2274       __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2275       __ VU.set(kScratchReg, E32, m1);
2276       __ vmv_xs(dst, kSimd128ScratchReg);
2277       break;
2278     }
2279     case kRiscvI16x8BitMask: {
2280       Register dst = i.OutputRegister();
2281       Simd128Register src = i.InputSimd128Register(0);
2282       __ VU.set(kScratchReg, E16, m1);
2283       __ vmv_vx(kSimd128RegZero, zero_reg);
2284       __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2285       __ VU.set(kScratchReg, E32, m1);
2286       __ vmv_xs(dst, kSimd128ScratchReg);
2287       break;
2288     }
2289     case kRiscvI32x4BitMask: {
2290       Register dst = i.OutputRegister();
2291       Simd128Register src = i.InputSimd128Register(0);
2292       __ VU.set(kScratchReg, E32, m1);
2293       __ vmv_vx(kSimd128RegZero, zero_reg);
2294       __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2295       __ vmv_xs(dst, kSimd128ScratchReg);
2296       break;
2297     }
2298     case kRiscvI64x2BitMask: {
2299       Register dst = i.OutputRegister();
2300       Simd128Register src = i.InputSimd128Register(0);
2301       __ VU.set(kScratchReg, E64, m1);
2302       __ vmv_vx(kSimd128RegZero, zero_reg);
2303       __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero);
2304       __ VU.set(kScratchReg, E32, m1);
2305       __ vmv_xs(dst, kSimd128ScratchReg);
2306       break;
2307     }
2308     case kRiscvV128AnyTrue: {
2309       __ VU.set(kScratchReg, E8, m1);
2310       Register dst = i.OutputRegister();
2311       Label t;
2312       __ vmv_sx(kSimd128ScratchReg, zero_reg);
2313       __ vredmaxu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2314                      kSimd128ScratchReg);
2315       __ vmv_xs(dst, kSimd128ScratchReg);
2316       __ beq(dst, zero_reg, &t);
2317       __ li(dst, 1);
2318       __ bind(&t);
2319       break;
2320     }
2321     case kRiscvI64x2AllTrue: {
2322       __ VU.set(kScratchReg, E64, m1);
2323       Register dst = i.OutputRegister();
2324       Label all_true;
2325       __ li(kScratchReg, -1);
2326       __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2327       __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2328                      kSimd128ScratchReg);
2329       __ vmv_xs(dst, kSimd128ScratchReg);
2330       __ beqz(dst, &all_true);
2331       __ li(dst, 1);
2332       __ bind(&all_true);
2333       break;
2334     }
2335     case kRiscvI32x4AllTrue: {
2336       __ VU.set(kScratchReg, E32, m1);
2337       Register dst = i.OutputRegister();
2338       Label all_true;
2339       __ li(kScratchReg, -1);
2340       __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2341       __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2342                      kSimd128ScratchReg);
2343       __ vmv_xs(dst, kSimd128ScratchReg);
2344       __ beqz(dst, &all_true);
2345       __ li(dst, 1);
2346       __ bind(&all_true);
2347       break;
2348     }
2349     case kRiscvI16x8AllTrue: {
2350       __ VU.set(kScratchReg, E16, m1);
2351       Register dst = i.OutputRegister();
2352       Label all_true;
2353       __ li(kScratchReg, -1);
2354       __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2355       __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2356                      kSimd128ScratchReg);
2357       __ vmv_xs(dst, kSimd128ScratchReg);
2358       __ beqz(dst, &all_true);
2359       __ li(dst, 1);
2360       __ bind(&all_true);
2361       break;
2362     }
2363     case kRiscvI8x16AllTrue: {
2364       __ VU.set(kScratchReg, E8, m1);
2365       Register dst = i.OutputRegister();
2366       Label all_true;
2367       __ li(kScratchReg, -1);
2368       __ vmv_sx(kSimd128ScratchReg, kScratchReg);
2369       __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0),
2370                      kSimd128ScratchReg);
2371       __ vmv_xs(dst, kSimd128ScratchReg);
2372       __ beqz(dst, &all_true);
2373       __ li(dst, 1);
2374       __ bind(&all_true);
2375       break;
2376     }
2377     case kRiscvI8x16Shuffle: {
2378       VRegister dst = i.OutputSimd128Register(),
2379                 src0 = i.InputSimd128Register(0),
2380                 src1 = i.InputSimd128Register(1);
2381 
2382       int64_t imm1 = make_uint64(i.InputInt32(3), i.InputInt32(2));
2383       int64_t imm2 = make_uint64(i.InputInt32(5), i.InputInt32(4));
2384       __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2385       __ li(kScratchReg, 1);
2386       __ vmv_vx(v0, kScratchReg);
2387       __ li(kScratchReg, imm1);
2388       __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg);
2389       __ li(kScratchReg, imm2);
2390       __ vsll_vi(v0, v0, 1);
2391       __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg);
2392 
2393       __ VU.set(kScratchReg, E8, m1);
2394       if (dst == src0) {
2395         __ vmv_vv(kSimd128ScratchReg2, src0);
2396         src0 = kSimd128ScratchReg2;
2397       } else if (dst == src1) {
2398         __ vmv_vv(kSimd128ScratchReg2, src1);
2399         src1 = kSimd128ScratchReg2;
2400       }
2401       __ vrgather_vv(dst, src0, kSimd128ScratchReg);
2402       __ vadd_vi(kSimd128ScratchReg, kSimd128ScratchReg, -16);
2403       __ vrgather_vv(kSimd128ScratchReg, src1, kSimd128ScratchReg);
2404       __ vor_vv(dst, dst, kSimd128ScratchReg);
2405       break;
2406     }
2407     case kRiscvF32x4Abs: {
2408       __ VU.set(kScratchReg, VSew::E32, Vlmul::m1);
2409       __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2410       break;
2411     }
2412     case kRiscvF64x2Abs: {
2413       __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2414       __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2415       break;
2416     }
2417     case kRiscvF32x4Neg: {
2418       __ VU.set(kScratchReg, VSew::E32, Vlmul::m1);
2419       __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2420       break;
2421     }
2422     case kRiscvF64x2Neg: {
2423       __ VU.set(kScratchReg, VSew::E64, Vlmul::m1);
2424       __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0));
2425       break;
2426     }
2427     case kRiscvF32x4DemoteF64x2Zero: {
2428       __ VU.set(kScratchReg, E32, m1);
2429       __ vfncvt_f_f_w(i.OutputSimd128Register(), i.InputSimd128Register(0));
2430       __ vmv_vi(v0, 12);
2431       __ vmerge_vx(i.OutputSimd128Register(), zero_reg,
2432                    i.OutputSimd128Register());
2433       break;
2434     }
2435     case kRiscvF32x4Add: {
2436       __ VU.set(kScratchReg, E32, m1);
2437       __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2438                   i.InputSimd128Register(1));
2439       break;
2440     }
2441     case kRiscvF32x4Sub: {
2442       __ VU.set(kScratchReg, E32, m1);
2443       __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2444                   i.InputSimd128Register(1));
2445       break;
2446     }
2447     case kRiscvF64x2Add: {
2448       __ VU.set(kScratchReg, E64, m1);
2449       __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2450                   i.InputSimd128Register(1));
2451       break;
2452     }
2453     case kRiscvF64x2Sub: {
2454       __ VU.set(kScratchReg, E64, m1);
2455       __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0),
2456                   i.InputSimd128Register(1));
2457       break;
2458     }
2459     case kRiscvF32x4Ceil: {
2460       __ Ceil_f(i.OutputSimd128Register(), i.InputSimd128Register(0),
2461                 kScratchReg, kSimd128ScratchReg);
2462       break;
2463     }
2464     case kRiscvF64x2Ceil: {
2465       __ Ceil_d(i.OutputSimd128Register(), i.InputSimd128Register(0),
2466                 kScratchReg, kSimd128ScratchReg);
2467       break;
2468     }
2469     case kRiscvF32x4Floor: {
2470       __ Floor_f(i.OutputSimd128Register(), i.InputSimd128Register(0),
2471                  kScratchReg, kSimd128ScratchReg);
2472       break;
2473     }
2474     case kRiscvF64x2Floor: {
2475       __ Floor_d(i.OutputSimd128Register(), i.InputSimd128Register(0),
2476                  kScratchReg, kSimd128ScratchReg);
2477       break;
2478     }
2479     case kRiscvS128Select: {
2480       __ VU.set(kScratchReg, E8, m1);
2481       __ vand_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2482                  i.InputSimd128Register(0));
2483       __ vnot_vv(kSimd128ScratchReg2, i.InputSimd128Register(0));
2484       __ vand_vv(kSimd128ScratchReg2, i.InputSimd128Register(2),
2485                  kSimd128ScratchReg2);
2486       __ vor_vv(i.OutputSimd128Register(), kSimd128ScratchReg,
2487                 kSimd128ScratchReg2);
2488       break;
2489     }
2490     case kRiscvF32x4UConvertI32x4: {
2491       __ VU.set(kScratchReg, E32, m1);
2492       __ VU.set(RoundingMode::RTZ);
2493       __ vfcvt_f_xu_v(i.OutputSimd128Register(), i.InputSimd128Register(0));
2494       break;
2495     }
2496     case kRiscvF32x4SConvertI32x4: {
2497       __ VU.set(kScratchReg, E32, m1);
2498       __ VU.set(RoundingMode::RTZ);
2499       __ vfcvt_f_x_v(i.OutputSimd128Register(), i.InputSimd128Register(0));
2500       break;
2501     }
2502     case kRiscvF32x4Div: {
2503       __ VU.set(kScratchReg, E32, m1);
2504       __ VU.set(RoundingMode::RTZ);
2505       __ vfdiv_vv(i.OutputSimd128Register(), i.InputSimd128Register(1),
2506                   i.InputSimd128Register(0));
2507       break;
2508     }
2509     case kRiscvF32x4Mul: {
2510       __ VU.set(kScratchReg, E32, m1);
2511       __ VU.set(RoundingMode::RTZ);
2512       __ vfmul_vv(i.OutputSimd128Register(), i.InputSimd128Register(1),
2513                   i.InputSimd128Register(0));
2514       break;
2515     }
2516     case kRiscvF32x4Eq: {
2517       __ VU.set(kScratchReg, E32, m1);
2518       __ vmfeq_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2519       __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2520       __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2521       break;
2522     }
2523     case kRiscvF32x4Ne: {
2524       __ VU.set(kScratchReg, E32, m1);
2525       __ vmfne_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2526       __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2527       __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2528       break;
2529     }
2530     case kRiscvF32x4Lt: {
2531       __ VU.set(kScratchReg, E32, m1);
2532       __ vmflt_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2533       __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2534       __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2535       break;
2536     }
2537     case kRiscvF32x4Le: {
2538       __ VU.set(kScratchReg, E32, m1);
2539       __ vmfle_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0));
2540       __ vmv_vx(i.OutputSimd128Register(), zero_reg);
2541       __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register());
2542       break;
2543     }
2544     case kRiscvF32x4Max: {
2545       __ VU.set(kScratchReg, E32, m1);
2546       const int32_t kNaN = 0x7FC00000;
2547       __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0));
2548       __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2549                   i.InputSimd128Register(1));
2550       __ vand_vv(v0, v0, kSimd128ScratchReg);
2551       __ li(kScratchReg, kNaN);
2552       __ vmv_vx(kSimd128ScratchReg, kScratchReg);
2553       __ vfmax_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2554                   i.InputSimd128Register(0), Mask);
2555       __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg);
2556       break;
2557     }
2558     case kRiscvF32x4Min: {
2559       __ VU.set(kScratchReg, E32, m1);
2560       const int32_t kNaN = 0x7FC00000;
2561       __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0));
2562       __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2563                   i.InputSimd128Register(1));
2564       __ vand_vv(v0, v0, kSimd128ScratchReg);
2565       __ li(kScratchReg, kNaN);
2566       __ vmv_vx(kSimd128ScratchReg, kScratchReg);
2567       __ vfmin_vv(kSimd128ScratchReg, i.InputSimd128Register(1),
2568                   i.InputSimd128Register(0), Mask);
2569       __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg);
2570       break;
2571     }
2572     default:
2573 #ifdef DEBUG
2574       switch (arch_opcode) {
2575 #define Print(name)       \
2576   case k##name:           \
2577     printf("k%s", #name); \
2578     break;
2579         TARGET_ARCH_OPCODE_LIST(Print);
2580 #undef Print
2581         default:
2582           break;
2583       }
2584 #endif
2585       UNIMPLEMENTED();
2586   }
2587   return kSuccess;
2588 }
2589 
2590 #define UNSUPPORTED_COND(opcode, condition)                                    \
2591   StdoutStream{} << "Unsupported " << #opcode << " condition: \"" << condition \
2592                  << "\"";                                                      \
2593   UNIMPLEMENTED();
2594 
IsInludeEqual(Condition cc)2595 bool IsInludeEqual(Condition cc) {
2596   switch (cc) {
2597     case equal:
2598     case greater_equal:
2599     case less_equal:
2600     case Uless_equal:
2601     case Ugreater_equal:
2602       return true;
2603     default:
2604       return false;
2605   }
2606 }
2607 
AssembleBranchToLabels(CodeGenerator * gen,TurboAssembler * tasm,Instruction * instr,FlagsCondition condition,Label * tlabel,Label * flabel,bool fallthru)2608 void AssembleBranchToLabels(CodeGenerator* gen, TurboAssembler* tasm,
2609                             Instruction* instr, FlagsCondition condition,
2610                             Label* tlabel, Label* flabel, bool fallthru) {
2611 #undef __
2612 #define __ tasm->
2613   RiscvOperandConverter i(gen, instr);
2614 
2615   Condition cc = kNoCondition;
2616   // RISC-V does not have condition code flags, so compare and branch are
2617   // implemented differently than on the other arch's. The compare operations
2618   // emit riscv64 pseudo-instructions, which are handled here by branch
2619   // instructions that do the actual comparison. Essential that the input
2620   // registers to compare pseudo-op are not modified before this branch op, as
2621   // they are tested here.
2622 
2623   if (instr->arch_opcode() == kRiscvTst) {
2624     cc = FlagsConditionToConditionTst(condition);
2625     __ Branch(tlabel, cc, kScratchReg, Operand(zero_reg));
2626   } else if (instr->arch_opcode() == kRiscvAdd64 ||
2627              instr->arch_opcode() == kRiscvSub64) {
2628     cc = FlagsConditionToConditionOvf(condition);
2629     __ Sra64(kScratchReg, i.OutputRegister(), 32);
2630     __ Sra64(kScratchReg2, i.OutputRegister(), 31);
2631     __ Branch(tlabel, cc, kScratchReg2, Operand(kScratchReg));
2632   } else if (instr->arch_opcode() == kRiscvAddOvf64 ||
2633              instr->arch_opcode() == kRiscvSubOvf64) {
2634     switch (condition) {
2635       // Overflow occurs if overflow register is negative
2636       case kOverflow:
2637         __ Branch(tlabel, lt, kScratchReg, Operand(zero_reg));
2638         break;
2639       case kNotOverflow:
2640         __ Branch(tlabel, ge, kScratchReg, Operand(zero_reg));
2641         break;
2642       default:
2643         UNSUPPORTED_COND(instr->arch_opcode(), condition);
2644     }
2645   } else if (instr->arch_opcode() == kRiscvMulOvf32) {
2646     // Overflow occurs if overflow register is not zero
2647     switch (condition) {
2648       case kOverflow:
2649         __ Branch(tlabel, ne, kScratchReg, Operand(zero_reg));
2650         break;
2651       case kNotOverflow:
2652         __ Branch(tlabel, eq, kScratchReg, Operand(zero_reg));
2653         break;
2654       default:
2655         UNSUPPORTED_COND(kRiscvMulOvf32, condition);
2656     }
2657   } else if (instr->arch_opcode() == kRiscvCmp) {
2658     cc = FlagsConditionToConditionCmp(condition);
2659     __ Branch(tlabel, cc, i.InputRegister(0), i.InputOperand(1));
2660   } else if (instr->arch_opcode() == kRiscvCmpZero) {
2661     cc = FlagsConditionToConditionCmp(condition);
2662     if (i.InputOrZeroRegister(0) == zero_reg && IsInludeEqual(cc)) {
2663       __ Branch(tlabel);
2664     } else if (i.InputOrZeroRegister(0) != zero_reg) {
2665       __ Branch(tlabel, cc, i.InputRegister(0), Operand(zero_reg));
2666     }
2667   } else if (instr->arch_opcode() == kArchStackPointerGreaterThan) {
2668     cc = FlagsConditionToConditionCmp(condition);
2669     Register lhs_register = sp;
2670     uint32_t offset;
2671     if (gen->ShouldApplyOffsetToStackCheck(instr, &offset)) {
2672       lhs_register = i.TempRegister(0);
2673       __ Sub64(lhs_register, sp, offset);
2674     }
2675     __ Branch(tlabel, cc, lhs_register, Operand(i.InputRegister(0)));
2676   } else if (instr->arch_opcode() == kRiscvCmpS ||
2677              instr->arch_opcode() == kRiscvCmpD) {
2678     bool predicate;
2679     FlagsConditionToConditionCmpFPU(&predicate, condition);
2680     // floating-point compare result is set in kScratchReg
2681     if (predicate) {
2682       __ BranchTrueF(kScratchReg, tlabel);
2683     } else {
2684       __ BranchFalseF(kScratchReg, tlabel);
2685     }
2686   } else {
2687     PrintF("AssembleArchBranch Unimplemented arch_opcode: %d\n",
2688            instr->arch_opcode());
2689     UNIMPLEMENTED();
2690   }
2691   if (!fallthru) __ Branch(flabel);  // no fallthru to flabel.
2692 #undef __
2693 #define __ tasm()->
2694 }
2695 
2696 // Assembles branches after an instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2697 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2698   Label* tlabel = branch->true_label;
2699   Label* flabel = branch->false_label;
2700 
2701   AssembleBranchToLabels(this, tasm(), instr, branch->condition, tlabel, flabel,
2702                          branch->fallthru);
2703 }
2704 
2705 #undef UNSUPPORTED_COND
2706 
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)2707 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2708                                             BranchInfo* branch) {
2709   AssembleArchBranch(instr, branch);
2710 }
2711 
AssembleArchJump(RpoNumber target)2712 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2713   if (!IsNextInAssemblyOrder(target)) __ Branch(GetLabel(target));
2714 }
2715 
AssembleArchTrap(Instruction * instr,FlagsCondition condition)2716 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2717                                      FlagsCondition condition) {
2718   class OutOfLineTrap final : public OutOfLineCode {
2719    public:
2720     OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
2721         : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
2722     void Generate() final {
2723       RiscvOperandConverter i(gen_, instr_);
2724       TrapId trap_id =
2725           static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
2726       GenerateCallToTrap(trap_id);
2727     }
2728 
2729    private:
2730     void GenerateCallToTrap(TrapId trap_id) {
2731       if (trap_id == TrapId::kInvalid) {
2732         // We cannot test calls to the runtime in cctest/test-run-wasm.
2733         // Therefore we emit a call to C here instead of a call to the runtime.
2734         // We use the context register as the scratch register, because we do
2735         // not have a context here.
2736         __ PrepareCallCFunction(0, 0, cp);
2737         __ CallCFunction(
2738             ExternalReference::wasm_call_trap_callback_for_testing(), 0);
2739         __ LeaveFrame(StackFrame::WASM);
2740         auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
2741         int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
2742         pop_count += (pop_count & 1);  // align
2743         __ Drop(pop_count);
2744         __ Ret();
2745       } else {
2746         gen_->AssembleSourcePosition(instr_);
2747         // A direct call to a wasm runtime stub defined in this module.
2748         // Just encode the stub index. This will be patched when the code
2749         // is added to the native module and copied into wasm code space.
2750         __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
2751         ReferenceMap* reference_map =
2752             gen_->zone()->New<ReferenceMap>(gen_->zone());
2753         gen_->RecordSafepoint(reference_map);
2754         if (FLAG_debug_code) {
2755           __ stop();
2756         }
2757       }
2758     }
2759     Instruction* instr_;
2760     CodeGenerator* gen_;
2761   };
2762   auto ool = zone()->New<OutOfLineTrap>(this, instr);
2763   Label* tlabel = ool->entry();
2764   AssembleBranchToLabels(this, tasm(), instr, condition, tlabel, nullptr, true);
2765 }
2766 
2767 // Assembles boolean materializations after an instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2768 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2769                                         FlagsCondition condition) {
2770   RiscvOperandConverter i(this, instr);
2771 
2772   // Materialize a full 32-bit 1 or 0 value. The result register is always the
2773   // last output of the instruction.
2774   DCHECK_NE(0u, instr->OutputCount());
2775   Register result = i.OutputRegister(instr->OutputCount() - 1);
2776   Condition cc = kNoCondition;
2777   // RISC-V does not have condition code flags, so compare and branch are
2778   // implemented differently than on the other arch's. The compare operations
2779   // emit riscv64 pseudo-instructions, which are checked and handled here.
2780 
2781   if (instr->arch_opcode() == kRiscvTst) {
2782     cc = FlagsConditionToConditionTst(condition);
2783     if (cc == eq) {
2784       __ Sltu(result, kScratchReg, 1);
2785     } else {
2786       __ Sltu(result, zero_reg, kScratchReg);
2787     }
2788     return;
2789   } else if (instr->arch_opcode() == kRiscvAdd64 ||
2790              instr->arch_opcode() == kRiscvSub64) {
2791     cc = FlagsConditionToConditionOvf(condition);
2792     // Check for overflow creates 1 or 0 for result.
2793     __ Srl64(kScratchReg, i.OutputRegister(), 63);
2794     __ Srl32(kScratchReg2, i.OutputRegister(), 31);
2795     __ Xor(result, kScratchReg, kScratchReg2);
2796     if (cc == eq)  // Toggle result for not overflow.
2797       __ Xor(result, result, 1);
2798     return;
2799   } else if (instr->arch_opcode() == kRiscvAddOvf64 ||
2800              instr->arch_opcode() == kRiscvSubOvf64) {
2801     // Overflow occurs if overflow register is negative
2802     __ Slt(result, kScratchReg, zero_reg);
2803   } else if (instr->arch_opcode() == kRiscvMulOvf32) {
2804     // Overflow occurs if overflow register is not zero
2805     __ Sgtu(result, kScratchReg, zero_reg);
2806   } else if (instr->arch_opcode() == kRiscvCmp) {
2807     cc = FlagsConditionToConditionCmp(condition);
2808     switch (cc) {
2809       case eq:
2810       case ne: {
2811         Register left = i.InputOrZeroRegister(0);
2812         Operand right = i.InputOperand(1);
2813         if (instr->InputAt(1)->IsImmediate()) {
2814           if (is_int12(-right.immediate())) {
2815             if (right.immediate() == 0) {
2816               if (cc == eq) {
2817                 __ Sltu(result, left, 1);
2818               } else {
2819                 __ Sltu(result, zero_reg, left);
2820               }
2821             } else {
2822               __ Add64(result, left, Operand(-right.immediate()));
2823               if (cc == eq) {
2824                 __ Sltu(result, result, 1);
2825               } else {
2826                 __ Sltu(result, zero_reg, result);
2827               }
2828             }
2829           } else {
2830             if (is_uint12(right.immediate())) {
2831               __ Xor(result, left, right);
2832             } else {
2833               __ li(kScratchReg, right);
2834               __ Xor(result, left, kScratchReg);
2835             }
2836             if (cc == eq) {
2837               __ Sltu(result, result, 1);
2838             } else {
2839               __ Sltu(result, zero_reg, result);
2840             }
2841           }
2842         } else {
2843           __ Xor(result, left, right);
2844           if (cc == eq) {
2845             __ Sltu(result, result, 1);
2846           } else {
2847             __ Sltu(result, zero_reg, result);
2848           }
2849         }
2850       } break;
2851       case lt:
2852       case ge: {
2853         Register left = i.InputOrZeroRegister(0);
2854         Operand right = i.InputOperand(1);
2855         __ Slt(result, left, right);
2856         if (cc == ge) {
2857           __ Xor(result, result, 1);
2858         }
2859       } break;
2860       case gt:
2861       case le: {
2862         Register left = i.InputOrZeroRegister(1);
2863         Operand right = i.InputOperand(0);
2864         __ Slt(result, left, right);
2865         if (cc == le) {
2866           __ Xor(result, result, 1);
2867         }
2868       } break;
2869       case Uless:
2870       case Ugreater_equal: {
2871         Register left = i.InputOrZeroRegister(0);
2872         Operand right = i.InputOperand(1);
2873         __ Sltu(result, left, right);
2874         if (cc == Ugreater_equal) {
2875           __ Xor(result, result, 1);
2876         }
2877       } break;
2878       case Ugreater:
2879       case Uless_equal: {
2880         Register left = i.InputRegister(1);
2881         Operand right = i.InputOperand(0);
2882         __ Sltu(result, left, right);
2883         if (cc == Uless_equal) {
2884           __ Xor(result, result, 1);
2885         }
2886       } break;
2887       default:
2888         UNREACHABLE();
2889     }
2890     return;
2891   } else if (instr->arch_opcode() == kRiscvCmpZero) {
2892     cc = FlagsConditionToConditionCmp(condition);
2893     switch (cc) {
2894       case eq: {
2895         Register left = i.InputOrZeroRegister(0);
2896         __ Sltu(result, left, 1);
2897         break;
2898       }
2899       case ne: {
2900         Register left = i.InputOrZeroRegister(0);
2901         __ Sltu(result, zero_reg, left);
2902         break;
2903       }
2904       case lt:
2905       case ge: {
2906         Register left = i.InputOrZeroRegister(0);
2907         Operand right = Operand(zero_reg);
2908         __ Slt(result, left, right);
2909         if (cc == ge) {
2910           __ Xor(result, result, 1);
2911         }
2912       } break;
2913       case gt:
2914       case le: {
2915         Operand left = i.InputOperand(0);
2916         __ Slt(result, zero_reg, left);
2917         if (cc == le) {
2918           __ Xor(result, result, 1);
2919         }
2920       } break;
2921       case Uless:
2922       case Ugreater_equal: {
2923         Register left = i.InputOrZeroRegister(0);
2924         Operand right = Operand(zero_reg);
2925         __ Sltu(result, left, right);
2926         if (cc == Ugreater_equal) {
2927           __ Xor(result, result, 1);
2928         }
2929       } break;
2930       case Ugreater:
2931       case Uless_equal: {
2932         Register left = zero_reg;
2933         Operand right = i.InputOperand(0);
2934         __ Sltu(result, left, right);
2935         if (cc == Uless_equal) {
2936           __ Xor(result, result, 1);
2937         }
2938       } break;
2939       default:
2940         UNREACHABLE();
2941     }
2942     return;
2943   } else if (instr->arch_opcode() == kArchStackPointerGreaterThan) {
2944     cc = FlagsConditionToConditionCmp(condition);
2945     Register lhs_register = sp;
2946     uint32_t offset;
2947     if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
2948       lhs_register = i.TempRegister(0);
2949       __ Sub64(lhs_register, sp, offset);
2950     }
2951     __ Sgtu(result, lhs_register, Operand(i.InputRegister(0)));
2952     return;
2953   } else if (instr->arch_opcode() == kRiscvCmpD ||
2954              instr->arch_opcode() == kRiscvCmpS) {
2955     FPURegister left = i.InputOrZeroDoubleRegister(0);
2956     FPURegister right = i.InputOrZeroDoubleRegister(1);
2957     if ((instr->arch_opcode() == kRiscvCmpD) &&
2958         (left == kDoubleRegZero || right == kDoubleRegZero) &&
2959         !__ IsDoubleZeroRegSet()) {
2960       __ LoadFPRImmediate(kDoubleRegZero, 0.0);
2961     } else if ((instr->arch_opcode() == kRiscvCmpS) &&
2962                (left == kDoubleRegZero || right == kDoubleRegZero) &&
2963                !__ IsSingleZeroRegSet()) {
2964       __ LoadFPRImmediate(kDoubleRegZero, 0.0f);
2965     }
2966     bool predicate;
2967     FlagsConditionToConditionCmpFPU(&predicate, condition);
2968     // RISCV compare returns 0 or 1, do nothing when predicate; otherwise
2969     // toggle kScratchReg (i.e., 0 -> 1, 1 -> 0)
2970     if (predicate) {
2971       __ Move(result, kScratchReg);
2972     } else {
2973       __ Xor(result, kScratchReg, 1);
2974     }
2975     return;
2976   } else {
2977     PrintF("AssembleArchBranch Unimplemented arch_opcode is : %d\n",
2978            instr->arch_opcode());
2979     TRACE_UNIMPL();
2980     UNIMPLEMENTED();
2981   }
2982 }
2983 
AssembleArchBinarySearchSwitch(Instruction * instr)2984 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2985   RiscvOperandConverter i(this, instr);
2986   Register input = i.InputRegister(0);
2987   std::vector<std::pair<int32_t, Label*>> cases;
2988   for (size_t index = 2; index < instr->InputCount(); index += 2) {
2989     cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2990   }
2991   AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2992                                       cases.data() + cases.size());
2993 }
2994 
AssembleArchTableSwitch(Instruction * instr)2995 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2996   RiscvOperandConverter i(this, instr);
2997   Register input = i.InputRegister(0);
2998   size_t const case_count = instr->InputCount() - 2;
2999 
3000   __ Branch(GetLabel(i.InputRpo(1)), Ugreater_equal, input,
3001             Operand(case_count));
3002   __ GenerateSwitchTable(input, case_count, [&i, this](size_t index) {
3003     return GetLabel(i.InputRpo(index + 2));
3004   });
3005 }
3006 
FinishFrame(Frame * frame)3007 void CodeGenerator::FinishFrame(Frame* frame) {
3008   auto call_descriptor = linkage()->GetIncomingDescriptor();
3009 
3010   const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3011   if (saves_fpu != 0) {
3012     int count = base::bits::CountPopulation(saves_fpu);
3013     DCHECK_EQ(kNumCalleeSavedFPU, count);
3014     frame->AllocateSavedCalleeRegisterSlots(count *
3015                                             (kDoubleSize / kSystemPointerSize));
3016   }
3017 
3018   const RegList saves = call_descriptor->CalleeSavedRegisters();
3019   if (saves != 0) {
3020     int count = base::bits::CountPopulation(saves);
3021     frame->AllocateSavedCalleeRegisterSlots(count);
3022   }
3023 }
3024 
AssembleConstructFrame()3025 void CodeGenerator::AssembleConstructFrame() {
3026   auto call_descriptor = linkage()->GetIncomingDescriptor();
3027 
3028   if (frame_access_state()->has_frame()) {
3029     if (call_descriptor->IsCFunctionCall()) {
3030       if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
3031         __ StubPrologue(StackFrame::C_WASM_ENTRY);
3032         // Reserve stack space for saving the c_entry_fp later.
3033         __ Sub64(sp, sp, Operand(kSystemPointerSize));
3034       } else {
3035         __ Push(ra, fp);
3036         __ Move(fp, sp);
3037       }
3038     } else if (call_descriptor->IsJSFunctionCall()) {
3039       __ Prologue();
3040     } else {
3041       __ StubPrologue(info()->GetOutputStackFrameType());
3042       if (call_descriptor->IsWasmFunctionCall()) {
3043         __ Push(kWasmInstanceRegister);
3044       } else if (call_descriptor->IsWasmImportWrapper() ||
3045                  call_descriptor->IsWasmCapiFunction()) {
3046         // Wasm import wrappers are passed a tuple in the place of the instance.
3047         // Unpack the tuple into the instance and the target callable.
3048         // This must be done here in the codegen because it cannot be expressed
3049         // properly in the graph.
3050         __ LoadTaggedPointerField(
3051             kJSFunctionRegister,
3052             FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue2Offset));
3053         __ LoadTaggedPointerField(
3054             kWasmInstanceRegister,
3055             FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue1Offset));
3056         __ Push(kWasmInstanceRegister);
3057         if (call_descriptor->IsWasmCapiFunction()) {
3058           // Reserve space for saving the PC later.
3059           __ Sub64(sp, sp, Operand(kSystemPointerSize));
3060         }
3061       }
3062     }
3063   }
3064 
3065   int required_slots =
3066       frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
3067 
3068   if (info()->is_osr()) {
3069     // TurboFan OSR-compiled functions cannot be entered directly.
3070     __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
3071 
3072     // Unoptimized code jumps directly to this entrypoint while the unoptimized
3073     // frame is still on the stack. Optimized code uses OSR values directly from
3074     // the unoptimized frame. Thus, all that needs to be done is to allocate the
3075     // remaining stack slots.
3076     __ RecordComment("-- OSR entrypoint --");
3077     osr_pc_offset_ = __ pc_offset();
3078     required_slots -= osr_helper()->UnoptimizedFrameSlots();
3079   }
3080 
3081   const RegList saves = call_descriptor->CalleeSavedRegisters();
3082   const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3083 
3084   if (required_slots > 0) {
3085     DCHECK(frame_access_state()->has_frame());
3086     if (info()->IsWasm() && required_slots > 128) {
3087       // For WebAssembly functions with big frames we have to do the stack
3088       // overflow check before we construct the frame. Otherwise we may not
3089       // have enough space on the stack to call the runtime for the stack
3090       // overflow.
3091       Label done;
3092 
3093       // If the frame is bigger than the stack, we throw the stack overflow
3094       // exception unconditionally. Thereby we can avoid the integer overflow
3095       // check in the condition code.
3096       if ((required_slots * kSystemPointerSize) < (FLAG_stack_size * 1024)) {
3097         __ Ld(
3098             kScratchReg,
3099             FieldMemOperand(kWasmInstanceRegister,
3100                             WasmInstanceObject::kRealStackLimitAddressOffset));
3101         __ Ld(kScratchReg, MemOperand(kScratchReg));
3102         __ Add64(kScratchReg, kScratchReg,
3103                  Operand(required_slots * kSystemPointerSize));
3104         __ BranchShort(&done, uge, sp, Operand(kScratchReg));
3105       }
3106 
3107       __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
3108       // We come from WebAssembly, there are no references for the GC.
3109       ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
3110       RecordSafepoint(reference_map);
3111       if (FLAG_debug_code) {
3112         __ stop();
3113       }
3114 
3115       __ bind(&done);
3116     }
3117   }
3118 
3119   const int returns = frame()->GetReturnSlotCount();
3120 
3121   // Skip callee-saved and return slots, which are pushed below.
3122   required_slots -= base::bits::CountPopulation(saves);
3123   required_slots -= base::bits::CountPopulation(saves_fpu);
3124   required_slots -= returns;
3125   if (required_slots > 0) {
3126     __ Sub64(sp, sp, Operand(required_slots * kSystemPointerSize));
3127   }
3128 
3129   if (saves_fpu != 0) {
3130     // Save callee-saved FPU registers.
3131     __ MultiPushFPU(saves_fpu);
3132     DCHECK_EQ(kNumCalleeSavedFPU, base::bits::CountPopulation(saves_fpu));
3133   }
3134 
3135   if (saves != 0) {
3136     // Save callee-saved registers.
3137     __ MultiPush(saves);
3138   }
3139 
3140   if (returns != 0) {
3141     // Create space for returns.
3142     __ Sub64(sp, sp, Operand(returns * kSystemPointerSize));
3143   }
3144 }
3145 
AssembleReturn(InstructionOperand * additional_pop_count)3146 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
3147   auto call_descriptor = linkage()->GetIncomingDescriptor();
3148 
3149   const int returns = frame()->GetReturnSlotCount();
3150   if (returns != 0) {
3151     __ Add64(sp, sp, Operand(returns * kSystemPointerSize));
3152   }
3153 
3154   // Restore GP registers.
3155   const RegList saves = call_descriptor->CalleeSavedRegisters();
3156   if (saves != 0) {
3157     __ MultiPop(saves);
3158   }
3159 
3160   // Restore FPU registers.
3161   const RegList saves_fpu = call_descriptor->CalleeSavedFPRegisters();
3162   if (saves_fpu != 0) {
3163     __ MultiPopFPU(saves_fpu);
3164   }
3165 
3166   RiscvOperandConverter g(this, nullptr);
3167 
3168   const int parameter_slots =
3169       static_cast<int>(call_descriptor->ParameterSlotCount());
3170 
3171   // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
3172   // Check RawMachineAssembler::PopAndReturn.
3173   if (parameter_slots != 0) {
3174     if (additional_pop_count->IsImmediate()) {
3175       DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
3176     } else if (FLAG_debug_code) {
3177       __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue,
3178                 g.ToRegister(additional_pop_count),
3179                 Operand(static_cast<int64_t>(0)));
3180     }
3181   }
3182 
3183   // Functions with JS linkage have at least one parameter (the receiver).
3184   // If {parameter_slots} == 0, it means it is a builtin with
3185   // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
3186   // itself.
3187   const bool drop_jsargs = frame_access_state()->has_frame() &&
3188                            call_descriptor->IsJSFunctionCall() &&
3189                            parameter_slots != 0;
3190 
3191   if (call_descriptor->IsCFunctionCall()) {
3192     AssembleDeconstructFrame();
3193   } else if (frame_access_state()->has_frame()) {
3194     // Canonicalize JSFunction return sites for now unless they have an variable
3195     // number of stack slot pops.
3196     if (additional_pop_count->IsImmediate() &&
3197         g.ToConstant(additional_pop_count).ToInt32() == 0) {
3198       if (return_label_.is_bound()) {
3199         __ Branch(&return_label_);
3200         return;
3201       } else {
3202         __ bind(&return_label_);
3203       }
3204     }
3205     if (drop_jsargs) {
3206       // Get the actual argument count
3207       __ Ld(t0, MemOperand(fp, StandardFrameConstants::kArgCOffset));
3208     }
3209     AssembleDeconstructFrame();
3210   }
3211   if (drop_jsargs) {
3212     // We must pop all arguments from the stack (including the receiver). This
3213     // number of arguments is given by max(1 + argc_reg, parameter_slots).
3214     __ Add64(t0, t0, Operand(1));  // Also pop the receiver.
3215     if (parameter_slots > 1) {
3216       Label done;
3217       __ li(kScratchReg, parameter_slots);
3218       __ BranchShort(&done, ge, t0, Operand(kScratchReg));
3219       __ Move(t0, kScratchReg);
3220       __ bind(&done);
3221     }
3222     __ Sll64(t0, t0, kSystemPointerSizeLog2);
3223     __ Add64(sp, sp, t0);
3224   } else if (additional_pop_count->IsImmediate()) {
3225     // it should be a kInt32 or a kInt64
3226     DCHECK_LE(g.ToConstant(additional_pop_count).type(), Constant::kInt64);
3227     int additional_count = g.ToConstant(additional_pop_count).ToInt32();
3228     __ Drop(parameter_slots + additional_count);
3229   } else {
3230     Register pop_reg = g.ToRegister(additional_pop_count);
3231     __ Drop(parameter_slots);
3232     __ Sll64(pop_reg, pop_reg, kSystemPointerSizeLog2);
3233     __ Add64(sp, sp, pop_reg);
3234   }
3235   __ Ret();
3236 }
3237 
FinishCode()3238 void CodeGenerator::FinishCode() { __ ForceConstantPoolEmissionWithoutJump(); }
3239 
PrepareForDeoptimizationExits(ZoneDeque<DeoptimizationExit * > * exits)3240 void CodeGenerator::PrepareForDeoptimizationExits(
3241     ZoneDeque<DeoptimizationExit*>* exits) {
3242   __ ForceConstantPoolEmissionWithoutJump();
3243   int total_size = 0;
3244   for (DeoptimizationExit* exit : deoptimization_exits_) {
3245     total_size += (exit->kind() == DeoptimizeKind::kLazy)
3246                       ? Deoptimizer::kLazyDeoptExitSize
3247                       : Deoptimizer::kNonLazyDeoptExitSize;
3248   }
3249 
3250   __ CheckTrampolinePoolQuick(total_size);
3251   DCHECK(Deoptimizer::kSupportsFixedDeoptExitSizes);
3252 }
3253 
AssembleMove(InstructionOperand * source,InstructionOperand * destination)3254 void CodeGenerator::AssembleMove(InstructionOperand* source,
3255                                  InstructionOperand* destination) {
3256   RiscvOperandConverter g(this, nullptr);
3257   // Dispatch on the source and destination operand kinds.  Not all
3258   // combinations are possible.
3259   if (source->IsRegister()) {
3260     DCHECK(destination->IsRegister() || destination->IsStackSlot());
3261     Register src = g.ToRegister(source);
3262     if (destination->IsRegister()) {
3263       __ Move(g.ToRegister(destination), src);
3264     } else {
3265       __ Sd(src, g.ToMemOperand(destination));
3266     }
3267   } else if (source->IsStackSlot()) {
3268     DCHECK(destination->IsRegister() || destination->IsStackSlot());
3269     MemOperand src = g.ToMemOperand(source);
3270     if (destination->IsRegister()) {
3271       __ Ld(g.ToRegister(destination), src);
3272     } else {
3273       Register temp = kScratchReg;
3274       __ Ld(temp, src);
3275       __ Sd(temp, g.ToMemOperand(destination));
3276     }
3277   } else if (source->IsConstant()) {
3278     Constant src = g.ToConstant(source);
3279     if (destination->IsRegister() || destination->IsStackSlot()) {
3280       Register dst =
3281           destination->IsRegister() ? g.ToRegister(destination) : kScratchReg;
3282       switch (src.type()) {
3283         case Constant::kInt32:
3284           if (src.ToInt32() == 0 && destination->IsStackSlot()) {
3285             dst = zero_reg;
3286           } else {
3287             __ li(dst, Operand(src.ToInt32()));
3288           }
3289           break;
3290         case Constant::kFloat32:
3291           __ li(dst, Operand::EmbeddedNumber(src.ToFloat32()));
3292           break;
3293         case Constant::kInt64:
3294           if (RelocInfo::IsWasmReference(src.rmode())) {
3295             __ li(dst, Operand(src.ToInt64(), src.rmode()));
3296           } else {
3297             if (src.ToInt64() == 0 && destination->IsStackSlot()) {
3298               dst = zero_reg;
3299             } else {
3300               __ li(dst, Operand(src.ToInt64()));
3301             }
3302           }
3303           break;
3304         case Constant::kFloat64:
3305           __ li(dst, Operand::EmbeddedNumber(src.ToFloat64().value()));
3306           break;
3307         case Constant::kExternalReference:
3308           __ li(dst, src.ToExternalReference());
3309           break;
3310         case Constant::kDelayedStringConstant:
3311           __ li(dst, src.ToDelayedStringConstant());
3312           break;
3313         case Constant::kHeapObject: {
3314           Handle<HeapObject> src_object = src.ToHeapObject();
3315           RootIndex index;
3316           if (IsMaterializableFromRoot(src_object, &index)) {
3317             __ LoadRoot(dst, index);
3318           } else {
3319             __ li(dst, src_object);
3320           }
3321           break;
3322         }
3323         case Constant::kCompressedHeapObject: {
3324           Handle<HeapObject> src_object = src.ToHeapObject();
3325           RootIndex index;
3326           if (IsMaterializableFromRoot(src_object, &index)) {
3327             __ LoadRoot(dst, index);
3328           } else {
3329             __ li(dst, src_object, RelocInfo::COMPRESSED_EMBEDDED_OBJECT);
3330           }
3331           break;
3332         }
3333         case Constant::kRpoNumber:
3334           UNREACHABLE();  // TODO(titzer): loading RPO numbers
3335       }
3336       if (destination->IsStackSlot()) __ Sd(dst, g.ToMemOperand(destination));
3337     } else if (src.type() == Constant::kFloat32) {
3338       if (destination->IsFPStackSlot()) {
3339         MemOperand dst = g.ToMemOperand(destination);
3340         if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
3341           __ Sw(zero_reg, dst);
3342         } else {
3343           __ li(kScratchReg, Operand(bit_cast<int32_t>(src.ToFloat32())));
3344           __ Sw(kScratchReg, dst);
3345         }
3346       } else {
3347         DCHECK(destination->IsFPRegister());
3348         FloatRegister dst = g.ToSingleRegister(destination);
3349         __ LoadFPRImmediate(dst, src.ToFloat32());
3350       }
3351     } else {
3352       DCHECK_EQ(Constant::kFloat64, src.type());
3353       DoubleRegister dst = destination->IsFPRegister()
3354                                ? g.ToDoubleRegister(destination)
3355                                : kScratchDoubleReg;
3356       __ LoadFPRImmediate(dst, src.ToFloat64().value());
3357       if (destination->IsFPStackSlot()) {
3358         __ StoreDouble(dst, g.ToMemOperand(destination));
3359       }
3360     }
3361   } else if (source->IsFPRegister()) {
3362     MachineRepresentation rep = LocationOperand::cast(source)->representation();
3363     if (rep == MachineRepresentation::kSimd128) {
3364       VRegister src = g.ToSimd128Register(source);
3365       if (destination->IsSimd128Register()) {
3366         VRegister dst = g.ToSimd128Register(destination);
3367         __ vmv_vv(dst, src);
3368       } else {
3369         DCHECK(destination->IsSimd128StackSlot());
3370         Register dst = g.ToMemOperand(destination).offset() == 0
3371                            ? g.ToMemOperand(destination).rm()
3372                            : kScratchReg;
3373         if (g.ToMemOperand(destination).offset() != 0) {
3374           __ Add64(dst, g.ToMemOperand(destination).rm(),
3375                    g.ToMemOperand(destination).offset());
3376         }
3377         __ vs(src, dst, 0, E8);
3378       }
3379     } else {
3380       FPURegister src = g.ToDoubleRegister(source);
3381       if (destination->IsFPRegister()) {
3382         FPURegister dst = g.ToDoubleRegister(destination);
3383         __ Move(dst, src);
3384       } else {
3385         DCHECK(destination->IsFPStackSlot());
3386         if (rep == MachineRepresentation::kFloat32) {
3387           __ StoreFloat(src, g.ToMemOperand(destination));
3388         } else {
3389           DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3390           __ StoreDouble(src, g.ToMemOperand(destination));
3391         }
3392       }
3393     }
3394   } else if (source->IsFPStackSlot()) {
3395     DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
3396     MemOperand src = g.ToMemOperand(source);
3397     MachineRepresentation rep = LocationOperand::cast(source)->representation();
3398     if (rep == MachineRepresentation::kSimd128) {
3399       Register src_reg = src.offset() == 0 ? src.rm() : kScratchReg;
3400       if (src.offset() != 0) {
3401         __ Add64(src_reg, src.rm(), src.offset());
3402       }
3403       if (destination->IsSimd128Register()) {
3404         __ vl(g.ToSimd128Register(destination), src_reg, 0, E8);
3405       } else {
3406         DCHECK(destination->IsSimd128StackSlot());
3407         VRegister temp = kSimd128ScratchReg;
3408         Register dst = g.ToMemOperand(destination).offset() == 0
3409                            ? g.ToMemOperand(destination).rm()
3410                            : kScratchReg;
3411         if (g.ToMemOperand(destination).offset() != 0) {
3412           __ Add64(dst, g.ToMemOperand(destination).rm(),
3413                    g.ToMemOperand(destination).offset());
3414         }
3415         __ vl(temp, src_reg, 0, E8);
3416         __ vs(temp, dst, 0, E8);
3417       }
3418     } else {
3419       if (destination->IsFPRegister()) {
3420         if (rep == MachineRepresentation::kFloat32) {
3421           __ LoadFloat(g.ToDoubleRegister(destination), src);
3422         } else {
3423           DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3424           __ LoadDouble(g.ToDoubleRegister(destination), src);
3425         }
3426       } else {
3427         DCHECK(destination->IsFPStackSlot());
3428         FPURegister temp = kScratchDoubleReg;
3429         if (rep == MachineRepresentation::kFloat32) {
3430           __ LoadFloat(temp, src);
3431           __ StoreFloat(temp, g.ToMemOperand(destination));
3432         } else {
3433           DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3434           __ LoadDouble(temp, src);
3435           __ StoreDouble(temp, g.ToMemOperand(destination));
3436         }
3437       }
3438     }
3439   } else {
3440     UNREACHABLE();
3441   }
3442 }
3443 
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)3444 void CodeGenerator::AssembleSwap(InstructionOperand* source,
3445                                  InstructionOperand* destination) {
3446   RiscvOperandConverter g(this, nullptr);
3447   // Dispatch on the source and destination operand kinds.  Not all
3448   // combinations are possible.
3449   if (source->IsRegister()) {
3450     // Register-register.
3451     Register temp = kScratchReg;
3452     Register src = g.ToRegister(source);
3453     if (destination->IsRegister()) {
3454       Register dst = g.ToRegister(destination);
3455       __ Move(temp, src);
3456       __ Move(src, dst);
3457       __ Move(dst, temp);
3458     } else {
3459       DCHECK(destination->IsStackSlot());
3460       MemOperand dst = g.ToMemOperand(destination);
3461       __ Move(temp, src);
3462       __ Ld(src, dst);
3463       __ Sd(temp, dst);
3464     }
3465   } else if (source->IsStackSlot()) {
3466     DCHECK(destination->IsStackSlot());
3467     Register temp_0 = kScratchReg;
3468     Register temp_1 = kScratchReg2;
3469     MemOperand src = g.ToMemOperand(source);
3470     MemOperand dst = g.ToMemOperand(destination);
3471     __ Ld(temp_0, src);
3472     __ Ld(temp_1, dst);
3473     __ Sd(temp_0, dst);
3474     __ Sd(temp_1, src);
3475   } else if (source->IsFPRegister()) {
3476     MachineRepresentation rep = LocationOperand::cast(source)->representation();
3477     if (rep == MachineRepresentation::kSimd128) {
3478       UNIMPLEMENTED();
3479     } else {
3480       FPURegister temp = kScratchDoubleReg;
3481       FPURegister src = g.ToDoubleRegister(source);
3482       if (destination->IsFPRegister()) {
3483         FPURegister dst = g.ToDoubleRegister(destination);
3484         __ Move(temp, src);
3485         __ Move(src, dst);
3486         __ Move(dst, temp);
3487       } else {
3488         DCHECK(destination->IsFPStackSlot());
3489         MemOperand dst = g.ToMemOperand(destination);
3490         if (rep == MachineRepresentation::kFloat32) {
3491           __ MoveFloat(temp, src);
3492           __ LoadFloat(src, dst);
3493           __ StoreFloat(temp, dst);
3494         } else {
3495           DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3496           __ MoveDouble(temp, src);
3497           __ LoadDouble(src, dst);
3498           __ StoreDouble(temp, dst);
3499         }
3500       }
3501     }
3502   } else if (source->IsFPStackSlot()) {
3503     DCHECK(destination->IsFPStackSlot());
3504     Register temp_0 = kScratchReg;
3505     MemOperand src0 = g.ToMemOperand(source);
3506     MemOperand src1(src0.rm(), src0.offset() + kIntSize);
3507     MemOperand dst0 = g.ToMemOperand(destination);
3508     MemOperand dst1(dst0.rm(), dst0.offset() + kIntSize);
3509     MachineRepresentation rep = LocationOperand::cast(source)->representation();
3510     if (rep == MachineRepresentation::kSimd128) {
3511       UNIMPLEMENTED();
3512     } else {
3513       FPURegister temp_1 = kScratchDoubleReg;
3514       if (rep == MachineRepresentation::kFloat32) {
3515         __ LoadFloat(temp_1, dst0);  // Save destination in temp_1.
3516         __ Lw(temp_0, src0);  // Then use temp_0 to copy source to destination.
3517         __ Sw(temp_0, dst0);
3518         __ StoreFloat(temp_1, src0);
3519       } else {
3520         DCHECK_EQ(rep, MachineRepresentation::kFloat64);
3521         __ LoadDouble(temp_1, dst0);  // Save destination in temp_1.
3522         __ Lw(temp_0, src0);  // Then use temp_0 to copy source to destination.
3523         __ Sw(temp_0, dst0);
3524         __ Lw(temp_0, src1);
3525         __ Sw(temp_0, dst1);
3526         __ StoreDouble(temp_1, src0);
3527       }
3528     }
3529   } else {
3530     // No other combinations are possible.
3531     UNREACHABLE();
3532   }
3533 }
3534 
AssembleJumpTable(Label ** targets,size_t target_count)3535 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3536   // On 64-bit RISC-V we emit the jump tables inline.
3537   UNREACHABLE();
3538 }
3539 
3540 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
3541 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
3542 #undef ASSEMBLE_ATOMIC_BINOP
3543 #undef ASSEMBLE_ATOMIC_BINOP_EXT
3544 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
3545 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT
3546 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
3547 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT
3548 #undef ASSEMBLE_IEEE754_BINOP
3549 #undef ASSEMBLE_IEEE754_UNOP
3550 
3551 #undef TRACE_MSG
3552 #undef TRACE_UNIMPL
3553 #undef __
3554 
3555 }  // namespace compiler
3556 }  // namespace internal
3557 }  // namespace v8
3558