1/*
2 *
3 * Copyright (c) 2020 T-Head Semiconductor Co., Ltd. All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2 or later, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program.  If not, see <http://www.gnu.org/licenses/>.
16 */
17#include "tcg/tcg-op-gvec.h"
18#include "tcg/tcg-gvec-desc.h"
19#include "internals.h"
20
21static inline bool is_overlapped(const int8_t astart, int8_t asize,
22                                 const int8_t bstart, int8_t bsize)
23{
24    const int8_t aend = astart + asize;
25    const int8_t bend = bstart + bsize;
26
27    return MAX(aend, bend) - MIN(astart, bstart) < asize + bsize;
28}
29
30static bool require_rvv(DisasContext *s)
31{
32    return s->mstatus_vs != 0;
33}
34
35static bool require_rvf(DisasContext *s)
36{
37    if (s->mstatus_fs == 0) {
38        return false;
39    }
40
41    switch (s->sew) {
42    case MO_16:
43    case MO_32:
44        return has_ext(s, RVF);
45    case MO_64:
46        return has_ext(s, RVD);
47    default:
48        return false;
49    }
50}
51
52static bool require_scale_rvf(DisasContext *s)
53{
54    if (s->mstatus_fs == 0) {
55        return false;
56    }
57
58    switch (s->sew) {
59    case MO_8:
60    case MO_16:
61        return has_ext(s, RVF);
62    case MO_32:
63        return has_ext(s, RVD);
64    default:
65        return false;
66    }
67}
68
69static bool require_zve32f(DisasContext *s)
70{
71    /* RVV + Zve32f = RVV. */
72    if (has_ext(s, RVV)) {
73        return true;
74    }
75
76    /* Zve32f doesn't support FP64. (Section 18.2) */
77    return s->cfg_ptr->ext_zve32f ? s->sew <= MO_32 : true;
78}
79
80static bool require_scale_zve32f(DisasContext *s)
81{
82    /* RVV + Zve32f = RVV. */
83    if (has_ext(s, RVV)) {
84        return true;
85    }
86
87    /* Zve32f doesn't support FP64. (Section 18.2) */
88    return s->cfg_ptr->ext_zve64f ? s->sew <= MO_16 : true;
89}
90
91static bool require_zve64f(DisasContext *s)
92{
93    /* RVV + Zve64f = RVV. */
94    if (has_ext(s, RVV)) {
95        return true;
96    }
97
98    /* Zve64f doesn't support FP64. (Section 18.2) */
99    return s->cfg_ptr->ext_zve64f ? s->sew <= MO_32 : true;
100}
101
102static bool require_scale_zve64f(DisasContext *s)
103{
104    /* RVV + Zve64f = RVV. */
105    if (has_ext(s, RVV)) {
106        return true;
107    }
108
109    /* Zve64f doesn't support FP64. (Section 18.2) */
110    return s->cfg_ptr->ext_zve64f ? s->sew <= MO_16 : true;
111}
112
113/* Destination vector register group cannot overlap source mask register. */
114static bool require_vm(int vm, int vd)
115{
116    return (vm != 0 || vd != 0);
117}
118
119static bool require_nf(int vd, int nf, int lmul)
120{
121    int size = nf << MAX(lmul, 0);
122    return size <= 8 && vd + size <= 32;
123}
124
125/*
126 * Vector register should aligned with the passed-in LMUL (EMUL).
127 * If LMUL < 0, i.e. fractional LMUL, any vector register is allowed.
128 */
129static bool require_align(const int8_t val, const int8_t lmul)
130{
131    return lmul <= 0 || extract32(val, 0, lmul) == 0;
132}
133
134/*
135 * A destination vector register group can overlap a source vector
136 * register group only if one of the following holds:
137 *  1. The destination EEW equals the source EEW.
138 *  2. The destination EEW is smaller than the source EEW and the overlap
139 *     is in the lowest-numbered part of the source register group.
140 *  3. The destination EEW is greater than the source EEW, the source EMUL
141 *     is at least 1, and the overlap is in the highest-numbered part of
142 *     the destination register group.
143 * (Section 5.2)
144 *
145 * This function returns true if one of the following holds:
146 *  * Destination vector register group does not overlap a source vector
147 *    register group.
148 *  * Rule 3 met.
149 * For rule 1, overlap is allowed so this function doesn't need to be called.
150 * For rule 2, (vd == vs). Caller has to check whether: (vd != vs) before
151 * calling this function.
152 */
153static bool require_noover(const int8_t dst, const int8_t dst_lmul,
154                           const int8_t src, const int8_t src_lmul)
155{
156    int8_t dst_size = dst_lmul <= 0 ? 1 : 1 << dst_lmul;
157    int8_t src_size = src_lmul <= 0 ? 1 : 1 << src_lmul;
158
159    /* Destination EEW is greater than the source EEW, check rule 3. */
160    if (dst_size > src_size) {
161        if (dst < src &&
162            src_lmul >= 0 &&
163            is_overlapped(dst, dst_size, src, src_size) &&
164            !is_overlapped(dst, dst_size, src + src_size, src_size)) {
165            return true;
166        }
167    }
168
169    return !is_overlapped(dst, dst_size, src, src_size);
170}
171
172static bool do_vsetvl(DisasContext *s, int rd, int rs1, TCGv s2)
173{
174    TCGv s1, dst;
175
176    if (!require_rvv(s) ||
177        !(has_ext(s, RVV) || s->cfg_ptr->ext_zve32f ||
178          s->cfg_ptr->ext_zve64f)) {
179        return false;
180    }
181
182    dst = dest_gpr(s, rd);
183
184    if (rd == 0 && rs1 == 0) {
185        s1 = tcg_temp_new();
186        tcg_gen_mov_tl(s1, cpu_vl);
187    } else if (rs1 == 0) {
188        /* As the mask is at least one bit, RV_VLEN_MAX is >= VLMAX */
189        s1 = tcg_constant_tl(RV_VLEN_MAX);
190    } else {
191        s1 = get_gpr(s, rs1, EXT_ZERO);
192    }
193
194    gen_helper_vsetvl(dst, cpu_env, s1, s2);
195    gen_set_gpr(s, rd, dst);
196    mark_vs_dirty(s);
197
198    gen_set_pc_imm(s, s->pc_succ_insn);
199    lookup_and_goto_ptr(s);
200    s->base.is_jmp = DISAS_NORETURN;
201
202    if (rd == 0 && rs1 == 0) {
203        tcg_temp_free(s1);
204    }
205
206    return true;
207}
208
209static bool do_vsetivli(DisasContext *s, int rd, TCGv s1, TCGv s2)
210{
211    TCGv dst;
212
213    if (!require_rvv(s) ||
214        !(has_ext(s, RVV) || s->cfg_ptr->ext_zve32f ||
215          s->cfg_ptr->ext_zve64f)) {
216        return false;
217    }
218
219    dst = dest_gpr(s, rd);
220
221    gen_helper_vsetvl(dst, cpu_env, s1, s2);
222    gen_set_gpr(s, rd, dst);
223    mark_vs_dirty(s);
224    gen_set_pc_imm(s, s->pc_succ_insn);
225    lookup_and_goto_ptr(s);
226    s->base.is_jmp = DISAS_NORETURN;
227
228    return true;
229}
230
231static bool trans_vsetvl(DisasContext *s, arg_vsetvl *a)
232{
233    TCGv s2 = get_gpr(s, a->rs2, EXT_ZERO);
234    return do_vsetvl(s, a->rd, a->rs1, s2);
235}
236
237static bool trans_vsetvli(DisasContext *s, arg_vsetvli *a)
238{
239    TCGv s2 = tcg_constant_tl(a->zimm);
240    return do_vsetvl(s, a->rd, a->rs1, s2);
241}
242
243static bool trans_vsetivli(DisasContext *s, arg_vsetivli *a)
244{
245    TCGv s1 = tcg_const_tl(a->rs1);
246    TCGv s2 = tcg_const_tl(a->zimm);
247    return do_vsetivli(s, a->rd, s1, s2);
248}
249
250/* vector register offset from env */
251static uint32_t vreg_ofs(DisasContext *s, int reg)
252{
253    return offsetof(CPURISCVState, vreg) + reg * s->cfg_ptr->vlen / 8;
254}
255
256/* check functions */
257
258/*
259 * Vector unit-stride, strided, unit-stride segment, strided segment
260 * store check function.
261 *
262 * Rules to be checked here:
263 *   1. EMUL must within the range: 1/8 <= EMUL <= 8. (Section 7.3)
264 *   2. Destination vector register number is multiples of EMUL.
265 *      (Section 3.4.2, 7.3)
266 *   3. The EMUL setting must be such that EMUL * NFIELDS ≤ 8. (Section 7.8)
267 *   4. Vector register numbers accessed by the segment load or store
268 *      cannot increment past 31. (Section 7.8)
269 */
270static bool vext_check_store(DisasContext *s, int vd, int nf, uint8_t eew)
271{
272    int8_t emul = eew - s->sew + s->lmul;
273    return (emul >= -3 && emul <= 3) &&
274            require_align(vd, emul) &&
275            require_nf(vd, nf, emul);
276}
277
278/*
279 * Vector unit-stride, strided, unit-stride segment, strided segment
280 * load check function.
281 *
282 * Rules to be checked here:
283 *   1. All rules applies to store instructions are applies
284 *      to load instructions.
285 *   2. Destination vector register group for a masked vector
286 *      instruction cannot overlap the source mask register (v0).
287 *      (Section 5.3)
288 */
289static bool vext_check_load(DisasContext *s, int vd, int nf, int vm,
290                            uint8_t eew)
291{
292    return vext_check_store(s, vd, nf, eew) && require_vm(vm, vd);
293}
294
295/*
296 * Vector indexed, indexed segment store check function.
297 *
298 * Rules to be checked here:
299 *   1. EMUL must within the range: 1/8 <= EMUL <= 8. (Section 7.3)
300 *   2. Index vector register number is multiples of EMUL.
301 *      (Section 3.4.2, 7.3)
302 *   3. Destination vector register number is multiples of LMUL.
303 *      (Section 3.4.2, 7.3)
304 *   4. The EMUL setting must be such that EMUL * NFIELDS ≤ 8. (Section 7.8)
305 *   5. Vector register numbers accessed by the segment load or store
306 *      cannot increment past 31. (Section 7.8)
307 */
308static bool vext_check_st_index(DisasContext *s, int vd, int vs2, int nf,
309                                uint8_t eew)
310{
311    int8_t emul = eew - s->sew + s->lmul;
312    bool ret = (emul >= -3 && emul <= 3) &&
313               require_align(vs2, emul) &&
314               require_align(vd, s->lmul) &&
315               require_nf(vd, nf, s->lmul);
316
317    /*
318     * All Zve* extensions support all vector load and store instructions,
319     * except Zve64* extensions do not support EEW=64 for index values
320     * when XLEN=32. (Section 18.2)
321     */
322    if (get_xl(s) == MXL_RV32) {
323        ret &= (!has_ext(s, RVV) &&
324                s->cfg_ptr->ext_zve64f ? eew != MO_64 : true);
325    }
326
327    return ret;
328}
329
330/*
331 * Vector indexed, indexed segment load check function.
332 *
333 * Rules to be checked here:
334 *   1. All rules applies to store instructions are applies
335 *      to load instructions.
336 *   2. Destination vector register group for a masked vector
337 *      instruction cannot overlap the source mask register (v0).
338 *      (Section 5.3)
339 *   3. Destination vector register cannot overlap a source vector
340 *      register (vs2) group.
341 *      (Section 5.2)
342 *   4. Destination vector register groups cannot overlap
343 *      the source vector register (vs2) group for
344 *      indexed segment load instructions. (Section 7.8.3)
345 */
346static bool vext_check_ld_index(DisasContext *s, int vd, int vs2,
347                                int nf, int vm, uint8_t eew)
348{
349    int8_t seg_vd;
350    int8_t emul = eew - s->sew + s->lmul;
351    bool ret = vext_check_st_index(s, vd, vs2, nf, eew) &&
352        require_vm(vm, vd);
353
354    /* Each segment register group has to follow overlap rules. */
355    for (int i = 0; i < nf; ++i) {
356        seg_vd = vd + (1 << MAX(s->lmul, 0)) * i;
357
358        if (eew > s->sew) {
359            if (seg_vd != vs2) {
360                ret &= require_noover(seg_vd, s->lmul, vs2, emul);
361            }
362        } else if (eew < s->sew) {
363            ret &= require_noover(seg_vd, s->lmul, vs2, emul);
364        }
365
366        /*
367         * Destination vector register groups cannot overlap
368         * the source vector register (vs2) group for
369         * indexed segment load instructions.
370         */
371        if (nf > 1) {
372            ret &= !is_overlapped(seg_vd, 1 << MAX(s->lmul, 0),
373                                  vs2, 1 << MAX(emul, 0));
374        }
375    }
376    return ret;
377}
378
379static bool vext_check_ss(DisasContext *s, int vd, int vs, int vm)
380{
381    return require_vm(vm, vd) &&
382        require_align(vd, s->lmul) &&
383        require_align(vs, s->lmul);
384}
385
386/*
387 * Check function for vector instruction with format:
388 * single-width result and single-width sources (SEW = SEW op SEW)
389 *
390 * Rules to be checked here:
391 *   1. Destination vector register group for a masked vector
392 *      instruction cannot overlap the source mask register (v0).
393 *      (Section 5.3)
394 *   2. Destination vector register number is multiples of LMUL.
395 *      (Section 3.4.2)
396 *   3. Source (vs2, vs1) vector register number are multiples of LMUL.
397 *      (Section 3.4.2)
398 */
399static bool vext_check_sss(DisasContext *s, int vd, int vs1, int vs2, int vm)
400{
401    return vext_check_ss(s, vd, vs2, vm) &&
402        require_align(vs1, s->lmul);
403}
404
405static bool vext_check_ms(DisasContext *s, int vd, int vs)
406{
407    bool ret = require_align(vs, s->lmul);
408    if (vd != vs) {
409        ret &= require_noover(vd, 0, vs, s->lmul);
410    }
411    return ret;
412}
413
414/*
415 * Check function for maskable vector instruction with format:
416 * single-width result and single-width sources (SEW = SEW op SEW)
417 *
418 * Rules to be checked here:
419 *   1. Source (vs2, vs1) vector register number are multiples of LMUL.
420 *      (Section 3.4.2)
421 *   2. Destination vector register cannot overlap a source vector
422 *      register (vs2, vs1) group.
423 *      (Section 5.2)
424 *   3. The destination vector register group for a masked vector
425 *      instruction cannot overlap the source mask register (v0),
426 *      unless the destination vector register is being written
427 *      with a mask value (e.g., comparisons) or the scalar result
428 *      of a reduction. (Section 5.3)
429 */
430static bool vext_check_mss(DisasContext *s, int vd, int vs1, int vs2)
431{
432    bool ret = vext_check_ms(s, vd, vs2) &&
433        require_align(vs1, s->lmul);
434    if (vd != vs1) {
435        ret &= require_noover(vd, 0, vs1, s->lmul);
436    }
437    return ret;
438}
439
440/*
441 * Common check function for vector widening instructions
442 * of double-width result (2*SEW).
443 *
444 * Rules to be checked here:
445 *   1. The largest vector register group used by an instruction
446 *      can not be greater than 8 vector registers (Section 5.2):
447 *      => LMUL < 8.
448 *      => SEW < 64.
449 *   2. Double-width SEW cannot greater than ELEN.
450 *   3. Destination vector register number is multiples of 2 * LMUL.
451 *      (Section 3.4.2)
452 *   4. Destination vector register group for a masked vector
453 *      instruction cannot overlap the source mask register (v0).
454 *      (Section 5.3)
455 */
456static bool vext_wide_check_common(DisasContext *s, int vd, int vm)
457{
458    return (s->lmul <= 2) &&
459           (s->sew < MO_64) &&
460           ((s->sew + 1) <= (s->cfg_ptr->elen >> 4)) &&
461           require_align(vd, s->lmul + 1) &&
462           require_vm(vm, vd);
463}
464
465/*
466 * Common check function for vector narrowing instructions
467 * of single-width result (SEW) and double-width source (2*SEW).
468 *
469 * Rules to be checked here:
470 *   1. The largest vector register group used by an instruction
471 *      can not be greater than 8 vector registers (Section 5.2):
472 *      => LMUL < 8.
473 *      => SEW < 64.
474 *   2. Double-width SEW cannot greater than ELEN.
475 *   3. Source vector register number is multiples of 2 * LMUL.
476 *      (Section 3.4.2)
477 *   4. Destination vector register number is multiples of LMUL.
478 *      (Section 3.4.2)
479 *   5. Destination vector register group for a masked vector
480 *      instruction cannot overlap the source mask register (v0).
481 *      (Section 5.3)
482 */
483static bool vext_narrow_check_common(DisasContext *s, int vd, int vs2,
484                                     int vm)
485{
486    return (s->lmul <= 2) &&
487           (s->sew < MO_64) &&
488           ((s->sew + 1) <= (s->cfg_ptr->elen >> 4)) &&
489           require_align(vs2, s->lmul + 1) &&
490           require_align(vd, s->lmul) &&
491           require_vm(vm, vd);
492}
493
494static bool vext_check_ds(DisasContext *s, int vd, int vs, int vm)
495{
496    return vext_wide_check_common(s, vd, vm) &&
497        require_align(vs, s->lmul) &&
498        require_noover(vd, s->lmul + 1, vs, s->lmul);
499}
500
501static bool vext_check_dd(DisasContext *s, int vd, int vs, int vm)
502{
503    return vext_wide_check_common(s, vd, vm) &&
504        require_align(vs, s->lmul + 1);
505}
506
507/*
508 * Check function for vector instruction with format:
509 * double-width result and single-width sources (2*SEW = SEW op SEW)
510 *
511 * Rules to be checked here:
512 *   1. All rules in defined in widen common rules are applied.
513 *   2. Source (vs2, vs1) vector register number are multiples of LMUL.
514 *      (Section 3.4.2)
515 *   3. Destination vector register cannot overlap a source vector
516 *      register (vs2, vs1) group.
517 *      (Section 5.2)
518 */
519static bool vext_check_dss(DisasContext *s, int vd, int vs1, int vs2, int vm)
520{
521    return vext_check_ds(s, vd, vs2, vm) &&
522        require_align(vs1, s->lmul) &&
523        require_noover(vd, s->lmul + 1, vs1, s->lmul);
524}
525
526/*
527 * Check function for vector instruction with format:
528 * double-width result and double-width source1 and single-width
529 * source2 (2*SEW = 2*SEW op SEW)
530 *
531 * Rules to be checked here:
532 *   1. All rules in defined in widen common rules are applied.
533 *   2. Source 1 (vs2) vector register number is multiples of 2 * LMUL.
534 *      (Section 3.4.2)
535 *   3. Source 2 (vs1) vector register number is multiples of LMUL.
536 *      (Section 3.4.2)
537 *   4. Destination vector register cannot overlap a source vector
538 *      register (vs1) group.
539 *      (Section 5.2)
540 */
541static bool vext_check_dds(DisasContext *s, int vd, int vs1, int vs2, int vm)
542{
543    return vext_check_ds(s, vd, vs1, vm) &&
544        require_align(vs2, s->lmul + 1);
545}
546
547static bool vext_check_sd(DisasContext *s, int vd, int vs, int vm)
548{
549    bool ret = vext_narrow_check_common(s, vd, vs, vm);
550    if (vd != vs) {
551        ret &= require_noover(vd, s->lmul, vs, s->lmul + 1);
552    }
553    return ret;
554}
555
556/*
557 * Check function for vector instruction with format:
558 * single-width result and double-width source 1 and single-width
559 * source 2 (SEW = 2*SEW op SEW)
560 *
561 * Rules to be checked here:
562 *   1. All rules in defined in narrow common rules are applied.
563 *   2. Destination vector register cannot overlap a source vector
564 *      register (vs2) group.
565 *      (Section 5.2)
566 *   3. Source 2 (vs1) vector register number is multiples of LMUL.
567 *      (Section 3.4.2)
568 */
569static bool vext_check_sds(DisasContext *s, int vd, int vs1, int vs2, int vm)
570{
571    return vext_check_sd(s, vd, vs2, vm) &&
572        require_align(vs1, s->lmul);
573}
574
575/*
576 * Check function for vector reduction instructions.
577 *
578 * Rules to be checked here:
579 *   1. Source 1 (vs2) vector register number is multiples of LMUL.
580 *      (Section 3.4.2)
581 */
582static bool vext_check_reduction(DisasContext *s, int vs2)
583{
584    return require_align(vs2, s->lmul) && (s->vstart == 0);
585}
586
587/*
588 * Check function for vector slide instructions.
589 *
590 * Rules to be checked here:
591 *   1. Source 1 (vs2) vector register number is multiples of LMUL.
592 *      (Section 3.4.2)
593 *   2. Destination vector register number is multiples of LMUL.
594 *      (Section 3.4.2)
595 *   3. Destination vector register group for a masked vector
596 *      instruction cannot overlap the source mask register (v0).
597 *      (Section 5.3)
598 *   4. The destination vector register group for vslideup, vslide1up,
599 *      vfslide1up, cannot overlap the source vector register (vs2) group.
600 *      (Section 5.2, 16.3.1, 16.3.3)
601 */
602static bool vext_check_slide(DisasContext *s, int vd, int vs2,
603                             int vm, bool is_over)
604{
605    bool ret = require_align(vs2, s->lmul) &&
606               require_align(vd, s->lmul) &&
607               require_vm(vm, vd);
608    if (is_over) {
609        ret &= (vd != vs2);
610    }
611    return ret;
612}
613
614/*
615 * In cpu_get_tb_cpu_state(), set VILL if RVV was not present.
616 * So RVV is also be checked in this function.
617 */
618static bool vext_check_isa_ill(DisasContext *s)
619{
620    return !s->vill;
621}
622
623/* common translation macro */
624#define GEN_VEXT_TRANS(NAME, EEW, ARGTYPE, OP, CHECK)        \
625static bool trans_##NAME(DisasContext *s, arg_##ARGTYPE * a) \
626{                                                            \
627    if (CHECK(s, a, EEW)) {                                  \
628        return OP(s, a, EEW);                                \
629    }                                                        \
630    return false;                                            \
631}
632
633static uint8_t vext_get_emul(DisasContext *s, uint8_t eew)
634{
635    int8_t emul = eew - s->sew + s->lmul;
636    return emul < 0 ? 0 : emul;
637}
638
639/*
640 *** unit stride load and store
641 */
642typedef void gen_helper_ldst_us(TCGv_ptr, TCGv_ptr, TCGv,
643                                TCGv_env, TCGv_i32);
644
645static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data,
646                          gen_helper_ldst_us *fn, DisasContext *s,
647                          bool is_store)
648{
649    TCGv_ptr dest, mask;
650    TCGv base;
651    TCGv_i32 desc;
652
653    TCGLabel *over = gen_new_label();
654    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
655    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
656
657    dest = tcg_temp_new_ptr();
658    mask = tcg_temp_new_ptr();
659    base = get_gpr(s, rs1, EXT_NONE);
660
661    /*
662     * As simd_desc supports at most 2048 bytes, and in this implementation,
663     * the max vector group length is 4096 bytes. So split it into two parts.
664     *
665     * The first part is vlen in bytes, encoded in maxsz of simd_desc.
666     * The second part is lmul, encoded in data of simd_desc.
667     */
668    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
669                                      s->cfg_ptr->vlen / 8, data));
670
671    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
672    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
673
674    fn(dest, mask, base, cpu_env, desc);
675
676    tcg_temp_free_ptr(dest);
677    tcg_temp_free_ptr(mask);
678
679    if (!is_store) {
680        mark_vs_dirty(s);
681    }
682
683    gen_set_label(over);
684    return true;
685}
686
687static bool ld_us_op(DisasContext *s, arg_r2nfvm *a, uint8_t eew)
688{
689    uint32_t data = 0;
690    gen_helper_ldst_us *fn;
691    static gen_helper_ldst_us * const fns[2][4] = {
692        /* masked unit stride load */
693        { gen_helper_vle8_v_mask, gen_helper_vle16_v_mask,
694          gen_helper_vle32_v_mask, gen_helper_vle64_v_mask },
695        /* unmasked unit stride load */
696        { gen_helper_vle8_v, gen_helper_vle16_v,
697          gen_helper_vle32_v, gen_helper_vle64_v }
698    };
699
700    fn =  fns[a->vm][eew];
701    if (fn == NULL) {
702        return false;
703    }
704
705    /*
706     * Vector load/store instructions have the EEW encoded
707     * directly in the instructions. The maximum vector size is
708     * calculated with EMUL rather than LMUL.
709     */
710    uint8_t emul = vext_get_emul(s, eew);
711    data = FIELD_DP32(data, VDATA, VM, a->vm);
712    data = FIELD_DP32(data, VDATA, LMUL, emul);
713    data = FIELD_DP32(data, VDATA, NF, a->nf);
714    data = FIELD_DP32(data, VDATA, VTA, s->vta);
715    data = FIELD_DP32(data, VDATA, VMA, s->vma);
716    return ldst_us_trans(a->rd, a->rs1, data, fn, s, false);
717}
718
719static bool ld_us_check(DisasContext *s, arg_r2nfvm* a, uint8_t eew)
720{
721    return require_rvv(s) &&
722           vext_check_isa_ill(s) &&
723           vext_check_load(s, a->rd, a->nf, a->vm, eew);
724}
725
726GEN_VEXT_TRANS(vle8_v,  MO_8,  r2nfvm, ld_us_op, ld_us_check)
727GEN_VEXT_TRANS(vle16_v, MO_16, r2nfvm, ld_us_op, ld_us_check)
728GEN_VEXT_TRANS(vle32_v, MO_32, r2nfvm, ld_us_op, ld_us_check)
729GEN_VEXT_TRANS(vle64_v, MO_64, r2nfvm, ld_us_op, ld_us_check)
730
731static bool st_us_op(DisasContext *s, arg_r2nfvm *a, uint8_t eew)
732{
733    uint32_t data = 0;
734    gen_helper_ldst_us *fn;
735    static gen_helper_ldst_us * const fns[2][4] = {
736        /* masked unit stride store */
737        { gen_helper_vse8_v_mask, gen_helper_vse16_v_mask,
738          gen_helper_vse32_v_mask, gen_helper_vse64_v_mask },
739        /* unmasked unit stride store */
740        { gen_helper_vse8_v, gen_helper_vse16_v,
741          gen_helper_vse32_v, gen_helper_vse64_v }
742    };
743
744    fn =  fns[a->vm][eew];
745    if (fn == NULL) {
746        return false;
747    }
748
749    uint8_t emul = vext_get_emul(s, eew);
750    data = FIELD_DP32(data, VDATA, VM, a->vm);
751    data = FIELD_DP32(data, VDATA, LMUL, emul);
752    data = FIELD_DP32(data, VDATA, NF, a->nf);
753    return ldst_us_trans(a->rd, a->rs1, data, fn, s, true);
754}
755
756static bool st_us_check(DisasContext *s, arg_r2nfvm* a, uint8_t eew)
757{
758    return require_rvv(s) &&
759           vext_check_isa_ill(s) &&
760           vext_check_store(s, a->rd, a->nf, eew);
761}
762
763GEN_VEXT_TRANS(vse8_v,  MO_8,  r2nfvm, st_us_op, st_us_check)
764GEN_VEXT_TRANS(vse16_v, MO_16, r2nfvm, st_us_op, st_us_check)
765GEN_VEXT_TRANS(vse32_v, MO_32, r2nfvm, st_us_op, st_us_check)
766GEN_VEXT_TRANS(vse64_v, MO_64, r2nfvm, st_us_op, st_us_check)
767
768/*
769 *** unit stride mask load and store
770 */
771static bool ld_us_mask_op(DisasContext *s, arg_vlm_v *a, uint8_t eew)
772{
773    uint32_t data = 0;
774    gen_helper_ldst_us *fn = gen_helper_vlm_v;
775
776    /* EMUL = 1, NFIELDS = 1 */
777    data = FIELD_DP32(data, VDATA, LMUL, 0);
778    data = FIELD_DP32(data, VDATA, NF, 1);
779    /* Mask destination register are always tail-agnostic */
780    data = FIELD_DP32(data, VDATA, VTA, s->cfg_vta_all_1s);
781    data = FIELD_DP32(data, VDATA, VMA, s->vma);
782    return ldst_us_trans(a->rd, a->rs1, data, fn, s, false);
783}
784
785static bool ld_us_mask_check(DisasContext *s, arg_vlm_v *a, uint8_t eew)
786{
787    /* EMUL = 1, NFIELDS = 1 */
788    return require_rvv(s) && vext_check_isa_ill(s);
789}
790
791static bool st_us_mask_op(DisasContext *s, arg_vsm_v *a, uint8_t eew)
792{
793    uint32_t data = 0;
794    gen_helper_ldst_us *fn = gen_helper_vsm_v;
795
796    /* EMUL = 1, NFIELDS = 1 */
797    data = FIELD_DP32(data, VDATA, LMUL, 0);
798    data = FIELD_DP32(data, VDATA, NF, 1);
799    return ldst_us_trans(a->rd, a->rs1, data, fn, s, true);
800}
801
802static bool st_us_mask_check(DisasContext *s, arg_vsm_v *a, uint8_t eew)
803{
804    /* EMUL = 1, NFIELDS = 1 */
805    return require_rvv(s) && vext_check_isa_ill(s);
806}
807
808GEN_VEXT_TRANS(vlm_v, MO_8, vlm_v, ld_us_mask_op, ld_us_mask_check)
809GEN_VEXT_TRANS(vsm_v, MO_8, vsm_v, st_us_mask_op, st_us_mask_check)
810
811/*
812 *** stride load and store
813 */
814typedef void gen_helper_ldst_stride(TCGv_ptr, TCGv_ptr, TCGv,
815                                    TCGv, TCGv_env, TCGv_i32);
816
817static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2,
818                              uint32_t data, gen_helper_ldst_stride *fn,
819                              DisasContext *s, bool is_store)
820{
821    TCGv_ptr dest, mask;
822    TCGv base, stride;
823    TCGv_i32 desc;
824
825    TCGLabel *over = gen_new_label();
826    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
827    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
828
829    dest = tcg_temp_new_ptr();
830    mask = tcg_temp_new_ptr();
831    base = get_gpr(s, rs1, EXT_NONE);
832    stride = get_gpr(s, rs2, EXT_NONE);
833    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
834                                      s->cfg_ptr->vlen / 8, data));
835
836    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
837    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
838
839    fn(dest, mask, base, stride, cpu_env, desc);
840
841    tcg_temp_free_ptr(dest);
842    tcg_temp_free_ptr(mask);
843
844    if (!is_store) {
845        mark_vs_dirty(s);
846    }
847
848    gen_set_label(over);
849    return true;
850}
851
852static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
853{
854    uint32_t data = 0;
855    gen_helper_ldst_stride *fn;
856    static gen_helper_ldst_stride * const fns[4] = {
857        gen_helper_vlse8_v, gen_helper_vlse16_v,
858        gen_helper_vlse32_v, gen_helper_vlse64_v
859    };
860
861    fn = fns[eew];
862    if (fn == NULL) {
863        return false;
864    }
865
866    uint8_t emul = vext_get_emul(s, eew);
867    data = FIELD_DP32(data, VDATA, VM, a->vm);
868    data = FIELD_DP32(data, VDATA, LMUL, emul);
869    data = FIELD_DP32(data, VDATA, NF, a->nf);
870    data = FIELD_DP32(data, VDATA, VTA, s->vta);
871    data = FIELD_DP32(data, VDATA, VMA, s->vma);
872    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, false);
873}
874
875static bool ld_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
876{
877    return require_rvv(s) &&
878           vext_check_isa_ill(s) &&
879           vext_check_load(s, a->rd, a->nf, a->vm, eew);
880}
881
882GEN_VEXT_TRANS(vlse8_v,  MO_8,  rnfvm, ld_stride_op, ld_stride_check)
883GEN_VEXT_TRANS(vlse16_v, MO_16, rnfvm, ld_stride_op, ld_stride_check)
884GEN_VEXT_TRANS(vlse32_v, MO_32, rnfvm, ld_stride_op, ld_stride_check)
885GEN_VEXT_TRANS(vlse64_v, MO_64, rnfvm, ld_stride_op, ld_stride_check)
886
887static bool st_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
888{
889    uint32_t data = 0;
890    gen_helper_ldst_stride *fn;
891    static gen_helper_ldst_stride * const fns[4] = {
892        /* masked stride store */
893        gen_helper_vsse8_v,  gen_helper_vsse16_v,
894        gen_helper_vsse32_v,  gen_helper_vsse64_v
895    };
896
897    uint8_t emul = vext_get_emul(s, eew);
898    data = FIELD_DP32(data, VDATA, VM, a->vm);
899    data = FIELD_DP32(data, VDATA, LMUL, emul);
900    data = FIELD_DP32(data, VDATA, NF, a->nf);
901    fn = fns[eew];
902    if (fn == NULL) {
903        return false;
904    }
905
906    return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, true);
907}
908
909static bool st_stride_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
910{
911    return require_rvv(s) &&
912           vext_check_isa_ill(s) &&
913           vext_check_store(s, a->rd, a->nf, eew);
914}
915
916GEN_VEXT_TRANS(vsse8_v,  MO_8,  rnfvm, st_stride_op, st_stride_check)
917GEN_VEXT_TRANS(vsse16_v, MO_16, rnfvm, st_stride_op, st_stride_check)
918GEN_VEXT_TRANS(vsse32_v, MO_32, rnfvm, st_stride_op, st_stride_check)
919GEN_VEXT_TRANS(vsse64_v, MO_64, rnfvm, st_stride_op, st_stride_check)
920
921/*
922 *** index load and store
923 */
924typedef void gen_helper_ldst_index(TCGv_ptr, TCGv_ptr, TCGv,
925                                   TCGv_ptr, TCGv_env, TCGv_i32);
926
927static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
928                             uint32_t data, gen_helper_ldst_index *fn,
929                             DisasContext *s, bool is_store)
930{
931    TCGv_ptr dest, mask, index;
932    TCGv base;
933    TCGv_i32 desc;
934
935    TCGLabel *over = gen_new_label();
936    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
937    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
938
939    dest = tcg_temp_new_ptr();
940    mask = tcg_temp_new_ptr();
941    index = tcg_temp_new_ptr();
942    base = get_gpr(s, rs1, EXT_NONE);
943    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
944                                      s->cfg_ptr->vlen / 8, data));
945
946    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
947    tcg_gen_addi_ptr(index, cpu_env, vreg_ofs(s, vs2));
948    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
949
950    fn(dest, mask, base, index, cpu_env, desc);
951
952    tcg_temp_free_ptr(dest);
953    tcg_temp_free_ptr(mask);
954    tcg_temp_free_ptr(index);
955
956    if (!is_store) {
957        mark_vs_dirty(s);
958    }
959
960    gen_set_label(over);
961    return true;
962}
963
964static bool ld_index_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
965{
966    uint32_t data = 0;
967    gen_helper_ldst_index *fn;
968    static gen_helper_ldst_index * const fns[4][4] = {
969        /*
970         * offset vector register group EEW = 8,
971         * data vector register group EEW = SEW
972         */
973        { gen_helper_vlxei8_8_v,  gen_helper_vlxei8_16_v,
974          gen_helper_vlxei8_32_v, gen_helper_vlxei8_64_v },
975        /*
976         * offset vector register group EEW = 16,
977         * data vector register group EEW = SEW
978         */
979        { gen_helper_vlxei16_8_v, gen_helper_vlxei16_16_v,
980          gen_helper_vlxei16_32_v, gen_helper_vlxei16_64_v },
981        /*
982         * offset vector register group EEW = 32,
983         * data vector register group EEW = SEW
984         */
985        { gen_helper_vlxei32_8_v, gen_helper_vlxei32_16_v,
986          gen_helper_vlxei32_32_v, gen_helper_vlxei32_64_v },
987        /*
988         * offset vector register group EEW = 64,
989         * data vector register group EEW = SEW
990         */
991        { gen_helper_vlxei64_8_v, gen_helper_vlxei64_16_v,
992          gen_helper_vlxei64_32_v, gen_helper_vlxei64_64_v }
993    };
994
995    fn = fns[eew][s->sew];
996
997    uint8_t emul = vext_get_emul(s, s->sew);
998    data = FIELD_DP32(data, VDATA, VM, a->vm);
999    data = FIELD_DP32(data, VDATA, LMUL, emul);
1000    data = FIELD_DP32(data, VDATA, NF, a->nf);
1001    data = FIELD_DP32(data, VDATA, VTA, s->vta);
1002    data = FIELD_DP32(data, VDATA, VMA, s->vma);
1003    return ldst_index_trans(a->rd, a->rs1, a->rs2, data, fn, s, false);
1004}
1005
1006static bool ld_index_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
1007{
1008    return require_rvv(s) &&
1009           vext_check_isa_ill(s) &&
1010           vext_check_ld_index(s, a->rd, a->rs2, a->nf, a->vm, eew);
1011}
1012
1013GEN_VEXT_TRANS(vlxei8_v,  MO_8,  rnfvm, ld_index_op, ld_index_check)
1014GEN_VEXT_TRANS(vlxei16_v, MO_16, rnfvm, ld_index_op, ld_index_check)
1015GEN_VEXT_TRANS(vlxei32_v, MO_32, rnfvm, ld_index_op, ld_index_check)
1016GEN_VEXT_TRANS(vlxei64_v, MO_64, rnfvm, ld_index_op, ld_index_check)
1017
1018static bool st_index_op(DisasContext *s, arg_rnfvm *a, uint8_t eew)
1019{
1020    uint32_t data = 0;
1021    gen_helper_ldst_index *fn;
1022    static gen_helper_ldst_index * const fns[4][4] = {
1023        /*
1024         * offset vector register group EEW = 8,
1025         * data vector register group EEW = SEW
1026         */
1027        { gen_helper_vsxei8_8_v,  gen_helper_vsxei8_16_v,
1028          gen_helper_vsxei8_32_v, gen_helper_vsxei8_64_v },
1029        /*
1030         * offset vector register group EEW = 16,
1031         * data vector register group EEW = SEW
1032         */
1033        { gen_helper_vsxei16_8_v, gen_helper_vsxei16_16_v,
1034          gen_helper_vsxei16_32_v, gen_helper_vsxei16_64_v },
1035        /*
1036         * offset vector register group EEW = 32,
1037         * data vector register group EEW = SEW
1038         */
1039        { gen_helper_vsxei32_8_v, gen_helper_vsxei32_16_v,
1040          gen_helper_vsxei32_32_v, gen_helper_vsxei32_64_v },
1041        /*
1042         * offset vector register group EEW = 64,
1043         * data vector register group EEW = SEW
1044         */
1045        { gen_helper_vsxei64_8_v, gen_helper_vsxei64_16_v,
1046          gen_helper_vsxei64_32_v, gen_helper_vsxei64_64_v }
1047    };
1048
1049    fn = fns[eew][s->sew];
1050
1051    uint8_t emul = vext_get_emul(s, s->sew);
1052    data = FIELD_DP32(data, VDATA, VM, a->vm);
1053    data = FIELD_DP32(data, VDATA, LMUL, emul);
1054    data = FIELD_DP32(data, VDATA, NF, a->nf);
1055    return ldst_index_trans(a->rd, a->rs1, a->rs2, data, fn, s, true);
1056}
1057
1058static bool st_index_check(DisasContext *s, arg_rnfvm* a, uint8_t eew)
1059{
1060    return require_rvv(s) &&
1061           vext_check_isa_ill(s) &&
1062           vext_check_st_index(s, a->rd, a->rs2, a->nf, eew);
1063}
1064
1065GEN_VEXT_TRANS(vsxei8_v,  MO_8,  rnfvm, st_index_op, st_index_check)
1066GEN_VEXT_TRANS(vsxei16_v, MO_16, rnfvm, st_index_op, st_index_check)
1067GEN_VEXT_TRANS(vsxei32_v, MO_32, rnfvm, st_index_op, st_index_check)
1068GEN_VEXT_TRANS(vsxei64_v, MO_64, rnfvm, st_index_op, st_index_check)
1069
1070/*
1071 *** unit stride fault-only-first load
1072 */
1073static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data,
1074                       gen_helper_ldst_us *fn, DisasContext *s)
1075{
1076    TCGv_ptr dest, mask;
1077    TCGv base;
1078    TCGv_i32 desc;
1079
1080    TCGLabel *over = gen_new_label();
1081    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1082    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1083
1084    dest = tcg_temp_new_ptr();
1085    mask = tcg_temp_new_ptr();
1086    base = get_gpr(s, rs1, EXT_NONE);
1087    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
1088                                      s->cfg_ptr->vlen / 8, data));
1089
1090    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
1091    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
1092
1093    fn(dest, mask, base, cpu_env, desc);
1094
1095    tcg_temp_free_ptr(dest);
1096    tcg_temp_free_ptr(mask);
1097    mark_vs_dirty(s);
1098    gen_set_label(over);
1099    return true;
1100}
1101
1102static bool ldff_op(DisasContext *s, arg_r2nfvm *a, uint8_t eew)
1103{
1104    uint32_t data = 0;
1105    gen_helper_ldst_us *fn;
1106    static gen_helper_ldst_us * const fns[4] = {
1107        gen_helper_vle8ff_v, gen_helper_vle16ff_v,
1108        gen_helper_vle32ff_v, gen_helper_vle64ff_v
1109    };
1110
1111    fn = fns[eew];
1112    if (fn == NULL) {
1113        return false;
1114    }
1115
1116    uint8_t emul = vext_get_emul(s, eew);
1117    data = FIELD_DP32(data, VDATA, VM, a->vm);
1118    data = FIELD_DP32(data, VDATA, LMUL, emul);
1119    data = FIELD_DP32(data, VDATA, NF, a->nf);
1120    data = FIELD_DP32(data, VDATA, VTA, s->vta);
1121    data = FIELD_DP32(data, VDATA, VMA, s->vma);
1122    return ldff_trans(a->rd, a->rs1, data, fn, s);
1123}
1124
1125GEN_VEXT_TRANS(vle8ff_v,  MO_8,  r2nfvm, ldff_op, ld_us_check)
1126GEN_VEXT_TRANS(vle16ff_v, MO_16, r2nfvm, ldff_op, ld_us_check)
1127GEN_VEXT_TRANS(vle32ff_v, MO_32, r2nfvm, ldff_op, ld_us_check)
1128GEN_VEXT_TRANS(vle64ff_v, MO_64, r2nfvm, ldff_op, ld_us_check)
1129
1130/*
1131 * load and store whole register instructions
1132 */
1133typedef void gen_helper_ldst_whole(TCGv_ptr, TCGv, TCGv_env, TCGv_i32);
1134
1135static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf,
1136                             uint32_t width, gen_helper_ldst_whole *fn,
1137                             DisasContext *s, bool is_store)
1138{
1139    uint32_t evl = (s->cfg_ptr->vlen / 8) * nf / width;
1140    TCGLabel *over = gen_new_label();
1141    tcg_gen_brcondi_tl(TCG_COND_GEU, cpu_vstart, evl, over);
1142
1143    TCGv_ptr dest;
1144    TCGv base;
1145    TCGv_i32 desc;
1146
1147    uint32_t data = FIELD_DP32(0, VDATA, NF, nf);
1148    dest = tcg_temp_new_ptr();
1149    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
1150                                      s->cfg_ptr->vlen / 8, data));
1151
1152    base = get_gpr(s, rs1, EXT_NONE);
1153    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
1154
1155    fn(dest, base, cpu_env, desc);
1156
1157    tcg_temp_free_ptr(dest);
1158
1159    if (!is_store) {
1160        mark_vs_dirty(s);
1161    }
1162    gen_set_label(over);
1163
1164    return true;
1165}
1166
1167/*
1168 * load and store whole register instructions ignore vtype and vl setting.
1169 * Thus, we don't need to check vill bit. (Section 7.9)
1170 */
1171#define GEN_LDST_WHOLE_TRANS(NAME, ARG_NF, WIDTH, IS_STORE)               \
1172static bool trans_##NAME(DisasContext *s, arg_##NAME * a)                 \
1173{                                                                         \
1174    if (require_rvv(s) &&                                                 \
1175        QEMU_IS_ALIGNED(a->rd, ARG_NF)) {                                 \
1176        return ldst_whole_trans(a->rd, a->rs1, ARG_NF, WIDTH,             \
1177                                gen_helper_##NAME, s, IS_STORE);          \
1178    }                                                                     \
1179    return false;                                                         \
1180}
1181
1182GEN_LDST_WHOLE_TRANS(vl1re8_v,  1, 1, false)
1183GEN_LDST_WHOLE_TRANS(vl1re16_v, 1, 2, false)
1184GEN_LDST_WHOLE_TRANS(vl1re32_v, 1, 4, false)
1185GEN_LDST_WHOLE_TRANS(vl1re64_v, 1, 8, false)
1186GEN_LDST_WHOLE_TRANS(vl2re8_v,  2, 1, false)
1187GEN_LDST_WHOLE_TRANS(vl2re16_v, 2, 2, false)
1188GEN_LDST_WHOLE_TRANS(vl2re32_v, 2, 4, false)
1189GEN_LDST_WHOLE_TRANS(vl2re64_v, 2, 8, false)
1190GEN_LDST_WHOLE_TRANS(vl4re8_v,  4, 1, false)
1191GEN_LDST_WHOLE_TRANS(vl4re16_v, 4, 2, false)
1192GEN_LDST_WHOLE_TRANS(vl4re32_v, 4, 4, false)
1193GEN_LDST_WHOLE_TRANS(vl4re64_v, 4, 8, false)
1194GEN_LDST_WHOLE_TRANS(vl8re8_v,  8, 1, false)
1195GEN_LDST_WHOLE_TRANS(vl8re16_v, 8, 2, false)
1196GEN_LDST_WHOLE_TRANS(vl8re32_v, 8, 4, false)
1197GEN_LDST_WHOLE_TRANS(vl8re64_v, 8, 8, false)
1198
1199/*
1200 * The vector whole register store instructions are encoded similar to
1201 * unmasked unit-stride store of elements with EEW=8.
1202 */
1203GEN_LDST_WHOLE_TRANS(vs1r_v, 1, 1, true)
1204GEN_LDST_WHOLE_TRANS(vs2r_v, 2, 1, true)
1205GEN_LDST_WHOLE_TRANS(vs4r_v, 4, 1, true)
1206GEN_LDST_WHOLE_TRANS(vs8r_v, 8, 1, true)
1207
1208/*
1209 *** Vector Integer Arithmetic Instructions
1210 */
1211
1212/*
1213 * MAXSZ returns the maximum vector size can be operated in bytes,
1214 * which is used in GVEC IR when vl_eq_vlmax flag is set to true
1215 * to accerlate vector operation.
1216 */
1217static inline uint32_t MAXSZ(DisasContext *s)
1218{
1219    int scale = s->lmul - 3;
1220    return s->cfg_ptr->vlen >> -scale;
1221}
1222
1223static bool opivv_check(DisasContext *s, arg_rmrr *a)
1224{
1225    return require_rvv(s) &&
1226           vext_check_isa_ill(s) &&
1227           vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm);
1228}
1229
1230typedef void GVecGen3Fn(unsigned, uint32_t, uint32_t,
1231                        uint32_t, uint32_t, uint32_t);
1232
1233static inline bool
1234do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn,
1235              gen_helper_gvec_4_ptr *fn)
1236{
1237    TCGLabel *over = gen_new_label();
1238    if (!opivv_check(s, a)) {
1239        return false;
1240    }
1241
1242    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1243    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1244
1245    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1246        gvec_fn(s->sew, vreg_ofs(s, a->rd),
1247                vreg_ofs(s, a->rs2), vreg_ofs(s, a->rs1),
1248                MAXSZ(s), MAXSZ(s));
1249    } else {
1250        uint32_t data = 0;
1251
1252        data = FIELD_DP32(data, VDATA, VM, a->vm);
1253        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1254        data = FIELD_DP32(data, VDATA, VTA, s->vta);
1255        data = FIELD_DP32(data, VDATA, VMA, s->vma);
1256        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1257                           vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2),
1258                           cpu_env, s->cfg_ptr->vlen / 8,
1259                           s->cfg_ptr->vlen / 8, data, fn);
1260    }
1261    mark_vs_dirty(s);
1262    gen_set_label(over);
1263    return true;
1264}
1265
1266/* OPIVV with GVEC IR */
1267#define GEN_OPIVV_GVEC_TRANS(NAME, SUF) \
1268static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
1269{                                                                  \
1270    static gen_helper_gvec_4_ptr * const fns[4] = {                \
1271        gen_helper_##NAME##_b, gen_helper_##NAME##_h,              \
1272        gen_helper_##NAME##_w, gen_helper_##NAME##_d,              \
1273    };                                                             \
1274    return do_opivv_gvec(s, a, tcg_gen_gvec_##SUF, fns[s->sew]);   \
1275}
1276
1277GEN_OPIVV_GVEC_TRANS(vadd_vv, add)
1278GEN_OPIVV_GVEC_TRANS(vsub_vv, sub)
1279
1280typedef void gen_helper_opivx(TCGv_ptr, TCGv_ptr, TCGv, TCGv_ptr,
1281                              TCGv_env, TCGv_i32);
1282
1283static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm,
1284                        gen_helper_opivx *fn, DisasContext *s)
1285{
1286    TCGv_ptr dest, src2, mask;
1287    TCGv src1;
1288    TCGv_i32 desc;
1289    uint32_t data = 0;
1290
1291    TCGLabel *over = gen_new_label();
1292    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1293    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1294
1295    dest = tcg_temp_new_ptr();
1296    mask = tcg_temp_new_ptr();
1297    src2 = tcg_temp_new_ptr();
1298    src1 = get_gpr(s, rs1, EXT_SIGN);
1299
1300    data = FIELD_DP32(data, VDATA, VM, vm);
1301    data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1302    data = FIELD_DP32(data, VDATA, VTA, s->vta);
1303    data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);
1304    data = FIELD_DP32(data, VDATA, VMA, s->vma);
1305    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
1306                                      s->cfg_ptr->vlen / 8, data));
1307
1308    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
1309    tcg_gen_addi_ptr(src2, cpu_env, vreg_ofs(s, vs2));
1310    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
1311
1312    fn(dest, mask, src1, src2, cpu_env, desc);
1313
1314    tcg_temp_free_ptr(dest);
1315    tcg_temp_free_ptr(mask);
1316    tcg_temp_free_ptr(src2);
1317    mark_vs_dirty(s);
1318    gen_set_label(over);
1319    return true;
1320}
1321
1322static bool opivx_check(DisasContext *s, arg_rmrr *a)
1323{
1324    return require_rvv(s) &&
1325           vext_check_isa_ill(s) &&
1326           vext_check_ss(s, a->rd, a->rs2, a->vm);
1327}
1328
1329typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t, TCGv_i64,
1330                         uint32_t, uint32_t);
1331
1332static inline bool
1333do_opivx_gvec(DisasContext *s, arg_rmrr *a, GVecGen2sFn *gvec_fn,
1334              gen_helper_opivx *fn)
1335{
1336    if (!opivx_check(s, a)) {
1337        return false;
1338    }
1339
1340    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1341        TCGv_i64 src1 = tcg_temp_new_i64();
1342
1343        tcg_gen_ext_tl_i64(src1, get_gpr(s, a->rs1, EXT_SIGN));
1344        gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1345                src1, MAXSZ(s), MAXSZ(s));
1346
1347        tcg_temp_free_i64(src1);
1348        mark_vs_dirty(s);
1349        return true;
1350    }
1351    return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1352}
1353
1354/* OPIVX with GVEC IR */
1355#define GEN_OPIVX_GVEC_TRANS(NAME, SUF) \
1356static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
1357{                                                                  \
1358    static gen_helper_opivx * const fns[4] = {                     \
1359        gen_helper_##NAME##_b, gen_helper_##NAME##_h,              \
1360        gen_helper_##NAME##_w, gen_helper_##NAME##_d,              \
1361    };                                                             \
1362    return do_opivx_gvec(s, a, tcg_gen_gvec_##SUF, fns[s->sew]);   \
1363}
1364
1365GEN_OPIVX_GVEC_TRANS(vadd_vx, adds)
1366GEN_OPIVX_GVEC_TRANS(vsub_vx, subs)
1367
1368static void gen_vec_rsub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1369{
1370    tcg_gen_vec_sub8_i64(d, b, a);
1371}
1372
1373static void gen_vec_rsub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1374{
1375    tcg_gen_vec_sub16_i64(d, b, a);
1376}
1377
1378static void gen_rsub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
1379{
1380    tcg_gen_sub_i32(ret, arg2, arg1);
1381}
1382
1383static void gen_rsub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1384{
1385    tcg_gen_sub_i64(ret, arg2, arg1);
1386}
1387
1388static void gen_rsub_vec(unsigned vece, TCGv_vec r, TCGv_vec a, TCGv_vec b)
1389{
1390    tcg_gen_sub_vec(vece, r, b, a);
1391}
1392
1393static void tcg_gen_gvec_rsubs(unsigned vece, uint32_t dofs, uint32_t aofs,
1394                               TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1395{
1396    static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
1397    static const GVecGen2s rsub_op[4] = {
1398        { .fni8 = gen_vec_rsub8_i64,
1399          .fniv = gen_rsub_vec,
1400          .fno = gen_helper_vec_rsubs8,
1401          .opt_opc = vecop_list,
1402          .vece = MO_8 },
1403        { .fni8 = gen_vec_rsub16_i64,
1404          .fniv = gen_rsub_vec,
1405          .fno = gen_helper_vec_rsubs16,
1406          .opt_opc = vecop_list,
1407          .vece = MO_16 },
1408        { .fni4 = gen_rsub_i32,
1409          .fniv = gen_rsub_vec,
1410          .fno = gen_helper_vec_rsubs32,
1411          .opt_opc = vecop_list,
1412          .vece = MO_32 },
1413        { .fni8 = gen_rsub_i64,
1414          .fniv = gen_rsub_vec,
1415          .fno = gen_helper_vec_rsubs64,
1416          .opt_opc = vecop_list,
1417          .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1418          .vece = MO_64 },
1419    };
1420
1421    tcg_debug_assert(vece <= MO_64);
1422    tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &rsub_op[vece]);
1423}
1424
1425GEN_OPIVX_GVEC_TRANS(vrsub_vx, rsubs)
1426
1427typedef enum {
1428    IMM_ZX,         /* Zero-extended */
1429    IMM_SX,         /* Sign-extended */
1430    IMM_TRUNC_SEW,  /* Truncate to log(SEW) bits */
1431    IMM_TRUNC_2SEW, /* Truncate to log(2*SEW) bits */
1432} imm_mode_t;
1433
1434static int64_t extract_imm(DisasContext *s, uint32_t imm, imm_mode_t imm_mode)
1435{
1436    switch (imm_mode) {
1437    case IMM_ZX:
1438        return extract64(imm, 0, 5);
1439    case IMM_SX:
1440        return sextract64(imm, 0, 5);
1441    case IMM_TRUNC_SEW:
1442        return extract64(imm, 0, s->sew + 3);
1443    case IMM_TRUNC_2SEW:
1444        return extract64(imm, 0, s->sew + 4);
1445    default:
1446        g_assert_not_reached();
1447    }
1448}
1449
1450static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm,
1451                        gen_helper_opivx *fn, DisasContext *s,
1452                        imm_mode_t imm_mode)
1453{
1454    TCGv_ptr dest, src2, mask;
1455    TCGv src1;
1456    TCGv_i32 desc;
1457    uint32_t data = 0;
1458
1459    TCGLabel *over = gen_new_label();
1460    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1461    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1462
1463    dest = tcg_temp_new_ptr();
1464    mask = tcg_temp_new_ptr();
1465    src2 = tcg_temp_new_ptr();
1466    src1 = tcg_constant_tl(extract_imm(s, imm, imm_mode));
1467
1468    data = FIELD_DP32(data, VDATA, VM, vm);
1469    data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1470    data = FIELD_DP32(data, VDATA, VTA, s->vta);
1471    data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);
1472    data = FIELD_DP32(data, VDATA, VMA, s->vma);
1473    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
1474                                      s->cfg_ptr->vlen / 8, data));
1475
1476    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
1477    tcg_gen_addi_ptr(src2, cpu_env, vreg_ofs(s, vs2));
1478    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
1479
1480    fn(dest, mask, src1, src2, cpu_env, desc);
1481
1482    tcg_temp_free_ptr(dest);
1483    tcg_temp_free_ptr(mask);
1484    tcg_temp_free_ptr(src2);
1485    mark_vs_dirty(s);
1486    gen_set_label(over);
1487    return true;
1488}
1489
1490typedef void GVecGen2iFn(unsigned, uint32_t, uint32_t, int64_t,
1491                         uint32_t, uint32_t);
1492
1493static inline bool
1494do_opivi_gvec(DisasContext *s, arg_rmrr *a, GVecGen2iFn *gvec_fn,
1495              gen_helper_opivx *fn, imm_mode_t imm_mode)
1496{
1497    if (!opivx_check(s, a)) {
1498        return false;
1499    }
1500
1501    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1502        gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1503                extract_imm(s, a->rs1, imm_mode), MAXSZ(s), MAXSZ(s));
1504        mark_vs_dirty(s);
1505        return true;
1506    }
1507    return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s, imm_mode);
1508}
1509
1510/* OPIVI with GVEC IR */
1511#define GEN_OPIVI_GVEC_TRANS(NAME, IMM_MODE, OPIVX, SUF) \
1512static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
1513{                                                                  \
1514    static gen_helper_opivx * const fns[4] = {                     \
1515        gen_helper_##OPIVX##_b, gen_helper_##OPIVX##_h,            \
1516        gen_helper_##OPIVX##_w, gen_helper_##OPIVX##_d,            \
1517    };                                                             \
1518    return do_opivi_gvec(s, a, tcg_gen_gvec_##SUF,                 \
1519                         fns[s->sew], IMM_MODE);                   \
1520}
1521
1522GEN_OPIVI_GVEC_TRANS(vadd_vi, IMM_SX, vadd_vx, addi)
1523
1524static void tcg_gen_gvec_rsubi(unsigned vece, uint32_t dofs, uint32_t aofs,
1525                               int64_t c, uint32_t oprsz, uint32_t maxsz)
1526{
1527    TCGv_i64 tmp = tcg_constant_i64(c);
1528    tcg_gen_gvec_rsubs(vece, dofs, aofs, tmp, oprsz, maxsz);
1529}
1530
1531GEN_OPIVI_GVEC_TRANS(vrsub_vi, IMM_SX, vrsub_vx, rsubi)
1532
1533/* Vector Widening Integer Add/Subtract */
1534
1535/* OPIVV with WIDEN */
1536static bool opivv_widen_check(DisasContext *s, arg_rmrr *a)
1537{
1538    return require_rvv(s) &&
1539           vext_check_isa_ill(s) &&
1540           vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm);
1541}
1542
1543static bool do_opivv_widen(DisasContext *s, arg_rmrr *a,
1544                           gen_helper_gvec_4_ptr *fn,
1545                           bool (*checkfn)(DisasContext *, arg_rmrr *))
1546{
1547    if (checkfn(s, a)) {
1548        uint32_t data = 0;
1549        TCGLabel *over = gen_new_label();
1550        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1551        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1552
1553        data = FIELD_DP32(data, VDATA, VM, a->vm);
1554        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1555        data = FIELD_DP32(data, VDATA, VTA, s->vta);
1556        data = FIELD_DP32(data, VDATA, VMA, s->vma);
1557        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1558                           vreg_ofs(s, a->rs1),
1559                           vreg_ofs(s, a->rs2),
1560                           cpu_env, s->cfg_ptr->vlen / 8,
1561                           s->cfg_ptr->vlen / 8,
1562                           data, fn);
1563        mark_vs_dirty(s);
1564        gen_set_label(over);
1565        return true;
1566    }
1567    return false;
1568}
1569
1570#define GEN_OPIVV_WIDEN_TRANS(NAME, CHECK) \
1571static bool trans_##NAME(DisasContext *s, arg_rmrr *a)       \
1572{                                                            \
1573    static gen_helper_gvec_4_ptr * const fns[3] = {          \
1574        gen_helper_##NAME##_b,                               \
1575        gen_helper_##NAME##_h,                               \
1576        gen_helper_##NAME##_w                                \
1577    };                                                       \
1578    return do_opivv_widen(s, a, fns[s->sew], CHECK);         \
1579}
1580
1581GEN_OPIVV_WIDEN_TRANS(vwaddu_vv, opivv_widen_check)
1582GEN_OPIVV_WIDEN_TRANS(vwadd_vv, opivv_widen_check)
1583GEN_OPIVV_WIDEN_TRANS(vwsubu_vv, opivv_widen_check)
1584GEN_OPIVV_WIDEN_TRANS(vwsub_vv, opivv_widen_check)
1585
1586/* OPIVX with WIDEN */
1587static bool opivx_widen_check(DisasContext *s, arg_rmrr *a)
1588{
1589    return require_rvv(s) &&
1590           vext_check_isa_ill(s) &&
1591           vext_check_ds(s, a->rd, a->rs2, a->vm);
1592}
1593
1594static bool do_opivx_widen(DisasContext *s, arg_rmrr *a,
1595                           gen_helper_opivx *fn)
1596{
1597    if (opivx_widen_check(s, a)) {
1598        return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1599    }
1600    return false;
1601}
1602
1603#define GEN_OPIVX_WIDEN_TRANS(NAME) \
1604static bool trans_##NAME(DisasContext *s, arg_rmrr *a)       \
1605{                                                            \
1606    static gen_helper_opivx * const fns[3] = {               \
1607        gen_helper_##NAME##_b,                               \
1608        gen_helper_##NAME##_h,                               \
1609        gen_helper_##NAME##_w                                \
1610    };                                                       \
1611    return do_opivx_widen(s, a, fns[s->sew]);                \
1612}
1613
1614GEN_OPIVX_WIDEN_TRANS(vwaddu_vx)
1615GEN_OPIVX_WIDEN_TRANS(vwadd_vx)
1616GEN_OPIVX_WIDEN_TRANS(vwsubu_vx)
1617GEN_OPIVX_WIDEN_TRANS(vwsub_vx)
1618
1619/* WIDEN OPIVV with WIDEN */
1620static bool opiwv_widen_check(DisasContext *s, arg_rmrr *a)
1621{
1622    return require_rvv(s) &&
1623           vext_check_isa_ill(s) &&
1624           vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm);
1625}
1626
1627static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a,
1628                           gen_helper_gvec_4_ptr *fn)
1629{
1630    if (opiwv_widen_check(s, a)) {
1631        uint32_t data = 0;
1632        TCGLabel *over = gen_new_label();
1633        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
1634        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
1635
1636        data = FIELD_DP32(data, VDATA, VM, a->vm);
1637        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
1638        data = FIELD_DP32(data, VDATA, VTA, s->vta);
1639        data = FIELD_DP32(data, VDATA, VMA, s->vma);
1640        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
1641                           vreg_ofs(s, a->rs1),
1642                           vreg_ofs(s, a->rs2),
1643                           cpu_env, s->cfg_ptr->vlen / 8,
1644                           s->cfg_ptr->vlen / 8, data, fn);
1645        mark_vs_dirty(s);
1646        gen_set_label(over);
1647        return true;
1648    }
1649    return false;
1650}
1651
1652#define GEN_OPIWV_WIDEN_TRANS(NAME) \
1653static bool trans_##NAME(DisasContext *s, arg_rmrr *a)       \
1654{                                                            \
1655    static gen_helper_gvec_4_ptr * const fns[3] = {          \
1656        gen_helper_##NAME##_b,                               \
1657        gen_helper_##NAME##_h,                               \
1658        gen_helper_##NAME##_w                                \
1659    };                                                       \
1660    return do_opiwv_widen(s, a, fns[s->sew]);                \
1661}
1662
1663GEN_OPIWV_WIDEN_TRANS(vwaddu_wv)
1664GEN_OPIWV_WIDEN_TRANS(vwadd_wv)
1665GEN_OPIWV_WIDEN_TRANS(vwsubu_wv)
1666GEN_OPIWV_WIDEN_TRANS(vwsub_wv)
1667
1668/* WIDEN OPIVX with WIDEN */
1669static bool opiwx_widen_check(DisasContext *s, arg_rmrr *a)
1670{
1671    return require_rvv(s) &&
1672           vext_check_isa_ill(s) &&
1673           vext_check_dd(s, a->rd, a->rs2, a->vm);
1674}
1675
1676static bool do_opiwx_widen(DisasContext *s, arg_rmrr *a,
1677                           gen_helper_opivx *fn)
1678{
1679    if (opiwx_widen_check(s, a)) {
1680        return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1681    }
1682    return false;
1683}
1684
1685#define GEN_OPIWX_WIDEN_TRANS(NAME) \
1686static bool trans_##NAME(DisasContext *s, arg_rmrr *a)       \
1687{                                                            \
1688    static gen_helper_opivx * const fns[3] = {               \
1689        gen_helper_##NAME##_b,                               \
1690        gen_helper_##NAME##_h,                               \
1691        gen_helper_##NAME##_w                                \
1692    };                                                       \
1693    return do_opiwx_widen(s, a, fns[s->sew]);                \
1694}
1695
1696GEN_OPIWX_WIDEN_TRANS(vwaddu_wx)
1697GEN_OPIWX_WIDEN_TRANS(vwadd_wx)
1698GEN_OPIWX_WIDEN_TRANS(vwsubu_wx)
1699GEN_OPIWX_WIDEN_TRANS(vwsub_wx)
1700
1701/* Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions */
1702/* OPIVV without GVEC IR */
1703#define GEN_OPIVV_TRANS(NAME, CHECK)                               \
1704static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
1705{                                                                  \
1706    if (CHECK(s, a)) {                                             \
1707        uint32_t data = 0;                                         \
1708        static gen_helper_gvec_4_ptr * const fns[4] = {            \
1709            gen_helper_##NAME##_b, gen_helper_##NAME##_h,          \
1710            gen_helper_##NAME##_w, gen_helper_##NAME##_d,          \
1711        };                                                         \
1712        TCGLabel *over = gen_new_label();                          \
1713        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
1714        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
1715                                                                   \
1716        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
1717        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
1718        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
1719        data =                                                     \
1720            FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
1721        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
1722        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
1723                           vreg_ofs(s, a->rs1),                    \
1724                           vreg_ofs(s, a->rs2), cpu_env,           \
1725                           s->cfg_ptr->vlen / 8,                   \
1726                           s->cfg_ptr->vlen / 8, data,             \
1727                           fns[s->sew]);                           \
1728        mark_vs_dirty(s);                                          \
1729        gen_set_label(over);                                       \
1730        return true;                                               \
1731    }                                                              \
1732    return false;                                                  \
1733}
1734
1735/*
1736 * For vadc and vsbc, an illegal instruction exception is raised if the
1737 * destination vector register is v0 and LMUL > 1. (Section 11.4)
1738 */
1739static bool opivv_vadc_check(DisasContext *s, arg_rmrr *a)
1740{
1741    return require_rvv(s) &&
1742           vext_check_isa_ill(s) &&
1743           (a->rd != 0) &&
1744           vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm);
1745}
1746
1747GEN_OPIVV_TRANS(vadc_vvm, opivv_vadc_check)
1748GEN_OPIVV_TRANS(vsbc_vvm, opivv_vadc_check)
1749
1750/*
1751 * For vmadc and vmsbc, an illegal instruction exception is raised if the
1752 * destination vector register overlaps a source vector register group.
1753 */
1754static bool opivv_vmadc_check(DisasContext *s, arg_rmrr *a)
1755{
1756    return require_rvv(s) &&
1757           vext_check_isa_ill(s) &&
1758           vext_check_mss(s, a->rd, a->rs1, a->rs2);
1759}
1760
1761GEN_OPIVV_TRANS(vmadc_vvm, opivv_vmadc_check)
1762GEN_OPIVV_TRANS(vmsbc_vvm, opivv_vmadc_check)
1763
1764static bool opivx_vadc_check(DisasContext *s, arg_rmrr *a)
1765{
1766    return require_rvv(s) &&
1767           vext_check_isa_ill(s) &&
1768           (a->rd != 0) &&
1769           vext_check_ss(s, a->rd, a->rs2, a->vm);
1770}
1771
1772/* OPIVX without GVEC IR */
1773#define GEN_OPIVX_TRANS(NAME, CHECK)                                     \
1774static bool trans_##NAME(DisasContext *s, arg_rmrr *a)                   \
1775{                                                                        \
1776    if (CHECK(s, a)) {                                                   \
1777        static gen_helper_opivx * const fns[4] = {                       \
1778            gen_helper_##NAME##_b, gen_helper_##NAME##_h,                \
1779            gen_helper_##NAME##_w, gen_helper_##NAME##_d,                \
1780        };                                                               \
1781                                                                         \
1782        return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);\
1783    }                                                                    \
1784    return false;                                                        \
1785}
1786
1787GEN_OPIVX_TRANS(vadc_vxm, opivx_vadc_check)
1788GEN_OPIVX_TRANS(vsbc_vxm, opivx_vadc_check)
1789
1790static bool opivx_vmadc_check(DisasContext *s, arg_rmrr *a)
1791{
1792    return require_rvv(s) &&
1793           vext_check_isa_ill(s) &&
1794           vext_check_ms(s, a->rd, a->rs2);
1795}
1796
1797GEN_OPIVX_TRANS(vmadc_vxm, opivx_vmadc_check)
1798GEN_OPIVX_TRANS(vmsbc_vxm, opivx_vmadc_check)
1799
1800/* OPIVI without GVEC IR */
1801#define GEN_OPIVI_TRANS(NAME, IMM_MODE, OPIVX, CHECK)                    \
1802static bool trans_##NAME(DisasContext *s, arg_rmrr *a)                   \
1803{                                                                        \
1804    if (CHECK(s, a)) {                                                   \
1805        static gen_helper_opivx * const fns[4] = {                       \
1806            gen_helper_##OPIVX##_b, gen_helper_##OPIVX##_h,              \
1807            gen_helper_##OPIVX##_w, gen_helper_##OPIVX##_d,              \
1808        };                                                               \
1809        return opivi_trans(a->rd, a->rs1, a->rs2, a->vm,                 \
1810                           fns[s->sew], s, IMM_MODE);                    \
1811    }                                                                    \
1812    return false;                                                        \
1813}
1814
1815GEN_OPIVI_TRANS(vadc_vim, IMM_SX, vadc_vxm, opivx_vadc_check)
1816GEN_OPIVI_TRANS(vmadc_vim, IMM_SX, vmadc_vxm, opivx_vmadc_check)
1817
1818/* Vector Bitwise Logical Instructions */
1819GEN_OPIVV_GVEC_TRANS(vand_vv, and)
1820GEN_OPIVV_GVEC_TRANS(vor_vv,  or)
1821GEN_OPIVV_GVEC_TRANS(vxor_vv, xor)
1822GEN_OPIVX_GVEC_TRANS(vand_vx, ands)
1823GEN_OPIVX_GVEC_TRANS(vor_vx,  ors)
1824GEN_OPIVX_GVEC_TRANS(vxor_vx, xors)
1825GEN_OPIVI_GVEC_TRANS(vand_vi, IMM_SX, vand_vx, andi)
1826GEN_OPIVI_GVEC_TRANS(vor_vi, IMM_SX, vor_vx,  ori)
1827GEN_OPIVI_GVEC_TRANS(vxor_vi, IMM_SX, vxor_vx, xori)
1828
1829/* Vector Single-Width Bit Shift Instructions */
1830GEN_OPIVV_GVEC_TRANS(vsll_vv,  shlv)
1831GEN_OPIVV_GVEC_TRANS(vsrl_vv,  shrv)
1832GEN_OPIVV_GVEC_TRANS(vsra_vv,  sarv)
1833
1834typedef void GVecGen2sFn32(unsigned, uint32_t, uint32_t, TCGv_i32,
1835                           uint32_t, uint32_t);
1836
1837static inline bool
1838do_opivx_gvec_shift(DisasContext *s, arg_rmrr *a, GVecGen2sFn32 *gvec_fn,
1839                    gen_helper_opivx *fn)
1840{
1841    if (!opivx_check(s, a)) {
1842        return false;
1843    }
1844
1845    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
1846        TCGv_i32 src1 = tcg_temp_new_i32();
1847
1848        tcg_gen_trunc_tl_i32(src1, get_gpr(s, a->rs1, EXT_NONE));
1849        tcg_gen_extract_i32(src1, src1, 0, s->sew + 3);
1850        gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2),
1851                src1, MAXSZ(s), MAXSZ(s));
1852
1853        tcg_temp_free_i32(src1);
1854        mark_vs_dirty(s);
1855        return true;
1856    }
1857    return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s);
1858}
1859
1860#define GEN_OPIVX_GVEC_SHIFT_TRANS(NAME, SUF) \
1861static bool trans_##NAME(DisasContext *s, arg_rmrr *a)                    \
1862{                                                                         \
1863    static gen_helper_opivx * const fns[4] = {                            \
1864        gen_helper_##NAME##_b, gen_helper_##NAME##_h,                     \
1865        gen_helper_##NAME##_w, gen_helper_##NAME##_d,                     \
1866    };                                                                    \
1867                                                                          \
1868    return do_opivx_gvec_shift(s, a, tcg_gen_gvec_##SUF, fns[s->sew]);    \
1869}
1870
1871GEN_OPIVX_GVEC_SHIFT_TRANS(vsll_vx,  shls)
1872GEN_OPIVX_GVEC_SHIFT_TRANS(vsrl_vx,  shrs)
1873GEN_OPIVX_GVEC_SHIFT_TRANS(vsra_vx,  sars)
1874
1875GEN_OPIVI_GVEC_TRANS(vsll_vi, IMM_TRUNC_SEW, vsll_vx, shli)
1876GEN_OPIVI_GVEC_TRANS(vsrl_vi, IMM_TRUNC_SEW, vsrl_vx, shri)
1877GEN_OPIVI_GVEC_TRANS(vsra_vi, IMM_TRUNC_SEW, vsra_vx, sari)
1878
1879/* Vector Narrowing Integer Right Shift Instructions */
1880static bool opiwv_narrow_check(DisasContext *s, arg_rmrr *a)
1881{
1882    return require_rvv(s) &&
1883           vext_check_isa_ill(s) &&
1884           vext_check_sds(s, a->rd, a->rs1, a->rs2, a->vm);
1885}
1886
1887/* OPIVV with NARROW */
1888#define GEN_OPIWV_NARROW_TRANS(NAME)                               \
1889static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
1890{                                                                  \
1891    if (opiwv_narrow_check(s, a)) {                                \
1892        uint32_t data = 0;                                         \
1893        static gen_helper_gvec_4_ptr * const fns[3] = {            \
1894            gen_helper_##NAME##_b,                                 \
1895            gen_helper_##NAME##_h,                                 \
1896            gen_helper_##NAME##_w,                                 \
1897        };                                                         \
1898        TCGLabel *over = gen_new_label();                          \
1899        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
1900        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
1901                                                                   \
1902        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
1903        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
1904        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
1905        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
1906        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
1907                           vreg_ofs(s, a->rs1),                    \
1908                           vreg_ofs(s, a->rs2), cpu_env,           \
1909                           s->cfg_ptr->vlen / 8,                   \
1910                           s->cfg_ptr->vlen / 8, data,             \
1911                           fns[s->sew]);                           \
1912        mark_vs_dirty(s);                                          \
1913        gen_set_label(over);                                       \
1914        return true;                                               \
1915    }                                                              \
1916    return false;                                                  \
1917}
1918GEN_OPIWV_NARROW_TRANS(vnsra_wv)
1919GEN_OPIWV_NARROW_TRANS(vnsrl_wv)
1920
1921static bool opiwx_narrow_check(DisasContext *s, arg_rmrr *a)
1922{
1923    return require_rvv(s) &&
1924           vext_check_isa_ill(s) &&
1925           vext_check_sd(s, a->rd, a->rs2, a->vm);
1926}
1927
1928/* OPIVX with NARROW */
1929#define GEN_OPIWX_NARROW_TRANS(NAME)                                     \
1930static bool trans_##NAME(DisasContext *s, arg_rmrr *a)                   \
1931{                                                                        \
1932    if (opiwx_narrow_check(s, a)) {                                      \
1933        static gen_helper_opivx * const fns[3] = {                       \
1934            gen_helper_##NAME##_b,                                       \
1935            gen_helper_##NAME##_h,                                       \
1936            gen_helper_##NAME##_w,                                       \
1937        };                                                               \
1938        return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);\
1939    }                                                                    \
1940    return false;                                                        \
1941}
1942
1943GEN_OPIWX_NARROW_TRANS(vnsra_wx)
1944GEN_OPIWX_NARROW_TRANS(vnsrl_wx)
1945
1946/* OPIWI with NARROW */
1947#define GEN_OPIWI_NARROW_TRANS(NAME, IMM_MODE, OPIVX)                    \
1948static bool trans_##NAME(DisasContext *s, arg_rmrr *a)                   \
1949{                                                                        \
1950    if (opiwx_narrow_check(s, a)) {                                      \
1951        static gen_helper_opivx * const fns[3] = {                       \
1952            gen_helper_##OPIVX##_b,                                      \
1953            gen_helper_##OPIVX##_h,                                      \
1954            gen_helper_##OPIVX##_w,                                      \
1955        };                                                               \
1956        return opivi_trans(a->rd, a->rs1, a->rs2, a->vm,                 \
1957                           fns[s->sew], s, IMM_MODE);                    \
1958    }                                                                    \
1959    return false;                                                        \
1960}
1961
1962GEN_OPIWI_NARROW_TRANS(vnsra_wi, IMM_ZX, vnsra_wx)
1963GEN_OPIWI_NARROW_TRANS(vnsrl_wi, IMM_ZX, vnsrl_wx)
1964
1965/* Vector Integer Comparison Instructions */
1966/*
1967 * For all comparison instructions, an illegal instruction exception is raised
1968 * if the destination vector register overlaps a source vector register group
1969 * and LMUL > 1.
1970 */
1971static bool opivv_cmp_check(DisasContext *s, arg_rmrr *a)
1972{
1973    return require_rvv(s) &&
1974           vext_check_isa_ill(s) &&
1975           vext_check_mss(s, a->rd, a->rs1, a->rs2);
1976}
1977
1978GEN_OPIVV_TRANS(vmseq_vv, opivv_cmp_check)
1979GEN_OPIVV_TRANS(vmsne_vv, opivv_cmp_check)
1980GEN_OPIVV_TRANS(vmsltu_vv, opivv_cmp_check)
1981GEN_OPIVV_TRANS(vmslt_vv, opivv_cmp_check)
1982GEN_OPIVV_TRANS(vmsleu_vv, opivv_cmp_check)
1983GEN_OPIVV_TRANS(vmsle_vv, opivv_cmp_check)
1984
1985static bool opivx_cmp_check(DisasContext *s, arg_rmrr *a)
1986{
1987    return require_rvv(s) &&
1988           vext_check_isa_ill(s) &&
1989           vext_check_ms(s, a->rd, a->rs2);
1990}
1991
1992GEN_OPIVX_TRANS(vmseq_vx, opivx_cmp_check)
1993GEN_OPIVX_TRANS(vmsne_vx, opivx_cmp_check)
1994GEN_OPIVX_TRANS(vmsltu_vx, opivx_cmp_check)
1995GEN_OPIVX_TRANS(vmslt_vx, opivx_cmp_check)
1996GEN_OPIVX_TRANS(vmsleu_vx, opivx_cmp_check)
1997GEN_OPIVX_TRANS(vmsle_vx, opivx_cmp_check)
1998GEN_OPIVX_TRANS(vmsgtu_vx, opivx_cmp_check)
1999GEN_OPIVX_TRANS(vmsgt_vx, opivx_cmp_check)
2000
2001GEN_OPIVI_TRANS(vmseq_vi, IMM_SX, vmseq_vx, opivx_cmp_check)
2002GEN_OPIVI_TRANS(vmsne_vi, IMM_SX, vmsne_vx, opivx_cmp_check)
2003GEN_OPIVI_TRANS(vmsleu_vi, IMM_SX, vmsleu_vx, opivx_cmp_check)
2004GEN_OPIVI_TRANS(vmsle_vi, IMM_SX, vmsle_vx, opivx_cmp_check)
2005GEN_OPIVI_TRANS(vmsgtu_vi, IMM_SX, vmsgtu_vx, opivx_cmp_check)
2006GEN_OPIVI_TRANS(vmsgt_vi, IMM_SX, vmsgt_vx, opivx_cmp_check)
2007
2008/* Vector Integer Min/Max Instructions */
2009GEN_OPIVV_GVEC_TRANS(vminu_vv, umin)
2010GEN_OPIVV_GVEC_TRANS(vmin_vv,  smin)
2011GEN_OPIVV_GVEC_TRANS(vmaxu_vv, umax)
2012GEN_OPIVV_GVEC_TRANS(vmax_vv,  smax)
2013GEN_OPIVX_TRANS(vminu_vx, opivx_check)
2014GEN_OPIVX_TRANS(vmin_vx,  opivx_check)
2015GEN_OPIVX_TRANS(vmaxu_vx, opivx_check)
2016GEN_OPIVX_TRANS(vmax_vx,  opivx_check)
2017
2018/* Vector Single-Width Integer Multiply Instructions */
2019
2020static bool vmulh_vv_check(DisasContext *s, arg_rmrr *a)
2021{
2022    /*
2023     * All Zve* extensions support all vector integer instructions,
2024     * except that the vmulh integer multiply variants
2025     * that return the high word of the product
2026     * (vmulh.vv, vmulh.vx, vmulhu.vv, vmulhu.vx, vmulhsu.vv, vmulhsu.vx)
2027     * are not included for EEW=64 in Zve64*. (Section 18.2)
2028     */
2029    return opivv_check(s, a) &&
2030           (!has_ext(s, RVV) &&
2031            s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true);
2032}
2033
2034static bool vmulh_vx_check(DisasContext *s, arg_rmrr *a)
2035{
2036    /*
2037     * All Zve* extensions support all vector integer instructions,
2038     * except that the vmulh integer multiply variants
2039     * that return the high word of the product
2040     * (vmulh.vv, vmulh.vx, vmulhu.vv, vmulhu.vx, vmulhsu.vv, vmulhsu.vx)
2041     * are not included for EEW=64 in Zve64*. (Section 18.2)
2042     */
2043    return opivx_check(s, a) &&
2044           (!has_ext(s, RVV) &&
2045            s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true);
2046}
2047
2048GEN_OPIVV_GVEC_TRANS(vmul_vv,  mul)
2049GEN_OPIVV_TRANS(vmulh_vv, vmulh_vv_check)
2050GEN_OPIVV_TRANS(vmulhu_vv, vmulh_vv_check)
2051GEN_OPIVV_TRANS(vmulhsu_vv, vmulh_vv_check)
2052GEN_OPIVX_GVEC_TRANS(vmul_vx,  muls)
2053GEN_OPIVX_TRANS(vmulh_vx, vmulh_vx_check)
2054GEN_OPIVX_TRANS(vmulhu_vx, vmulh_vx_check)
2055GEN_OPIVX_TRANS(vmulhsu_vx, vmulh_vx_check)
2056
2057/* Vector Integer Divide Instructions */
2058GEN_OPIVV_TRANS(vdivu_vv, opivv_check)
2059GEN_OPIVV_TRANS(vdiv_vv, opivv_check)
2060GEN_OPIVV_TRANS(vremu_vv, opivv_check)
2061GEN_OPIVV_TRANS(vrem_vv, opivv_check)
2062GEN_OPIVX_TRANS(vdivu_vx, opivx_check)
2063GEN_OPIVX_TRANS(vdiv_vx, opivx_check)
2064GEN_OPIVX_TRANS(vremu_vx, opivx_check)
2065GEN_OPIVX_TRANS(vrem_vx, opivx_check)
2066
2067/* Vector Widening Integer Multiply Instructions */
2068GEN_OPIVV_WIDEN_TRANS(vwmul_vv, opivv_widen_check)
2069GEN_OPIVV_WIDEN_TRANS(vwmulu_vv, opivv_widen_check)
2070GEN_OPIVV_WIDEN_TRANS(vwmulsu_vv, opivv_widen_check)
2071GEN_OPIVX_WIDEN_TRANS(vwmul_vx)
2072GEN_OPIVX_WIDEN_TRANS(vwmulu_vx)
2073GEN_OPIVX_WIDEN_TRANS(vwmulsu_vx)
2074
2075/* Vector Single-Width Integer Multiply-Add Instructions */
2076GEN_OPIVV_TRANS(vmacc_vv, opivv_check)
2077GEN_OPIVV_TRANS(vnmsac_vv, opivv_check)
2078GEN_OPIVV_TRANS(vmadd_vv, opivv_check)
2079GEN_OPIVV_TRANS(vnmsub_vv, opivv_check)
2080GEN_OPIVX_TRANS(vmacc_vx, opivx_check)
2081GEN_OPIVX_TRANS(vnmsac_vx, opivx_check)
2082GEN_OPIVX_TRANS(vmadd_vx, opivx_check)
2083GEN_OPIVX_TRANS(vnmsub_vx, opivx_check)
2084
2085/* Vector Widening Integer Multiply-Add Instructions */
2086GEN_OPIVV_WIDEN_TRANS(vwmaccu_vv, opivv_widen_check)
2087GEN_OPIVV_WIDEN_TRANS(vwmacc_vv, opivv_widen_check)
2088GEN_OPIVV_WIDEN_TRANS(vwmaccsu_vv, opivv_widen_check)
2089GEN_OPIVX_WIDEN_TRANS(vwmaccu_vx)
2090GEN_OPIVX_WIDEN_TRANS(vwmacc_vx)
2091GEN_OPIVX_WIDEN_TRANS(vwmaccsu_vx)
2092GEN_OPIVX_WIDEN_TRANS(vwmaccus_vx)
2093
2094/* Vector Integer Merge and Move Instructions */
2095static bool trans_vmv_v_v(DisasContext *s, arg_vmv_v_v *a)
2096{
2097    if (require_rvv(s) &&
2098        vext_check_isa_ill(s) &&
2099        /* vmv.v.v has rs2 = 0 and vm = 1 */
2100        vext_check_sss(s, a->rd, a->rs1, 0, 1)) {
2101        if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2102            tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd),
2103                             vreg_ofs(s, a->rs1),
2104                             MAXSZ(s), MAXSZ(s));
2105        } else {
2106            uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2107            data = FIELD_DP32(data, VDATA, VTA, s->vta);
2108            static gen_helper_gvec_2_ptr * const fns[4] = {
2109                gen_helper_vmv_v_v_b, gen_helper_vmv_v_v_h,
2110                gen_helper_vmv_v_v_w, gen_helper_vmv_v_v_d,
2111            };
2112            TCGLabel *over = gen_new_label();
2113            tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2114            tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2115
2116            tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1),
2117                               cpu_env, s->cfg_ptr->vlen / 8,
2118                               s->cfg_ptr->vlen / 8, data,
2119                               fns[s->sew]);
2120            gen_set_label(over);
2121        }
2122        mark_vs_dirty(s);
2123        return true;
2124    }
2125    return false;
2126}
2127
2128typedef void gen_helper_vmv_vx(TCGv_ptr, TCGv_i64, TCGv_env, TCGv_i32);
2129static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a)
2130{
2131    if (require_rvv(s) &&
2132        vext_check_isa_ill(s) &&
2133        /* vmv.v.x has rs2 = 0 and vm = 1 */
2134        vext_check_ss(s, a->rd, 0, 1)) {
2135        TCGv s1;
2136        TCGLabel *over = gen_new_label();
2137        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2138        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2139
2140        s1 = get_gpr(s, a->rs1, EXT_SIGN);
2141
2142        if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2143            if (get_xl(s) == MXL_RV32 && s->sew == MO_64) {
2144                TCGv_i64 s1_i64 = tcg_temp_new_i64();
2145                tcg_gen_ext_tl_i64(s1_i64, s1);
2146                tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
2147                                     MAXSZ(s), MAXSZ(s), s1_i64);
2148                tcg_temp_free_i64(s1_i64);
2149            } else {
2150                tcg_gen_gvec_dup_tl(s->sew, vreg_ofs(s, a->rd),
2151                                    MAXSZ(s), MAXSZ(s), s1);
2152            }
2153        } else {
2154            TCGv_i32 desc;
2155            TCGv_i64 s1_i64 = tcg_temp_new_i64();
2156            TCGv_ptr dest = tcg_temp_new_ptr();
2157            uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2158            data = FIELD_DP32(data, VDATA, VTA, s->vta);
2159            static gen_helper_vmv_vx * const fns[4] = {
2160                gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h,
2161                gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d,
2162            };
2163
2164            tcg_gen_ext_tl_i64(s1_i64, s1);
2165            desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
2166                                              s->cfg_ptr->vlen / 8, data));
2167            tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, a->rd));
2168            fns[s->sew](dest, s1_i64, cpu_env, desc);
2169
2170            tcg_temp_free_ptr(dest);
2171            tcg_temp_free_i64(s1_i64);
2172        }
2173
2174        mark_vs_dirty(s);
2175        gen_set_label(over);
2176        return true;
2177    }
2178    return false;
2179}
2180
2181static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a)
2182{
2183    if (require_rvv(s) &&
2184        vext_check_isa_ill(s) &&
2185        /* vmv.v.i has rs2 = 0 and vm = 1 */
2186        vext_check_ss(s, a->rd, 0, 1)) {
2187        int64_t simm = sextract64(a->rs1, 0, 5);
2188        if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2189            tcg_gen_gvec_dup_imm(s->sew, vreg_ofs(s, a->rd),
2190                                 MAXSZ(s), MAXSZ(s), simm);
2191            mark_vs_dirty(s);
2192        } else {
2193            TCGv_i32 desc;
2194            TCGv_i64 s1;
2195            TCGv_ptr dest;
2196            uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2197            data = FIELD_DP32(data, VDATA, VTA, s->vta);
2198            static gen_helper_vmv_vx * const fns[4] = {
2199                gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h,
2200                gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d,
2201            };
2202            TCGLabel *over = gen_new_label();
2203            tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2204            tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2205
2206            s1 = tcg_constant_i64(simm);
2207            dest = tcg_temp_new_ptr();
2208            desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
2209                                              s->cfg_ptr->vlen / 8, data));
2210            tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, a->rd));
2211            fns[s->sew](dest, s1, cpu_env, desc);
2212
2213            tcg_temp_free_ptr(dest);
2214            mark_vs_dirty(s);
2215            gen_set_label(over);
2216        }
2217        return true;
2218    }
2219    return false;
2220}
2221
2222GEN_OPIVV_TRANS(vmerge_vvm, opivv_vadc_check)
2223GEN_OPIVX_TRANS(vmerge_vxm, opivx_vadc_check)
2224GEN_OPIVI_TRANS(vmerge_vim, IMM_SX, vmerge_vxm, opivx_vadc_check)
2225
2226/*
2227 *** Vector Fixed-Point Arithmetic Instructions
2228 */
2229
2230/* Vector Single-Width Saturating Add and Subtract */
2231GEN_OPIVV_TRANS(vsaddu_vv, opivv_check)
2232GEN_OPIVV_TRANS(vsadd_vv,  opivv_check)
2233GEN_OPIVV_TRANS(vssubu_vv, opivv_check)
2234GEN_OPIVV_TRANS(vssub_vv,  opivv_check)
2235GEN_OPIVX_TRANS(vsaddu_vx,  opivx_check)
2236GEN_OPIVX_TRANS(vsadd_vx,  opivx_check)
2237GEN_OPIVX_TRANS(vssubu_vx,  opivx_check)
2238GEN_OPIVX_TRANS(vssub_vx,  opivx_check)
2239GEN_OPIVI_TRANS(vsaddu_vi, IMM_SX, vsaddu_vx, opivx_check)
2240GEN_OPIVI_TRANS(vsadd_vi, IMM_SX, vsadd_vx, opivx_check)
2241
2242/* Vector Single-Width Averaging Add and Subtract */
2243GEN_OPIVV_TRANS(vaadd_vv, opivv_check)
2244GEN_OPIVV_TRANS(vaaddu_vv, opivv_check)
2245GEN_OPIVV_TRANS(vasub_vv, opivv_check)
2246GEN_OPIVV_TRANS(vasubu_vv, opivv_check)
2247GEN_OPIVX_TRANS(vaadd_vx,  opivx_check)
2248GEN_OPIVX_TRANS(vaaddu_vx,  opivx_check)
2249GEN_OPIVX_TRANS(vasub_vx,  opivx_check)
2250GEN_OPIVX_TRANS(vasubu_vx,  opivx_check)
2251
2252/* Vector Single-Width Fractional Multiply with Rounding and Saturation */
2253
2254static bool vsmul_vv_check(DisasContext *s, arg_rmrr *a)
2255{
2256    /*
2257     * All Zve* extensions support all vector fixed-point arithmetic
2258     * instructions, except that vsmul.vv and vsmul.vx are not supported
2259     * for EEW=64 in Zve64*. (Section 18.2)
2260     */
2261    return opivv_check(s, a) &&
2262           (!has_ext(s, RVV) &&
2263            s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true);
2264}
2265
2266static bool vsmul_vx_check(DisasContext *s, arg_rmrr *a)
2267{
2268    /*
2269     * All Zve* extensions support all vector fixed-point arithmetic
2270     * instructions, except that vsmul.vv and vsmul.vx are not supported
2271     * for EEW=64 in Zve64*. (Section 18.2)
2272     */
2273    return opivx_check(s, a) &&
2274           (!has_ext(s, RVV) &&
2275            s->cfg_ptr->ext_zve64f ? s->sew != MO_64 : true);
2276}
2277
2278GEN_OPIVV_TRANS(vsmul_vv, vsmul_vv_check)
2279GEN_OPIVX_TRANS(vsmul_vx,  vsmul_vx_check)
2280
2281/* Vector Single-Width Scaling Shift Instructions */
2282GEN_OPIVV_TRANS(vssrl_vv, opivv_check)
2283GEN_OPIVV_TRANS(vssra_vv, opivv_check)
2284GEN_OPIVX_TRANS(vssrl_vx,  opivx_check)
2285GEN_OPIVX_TRANS(vssra_vx,  opivx_check)
2286GEN_OPIVI_TRANS(vssrl_vi, IMM_TRUNC_SEW, vssrl_vx, opivx_check)
2287GEN_OPIVI_TRANS(vssra_vi, IMM_TRUNC_SEW, vssra_vx, opivx_check)
2288
2289/* Vector Narrowing Fixed-Point Clip Instructions */
2290GEN_OPIWV_NARROW_TRANS(vnclipu_wv)
2291GEN_OPIWV_NARROW_TRANS(vnclip_wv)
2292GEN_OPIWX_NARROW_TRANS(vnclipu_wx)
2293GEN_OPIWX_NARROW_TRANS(vnclip_wx)
2294GEN_OPIWI_NARROW_TRANS(vnclipu_wi, IMM_ZX, vnclipu_wx)
2295GEN_OPIWI_NARROW_TRANS(vnclip_wi, IMM_ZX, vnclip_wx)
2296
2297/*
2298 *** Vector Float Point Arithmetic Instructions
2299 */
2300
2301/*
2302 * As RVF-only cpus always have values NaN-boxed to 64-bits,
2303 * RVF and RVD can be treated equally.
2304 * We don't have to deal with the cases of: SEW > FLEN.
2305 *
2306 * If SEW < FLEN, check whether input fp register is a valid
2307 * NaN-boxed value, in which case the least-significant SEW bits
2308 * of the f regsiter are used, else the canonical NaN value is used.
2309 */
2310static void do_nanbox(DisasContext *s, TCGv_i64 out, TCGv_i64 in)
2311{
2312    switch (s->sew) {
2313    case 1:
2314        gen_check_nanbox_h(out, in);
2315        break;
2316    case 2:
2317        gen_check_nanbox_s(out, in);
2318        break;
2319    case 3:
2320        tcg_gen_mov_i64(out, in);
2321        break;
2322    default:
2323        g_assert_not_reached();
2324    }
2325}
2326
2327/* Vector Single-Width Floating-Point Add/Subtract Instructions */
2328
2329/*
2330 * If the current SEW does not correspond to a supported IEEE floating-point
2331 * type, an illegal instruction exception is raised.
2332 */
2333static bool opfvv_check(DisasContext *s, arg_rmrr *a)
2334{
2335    return require_rvv(s) &&
2336           require_rvf(s) &&
2337           vext_check_isa_ill(s) &&
2338           vext_check_sss(s, a->rd, a->rs1, a->rs2, a->vm) &&
2339           require_zve32f(s) &&
2340           require_zve64f(s);
2341}
2342
2343/* OPFVV without GVEC IR */
2344#define GEN_OPFVV_TRANS(NAME, CHECK)                               \
2345static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
2346{                                                                  \
2347    if (CHECK(s, a)) {                                             \
2348        uint32_t data = 0;                                         \
2349        static gen_helper_gvec_4_ptr * const fns[3] = {            \
2350            gen_helper_##NAME##_h,                                 \
2351            gen_helper_##NAME##_w,                                 \
2352            gen_helper_##NAME##_d,                                 \
2353        };                                                         \
2354        TCGLabel *over = gen_new_label();                          \
2355        gen_set_rm(s, RISCV_FRM_DYN);                              \
2356        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
2357        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
2358                                                                   \
2359        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
2360        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
2361        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
2362        data =                                                     \
2363            FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
2364        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
2365        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
2366                           vreg_ofs(s, a->rs1),                    \
2367                           vreg_ofs(s, a->rs2), cpu_env,           \
2368                           s->cfg_ptr->vlen / 8,                   \
2369                           s->cfg_ptr->vlen / 8, data,             \
2370                           fns[s->sew - 1]);                       \
2371        mark_vs_dirty(s);                                          \
2372        gen_set_label(over);                                       \
2373        return true;                                               \
2374    }                                                              \
2375    return false;                                                  \
2376}
2377GEN_OPFVV_TRANS(vfadd_vv, opfvv_check)
2378GEN_OPFVV_TRANS(vfsub_vv, opfvv_check)
2379
2380typedef void gen_helper_opfvf(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_ptr,
2381                              TCGv_env, TCGv_i32);
2382
2383static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2,
2384                        uint32_t data, gen_helper_opfvf *fn, DisasContext *s)
2385{
2386    TCGv_ptr dest, src2, mask;
2387    TCGv_i32 desc;
2388    TCGv_i64 t1;
2389
2390    TCGLabel *over = gen_new_label();
2391    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2392    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2393
2394    dest = tcg_temp_new_ptr();
2395    mask = tcg_temp_new_ptr();
2396    src2 = tcg_temp_new_ptr();
2397    desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
2398                                      s->cfg_ptr->vlen / 8, data));
2399
2400    tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, vd));
2401    tcg_gen_addi_ptr(src2, cpu_env, vreg_ofs(s, vs2));
2402    tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
2403
2404    /* NaN-box f[rs1] */
2405    t1 = tcg_temp_new_i64();
2406    do_nanbox(s, t1, cpu_fpr[rs1]);
2407
2408    fn(dest, mask, t1, src2, cpu_env, desc);
2409
2410    tcg_temp_free_ptr(dest);
2411    tcg_temp_free_ptr(mask);
2412    tcg_temp_free_ptr(src2);
2413    tcg_temp_free_i64(t1);
2414    mark_vs_dirty(s);
2415    gen_set_label(over);
2416    return true;
2417}
2418
2419/*
2420 * If the current SEW does not correspond to a supported IEEE floating-point
2421 * type, an illegal instruction exception is raised
2422 */
2423static bool opfvf_check(DisasContext *s, arg_rmrr *a)
2424{
2425    return require_rvv(s) &&
2426           require_rvf(s) &&
2427           vext_check_isa_ill(s) &&
2428           vext_check_ss(s, a->rd, a->rs2, a->vm) &&
2429           require_zve32f(s) &&
2430           require_zve64f(s);
2431}
2432
2433/* OPFVF without GVEC IR */
2434#define GEN_OPFVF_TRANS(NAME, CHECK)                              \
2435static bool trans_##NAME(DisasContext *s, arg_rmrr *a)            \
2436{                                                                 \
2437    if (CHECK(s, a)) {                                            \
2438        uint32_t data = 0;                                        \
2439        static gen_helper_opfvf *const fns[3] = {                 \
2440            gen_helper_##NAME##_h,                                \
2441            gen_helper_##NAME##_w,                                \
2442            gen_helper_##NAME##_d,                                \
2443        };                                                        \
2444        gen_set_rm(s, RISCV_FRM_DYN);                             \
2445        data = FIELD_DP32(data, VDATA, VM, a->vm);                \
2446        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);            \
2447        data = FIELD_DP32(data, VDATA, VTA, s->vta);              \
2448        data = FIELD_DP32(data, VDATA, VTA_ALL_1S,                \
2449                          s->cfg_vta_all_1s);                     \
2450        data = FIELD_DP32(data, VDATA, VMA, s->vma);              \
2451        return opfvf_trans(a->rd, a->rs1, a->rs2, data,           \
2452                           fns[s->sew - 1], s);                   \
2453    }                                                             \
2454    return false;                                                 \
2455}
2456
2457GEN_OPFVF_TRANS(vfadd_vf,  opfvf_check)
2458GEN_OPFVF_TRANS(vfsub_vf,  opfvf_check)
2459GEN_OPFVF_TRANS(vfrsub_vf,  opfvf_check)
2460
2461/* Vector Widening Floating-Point Add/Subtract Instructions */
2462static bool opfvv_widen_check(DisasContext *s, arg_rmrr *a)
2463{
2464    return require_rvv(s) &&
2465           require_scale_rvf(s) &&
2466           (s->sew != MO_8) &&
2467           vext_check_isa_ill(s) &&
2468           vext_check_dss(s, a->rd, a->rs1, a->rs2, a->vm) &&
2469           require_scale_zve32f(s) &&
2470           require_scale_zve64f(s);
2471}
2472
2473/* OPFVV with WIDEN */
2474#define GEN_OPFVV_WIDEN_TRANS(NAME, CHECK)                       \
2475static bool trans_##NAME(DisasContext *s, arg_rmrr *a)           \
2476{                                                                \
2477    if (CHECK(s, a)) {                                           \
2478        uint32_t data = 0;                                       \
2479        static gen_helper_gvec_4_ptr * const fns[2] = {          \
2480            gen_helper_##NAME##_h, gen_helper_##NAME##_w,        \
2481        };                                                       \
2482        TCGLabel *over = gen_new_label();                        \
2483        gen_set_rm(s, RISCV_FRM_DYN);                            \
2484        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);        \
2485        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);\
2486                                                                 \
2487        data = FIELD_DP32(data, VDATA, VM, a->vm);               \
2488        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);           \
2489        data = FIELD_DP32(data, VDATA, VTA, s->vta);             \
2490        data = FIELD_DP32(data, VDATA, VMA, s->vma);             \
2491        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),   \
2492                           vreg_ofs(s, a->rs1),                  \
2493                           vreg_ofs(s, a->rs2), cpu_env,         \
2494                           s->cfg_ptr->vlen / 8,                 \
2495                           s->cfg_ptr->vlen / 8, data,           \
2496                           fns[s->sew - 1]);                     \
2497        mark_vs_dirty(s);                                        \
2498        gen_set_label(over);                                     \
2499        return true;                                             \
2500    }                                                            \
2501    return false;                                                \
2502}
2503
2504GEN_OPFVV_WIDEN_TRANS(vfwadd_vv, opfvv_widen_check)
2505GEN_OPFVV_WIDEN_TRANS(vfwsub_vv, opfvv_widen_check)
2506
2507static bool opfvf_widen_check(DisasContext *s, arg_rmrr *a)
2508{
2509    return require_rvv(s) &&
2510           require_scale_rvf(s) &&
2511           (s->sew != MO_8) &&
2512           vext_check_isa_ill(s) &&
2513           vext_check_ds(s, a->rd, a->rs2, a->vm) &&
2514           require_scale_zve32f(s) &&
2515           require_scale_zve64f(s);
2516}
2517
2518/* OPFVF with WIDEN */
2519#define GEN_OPFVF_WIDEN_TRANS(NAME)                              \
2520static bool trans_##NAME(DisasContext *s, arg_rmrr *a)           \
2521{                                                                \
2522    if (opfvf_widen_check(s, a)) {                               \
2523        uint32_t data = 0;                                       \
2524        static gen_helper_opfvf *const fns[2] = {                \
2525            gen_helper_##NAME##_h, gen_helper_##NAME##_w,        \
2526        };                                                       \
2527        gen_set_rm(s, RISCV_FRM_DYN);                            \
2528        data = FIELD_DP32(data, VDATA, VM, a->vm);               \
2529        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);           \
2530        data = FIELD_DP32(data, VDATA, VTA, s->vta);             \
2531        data = FIELD_DP32(data, VDATA, VMA, s->vma);             \
2532        return opfvf_trans(a->rd, a->rs1, a->rs2, data,          \
2533                           fns[s->sew - 1], s);                  \
2534    }                                                            \
2535    return false;                                                \
2536}
2537
2538GEN_OPFVF_WIDEN_TRANS(vfwadd_vf)
2539GEN_OPFVF_WIDEN_TRANS(vfwsub_vf)
2540
2541static bool opfwv_widen_check(DisasContext *s, arg_rmrr *a)
2542{
2543    return require_rvv(s) &&
2544           require_scale_rvf(s) &&
2545           (s->sew != MO_8) &&
2546           vext_check_isa_ill(s) &&
2547           vext_check_dds(s, a->rd, a->rs1, a->rs2, a->vm) &&
2548           require_scale_zve32f(s) &&
2549           require_scale_zve64f(s);
2550}
2551
2552/* WIDEN OPFVV with WIDEN */
2553#define GEN_OPFWV_WIDEN_TRANS(NAME)                                \
2554static bool trans_##NAME(DisasContext *s, arg_rmrr *a)             \
2555{                                                                  \
2556    if (opfwv_widen_check(s, a)) {                                 \
2557        uint32_t data = 0;                                         \
2558        static gen_helper_gvec_4_ptr * const fns[2] = {            \
2559            gen_helper_##NAME##_h, gen_helper_##NAME##_w,          \
2560        };                                                         \
2561        TCGLabel *over = gen_new_label();                          \
2562        gen_set_rm(s, RISCV_FRM_DYN);                              \
2563        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
2564        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
2565                                                                   \
2566        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
2567        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
2568        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
2569        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
2570        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
2571                           vreg_ofs(s, a->rs1),                    \
2572                           vreg_ofs(s, a->rs2), cpu_env,           \
2573                           s->cfg_ptr->vlen / 8,                   \
2574                           s->cfg_ptr->vlen / 8, data,             \
2575                           fns[s->sew - 1]);                       \
2576        mark_vs_dirty(s);                                          \
2577        gen_set_label(over);                                       \
2578        return true;                                               \
2579    }                                                              \
2580    return false;                                                  \
2581}
2582
2583GEN_OPFWV_WIDEN_TRANS(vfwadd_wv)
2584GEN_OPFWV_WIDEN_TRANS(vfwsub_wv)
2585
2586static bool opfwf_widen_check(DisasContext *s, arg_rmrr *a)
2587{
2588    return require_rvv(s) &&
2589           require_scale_rvf(s) &&
2590           (s->sew != MO_8) &&
2591           vext_check_isa_ill(s) &&
2592           vext_check_dd(s, a->rd, a->rs2, a->vm) &&
2593           require_scale_zve32f(s) &&
2594           require_scale_zve64f(s);
2595}
2596
2597/* WIDEN OPFVF with WIDEN */
2598#define GEN_OPFWF_WIDEN_TRANS(NAME)                              \
2599static bool trans_##NAME(DisasContext *s, arg_rmrr *a)           \
2600{                                                                \
2601    if (opfwf_widen_check(s, a)) {                               \
2602        uint32_t data = 0;                                       \
2603        static gen_helper_opfvf *const fns[2] = {                \
2604            gen_helper_##NAME##_h, gen_helper_##NAME##_w,        \
2605        };                                                       \
2606        gen_set_rm(s, RISCV_FRM_DYN);                            \
2607        data = FIELD_DP32(data, VDATA, VM, a->vm);               \
2608        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);           \
2609        data = FIELD_DP32(data, VDATA, VTA, s->vta);             \
2610        data = FIELD_DP32(data, VDATA, VMA, s->vma);             \
2611        return opfvf_trans(a->rd, a->rs1, a->rs2, data,          \
2612                           fns[s->sew - 1], s);                  \
2613    }                                                            \
2614    return false;                                                \
2615}
2616
2617GEN_OPFWF_WIDEN_TRANS(vfwadd_wf)
2618GEN_OPFWF_WIDEN_TRANS(vfwsub_wf)
2619
2620/* Vector Single-Width Floating-Point Multiply/Divide Instructions */
2621GEN_OPFVV_TRANS(vfmul_vv, opfvv_check)
2622GEN_OPFVV_TRANS(vfdiv_vv, opfvv_check)
2623GEN_OPFVF_TRANS(vfmul_vf,  opfvf_check)
2624GEN_OPFVF_TRANS(vfdiv_vf,  opfvf_check)
2625GEN_OPFVF_TRANS(vfrdiv_vf,  opfvf_check)
2626
2627/* Vector Widening Floating-Point Multiply */
2628GEN_OPFVV_WIDEN_TRANS(vfwmul_vv, opfvv_widen_check)
2629GEN_OPFVF_WIDEN_TRANS(vfwmul_vf)
2630
2631/* Vector Single-Width Floating-Point Fused Multiply-Add Instructions */
2632GEN_OPFVV_TRANS(vfmacc_vv, opfvv_check)
2633GEN_OPFVV_TRANS(vfnmacc_vv, opfvv_check)
2634GEN_OPFVV_TRANS(vfmsac_vv, opfvv_check)
2635GEN_OPFVV_TRANS(vfnmsac_vv, opfvv_check)
2636GEN_OPFVV_TRANS(vfmadd_vv, opfvv_check)
2637GEN_OPFVV_TRANS(vfnmadd_vv, opfvv_check)
2638GEN_OPFVV_TRANS(vfmsub_vv, opfvv_check)
2639GEN_OPFVV_TRANS(vfnmsub_vv, opfvv_check)
2640GEN_OPFVF_TRANS(vfmacc_vf, opfvf_check)
2641GEN_OPFVF_TRANS(vfnmacc_vf, opfvf_check)
2642GEN_OPFVF_TRANS(vfmsac_vf, opfvf_check)
2643GEN_OPFVF_TRANS(vfnmsac_vf, opfvf_check)
2644GEN_OPFVF_TRANS(vfmadd_vf, opfvf_check)
2645GEN_OPFVF_TRANS(vfnmadd_vf, opfvf_check)
2646GEN_OPFVF_TRANS(vfmsub_vf, opfvf_check)
2647GEN_OPFVF_TRANS(vfnmsub_vf, opfvf_check)
2648
2649/* Vector Widening Floating-Point Fused Multiply-Add Instructions */
2650GEN_OPFVV_WIDEN_TRANS(vfwmacc_vv, opfvv_widen_check)
2651GEN_OPFVV_WIDEN_TRANS(vfwnmacc_vv, opfvv_widen_check)
2652GEN_OPFVV_WIDEN_TRANS(vfwmsac_vv, opfvv_widen_check)
2653GEN_OPFVV_WIDEN_TRANS(vfwnmsac_vv, opfvv_widen_check)
2654GEN_OPFVF_WIDEN_TRANS(vfwmacc_vf)
2655GEN_OPFVF_WIDEN_TRANS(vfwnmacc_vf)
2656GEN_OPFVF_WIDEN_TRANS(vfwmsac_vf)
2657GEN_OPFVF_WIDEN_TRANS(vfwnmsac_vf)
2658
2659/* Vector Floating-Point Square-Root Instruction */
2660
2661/*
2662 * If the current SEW does not correspond to a supported IEEE floating-point
2663 * type, an illegal instruction exception is raised
2664 */
2665static bool opfv_check(DisasContext *s, arg_rmr *a)
2666{
2667    return require_rvv(s) &&
2668           require_rvf(s) &&
2669           vext_check_isa_ill(s) &&
2670           /* OPFV instructions ignore vs1 check */
2671           vext_check_ss(s, a->rd, a->rs2, a->vm) &&
2672           require_zve32f(s) &&
2673           require_zve64f(s);
2674}
2675
2676static bool do_opfv(DisasContext *s, arg_rmr *a,
2677                    gen_helper_gvec_3_ptr *fn,
2678                    bool (*checkfn)(DisasContext *, arg_rmr *),
2679                    int rm)
2680{
2681    if (checkfn(s, a)) {
2682        if (rm != RISCV_FRM_DYN) {
2683            gen_set_rm(s, RISCV_FRM_DYN);
2684        }
2685
2686        uint32_t data = 0;
2687        TCGLabel *over = gen_new_label();
2688        gen_set_rm(s, rm);
2689        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2690        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2691
2692        data = FIELD_DP32(data, VDATA, VM, a->vm);
2693        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
2694        data = FIELD_DP32(data, VDATA, VTA, s->vta);
2695        data = FIELD_DP32(data, VDATA, VMA, s->vma);
2696        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
2697                           vreg_ofs(s, a->rs2), cpu_env,
2698                           s->cfg_ptr->vlen / 8,
2699                           s->cfg_ptr->vlen / 8, data, fn);
2700        mark_vs_dirty(s);
2701        gen_set_label(over);
2702        return true;
2703    }
2704    return false;
2705}
2706
2707#define GEN_OPFV_TRANS(NAME, CHECK, FRM)               \
2708static bool trans_##NAME(DisasContext *s, arg_rmr *a)  \
2709{                                                      \
2710    static gen_helper_gvec_3_ptr * const fns[3] = {    \
2711        gen_helper_##NAME##_h,                         \
2712        gen_helper_##NAME##_w,                         \
2713        gen_helper_##NAME##_d                          \
2714    };                                                 \
2715    return do_opfv(s, a, fns[s->sew - 1], CHECK, FRM); \
2716}
2717
2718GEN_OPFV_TRANS(vfsqrt_v, opfv_check, RISCV_FRM_DYN)
2719GEN_OPFV_TRANS(vfrsqrt7_v, opfv_check, RISCV_FRM_DYN)
2720GEN_OPFV_TRANS(vfrec7_v, opfv_check, RISCV_FRM_DYN)
2721
2722/* Vector Floating-Point MIN/MAX Instructions */
2723GEN_OPFVV_TRANS(vfmin_vv, opfvv_check)
2724GEN_OPFVV_TRANS(vfmax_vv, opfvv_check)
2725GEN_OPFVF_TRANS(vfmin_vf, opfvf_check)
2726GEN_OPFVF_TRANS(vfmax_vf, opfvf_check)
2727
2728/* Vector Floating-Point Sign-Injection Instructions */
2729GEN_OPFVV_TRANS(vfsgnj_vv, opfvv_check)
2730GEN_OPFVV_TRANS(vfsgnjn_vv, opfvv_check)
2731GEN_OPFVV_TRANS(vfsgnjx_vv, opfvv_check)
2732GEN_OPFVF_TRANS(vfsgnj_vf, opfvf_check)
2733GEN_OPFVF_TRANS(vfsgnjn_vf, opfvf_check)
2734GEN_OPFVF_TRANS(vfsgnjx_vf, opfvf_check)
2735
2736/* Vector Floating-Point Compare Instructions */
2737static bool opfvv_cmp_check(DisasContext *s, arg_rmrr *a)
2738{
2739    return require_rvv(s) &&
2740           require_rvf(s) &&
2741           vext_check_isa_ill(s) &&
2742           vext_check_mss(s, a->rd, a->rs1, a->rs2) &&
2743           require_zve32f(s) &&
2744           require_zve64f(s);
2745}
2746
2747GEN_OPFVV_TRANS(vmfeq_vv, opfvv_cmp_check)
2748GEN_OPFVV_TRANS(vmfne_vv, opfvv_cmp_check)
2749GEN_OPFVV_TRANS(vmflt_vv, opfvv_cmp_check)
2750GEN_OPFVV_TRANS(vmfle_vv, opfvv_cmp_check)
2751
2752static bool opfvf_cmp_check(DisasContext *s, arg_rmrr *a)
2753{
2754    return require_rvv(s) &&
2755           require_rvf(s) &&
2756           vext_check_isa_ill(s) &&
2757           vext_check_ms(s, a->rd, a->rs2) &&
2758           require_zve32f(s) &&
2759           require_zve64f(s);
2760}
2761
2762GEN_OPFVF_TRANS(vmfeq_vf, opfvf_cmp_check)
2763GEN_OPFVF_TRANS(vmfne_vf, opfvf_cmp_check)
2764GEN_OPFVF_TRANS(vmflt_vf, opfvf_cmp_check)
2765GEN_OPFVF_TRANS(vmfle_vf, opfvf_cmp_check)
2766GEN_OPFVF_TRANS(vmfgt_vf, opfvf_cmp_check)
2767GEN_OPFVF_TRANS(vmfge_vf, opfvf_cmp_check)
2768
2769/* Vector Floating-Point Classify Instruction */
2770GEN_OPFV_TRANS(vfclass_v, opfv_check, RISCV_FRM_DYN)
2771
2772/* Vector Floating-Point Merge Instruction */
2773GEN_OPFVF_TRANS(vfmerge_vfm,  opfvf_check)
2774
2775static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a)
2776{
2777    if (require_rvv(s) &&
2778        require_rvf(s) &&
2779        vext_check_isa_ill(s) &&
2780        require_align(a->rd, s->lmul) &&
2781        require_zve32f(s) &&
2782        require_zve64f(s)) {
2783        gen_set_rm(s, RISCV_FRM_DYN);
2784
2785        TCGv_i64 t1;
2786
2787        if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
2788            t1 = tcg_temp_new_i64();
2789            /* NaN-box f[rs1] */
2790            do_nanbox(s, t1, cpu_fpr[a->rs1]);
2791
2792            tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
2793                                 MAXSZ(s), MAXSZ(s), t1);
2794            mark_vs_dirty(s);
2795        } else {
2796            TCGv_ptr dest;
2797            TCGv_i32 desc;
2798            uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul);
2799            data = FIELD_DP32(data, VDATA, VTA, s->vta);
2800            data = FIELD_DP32(data, VDATA, VMA, s->vma);
2801            static gen_helper_vmv_vx * const fns[3] = {
2802                gen_helper_vmv_v_x_h,
2803                gen_helper_vmv_v_x_w,
2804                gen_helper_vmv_v_x_d,
2805            };
2806            TCGLabel *over = gen_new_label();
2807            tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
2808            tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
2809
2810            t1 = tcg_temp_new_i64();
2811            /* NaN-box f[rs1] */
2812            do_nanbox(s, t1, cpu_fpr[a->rs1]);
2813
2814            dest = tcg_temp_new_ptr();
2815            desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
2816                                              s->cfg_ptr->vlen / 8, data));
2817            tcg_gen_addi_ptr(dest, cpu_env, vreg_ofs(s, a->rd));
2818
2819            fns[s->sew - 1](dest, t1, cpu_env, desc);
2820
2821            tcg_temp_free_ptr(dest);
2822            mark_vs_dirty(s);
2823            gen_set_label(over);
2824        }
2825        tcg_temp_free_i64(t1);
2826        return true;
2827    }
2828    return false;
2829}
2830
2831/* Single-Width Floating-Point/Integer Type-Convert Instructions */
2832#define GEN_OPFV_CVT_TRANS(NAME, HELPER, FRM)               \
2833static bool trans_##NAME(DisasContext *s, arg_rmr *a)       \
2834{                                                           \
2835    static gen_helper_gvec_3_ptr * const fns[3] = {         \
2836        gen_helper_##HELPER##_h,                            \
2837        gen_helper_##HELPER##_w,                            \
2838        gen_helper_##HELPER##_d                             \
2839    };                                                      \
2840    return do_opfv(s, a, fns[s->sew - 1], opfv_check, FRM); \
2841}
2842
2843GEN_OPFV_CVT_TRANS(vfcvt_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_DYN)
2844GEN_OPFV_CVT_TRANS(vfcvt_x_f_v, vfcvt_x_f_v, RISCV_FRM_DYN)
2845GEN_OPFV_CVT_TRANS(vfcvt_f_xu_v, vfcvt_f_xu_v, RISCV_FRM_DYN)
2846GEN_OPFV_CVT_TRANS(vfcvt_f_x_v, vfcvt_f_x_v, RISCV_FRM_DYN)
2847/* Reuse the helper functions from vfcvt.xu.f.v and vfcvt.x.f.v */
2848GEN_OPFV_CVT_TRANS(vfcvt_rtz_xu_f_v, vfcvt_xu_f_v, RISCV_FRM_RTZ)
2849GEN_OPFV_CVT_TRANS(vfcvt_rtz_x_f_v, vfcvt_x_f_v, RISCV_FRM_RTZ)
2850
2851/* Widening Floating-Point/Integer Type-Convert Instructions */
2852
2853/*
2854 * If the current SEW does not correspond to a supported IEEE floating-point
2855 * type, an illegal instruction exception is raised
2856 */
2857static bool opfv_widen_check(DisasContext *s, arg_rmr *a)
2858{
2859    return require_rvv(s) &&
2860           vext_check_isa_ill(s) &&
2861           vext_check_ds(s, a->rd, a->rs2, a->vm);
2862}
2863
2864static bool opxfv_widen_check(DisasContext *s, arg_rmr *a)
2865{
2866    return opfv_widen_check(s, a) &&
2867           require_rvf(s) &&
2868           require_zve32f(s) &&
2869           require_zve64f(s);
2870}
2871
2872static bool opffv_widen_check(DisasContext *s, arg_rmr *a)
2873{
2874    return opfv_widen_check(s, a) &&
2875           require_scale_rvf(s) &&
2876           (s->sew != MO_8) &&
2877           require_scale_zve32f(s) &&
2878           require_scale_zve64f(s);
2879}
2880
2881#define GEN_OPFV_WIDEN_TRANS(NAME, CHECK, HELPER, FRM)             \
2882static bool trans_##NAME(DisasContext *s, arg_rmr *a)              \
2883{                                                                  \
2884    if (CHECK(s, a)) {                                             \
2885        if (FRM != RISCV_FRM_DYN) {                                \
2886            gen_set_rm(s, RISCV_FRM_DYN);                          \
2887        }                                                          \
2888                                                                   \
2889        uint32_t data = 0;                                         \
2890        static gen_helper_gvec_3_ptr * const fns[2] = {            \
2891            gen_helper_##HELPER##_h,                               \
2892            gen_helper_##HELPER##_w,                               \
2893        };                                                         \
2894        TCGLabel *over = gen_new_label();                          \
2895        gen_set_rm(s, FRM);                                        \
2896        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
2897        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
2898                                                                   \
2899        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
2900        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
2901        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
2902        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
2903        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
2904                           vreg_ofs(s, a->rs2), cpu_env,           \
2905                           s->cfg_ptr->vlen / 8,                   \
2906                           s->cfg_ptr->vlen / 8, data,             \
2907                           fns[s->sew - 1]);                       \
2908        mark_vs_dirty(s);                                          \
2909        gen_set_label(over);                                       \
2910        return true;                                               \
2911    }                                                              \
2912    return false;                                                  \
2913}
2914
2915GEN_OPFV_WIDEN_TRANS(vfwcvt_xu_f_v, opxfv_widen_check, vfwcvt_xu_f_v,
2916                     RISCV_FRM_DYN)
2917GEN_OPFV_WIDEN_TRANS(vfwcvt_x_f_v, opxfv_widen_check, vfwcvt_x_f_v,
2918                     RISCV_FRM_DYN)
2919GEN_OPFV_WIDEN_TRANS(vfwcvt_f_f_v, opffv_widen_check, vfwcvt_f_f_v,
2920                     RISCV_FRM_DYN)
2921/* Reuse the helper functions from vfwcvt.xu.f.v and vfwcvt.x.f.v */
2922GEN_OPFV_WIDEN_TRANS(vfwcvt_rtz_xu_f_v, opxfv_widen_check, vfwcvt_xu_f_v,
2923                     RISCV_FRM_RTZ)
2924GEN_OPFV_WIDEN_TRANS(vfwcvt_rtz_x_f_v, opxfv_widen_check, vfwcvt_x_f_v,
2925                     RISCV_FRM_RTZ)
2926
2927static bool opfxv_widen_check(DisasContext *s, arg_rmr *a)
2928{
2929    return require_rvv(s) &&
2930           require_scale_rvf(s) &&
2931           vext_check_isa_ill(s) &&
2932           /* OPFV widening instructions ignore vs1 check */
2933           vext_check_ds(s, a->rd, a->rs2, a->vm) &&
2934           require_scale_zve32f(s) &&
2935           require_scale_zve64f(s);
2936}
2937
2938#define GEN_OPFXV_WIDEN_TRANS(NAME)                                \
2939static bool trans_##NAME(DisasContext *s, arg_rmr *a)              \
2940{                                                                  \
2941    if (opfxv_widen_check(s, a)) {                                 \
2942        uint32_t data = 0;                                         \
2943        static gen_helper_gvec_3_ptr * const fns[3] = {            \
2944            gen_helper_##NAME##_b,                                 \
2945            gen_helper_##NAME##_h,                                 \
2946            gen_helper_##NAME##_w,                                 \
2947        };                                                         \
2948        TCGLabel *over = gen_new_label();                          \
2949        gen_set_rm(s, RISCV_FRM_DYN);                              \
2950        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
2951        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
2952                                                                   \
2953        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
2954        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
2955        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
2956        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
2957        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
2958                           vreg_ofs(s, a->rs2), cpu_env,           \
2959                           s->cfg_ptr->vlen / 8,                   \
2960                           s->cfg_ptr->vlen / 8, data,             \
2961                           fns[s->sew]);                           \
2962        mark_vs_dirty(s);                                          \
2963        gen_set_label(over);                                       \
2964        return true;                                               \
2965    }                                                              \
2966    return false;                                                  \
2967}
2968
2969GEN_OPFXV_WIDEN_TRANS(vfwcvt_f_xu_v)
2970GEN_OPFXV_WIDEN_TRANS(vfwcvt_f_x_v)
2971
2972/* Narrowing Floating-Point/Integer Type-Convert Instructions */
2973
2974/*
2975 * If the current SEW does not correspond to a supported IEEE floating-point
2976 * type, an illegal instruction exception is raised
2977 */
2978static bool opfv_narrow_check(DisasContext *s, arg_rmr *a)
2979{
2980    return require_rvv(s) &&
2981           vext_check_isa_ill(s) &&
2982           /* OPFV narrowing instructions ignore vs1 check */
2983           vext_check_sd(s, a->rd, a->rs2, a->vm);
2984}
2985
2986static bool opfxv_narrow_check(DisasContext *s, arg_rmr *a)
2987{
2988    return opfv_narrow_check(s, a) &&
2989           require_rvf(s) &&
2990           (s->sew != MO_64) &&
2991           require_zve32f(s) &&
2992           require_zve64f(s);
2993}
2994
2995static bool opffv_narrow_check(DisasContext *s, arg_rmr *a)
2996{
2997    return opfv_narrow_check(s, a) &&
2998           require_scale_rvf(s) &&
2999           (s->sew != MO_8) &&
3000           require_scale_zve32f(s) &&
3001           require_scale_zve64f(s);
3002}
3003
3004#define GEN_OPFV_NARROW_TRANS(NAME, CHECK, HELPER, FRM)            \
3005static bool trans_##NAME(DisasContext *s, arg_rmr *a)              \
3006{                                                                  \
3007    if (CHECK(s, a)) {                                             \
3008        if (FRM != RISCV_FRM_DYN) {                                \
3009            gen_set_rm(s, RISCV_FRM_DYN);                          \
3010        }                                                          \
3011                                                                   \
3012        uint32_t data = 0;                                         \
3013        static gen_helper_gvec_3_ptr * const fns[2] = {            \
3014            gen_helper_##HELPER##_h,                               \
3015            gen_helper_##HELPER##_w,                               \
3016        };                                                         \
3017        TCGLabel *over = gen_new_label();                          \
3018        gen_set_rm(s, FRM);                                        \
3019        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
3020        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
3021                                                                   \
3022        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
3023        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
3024        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
3025        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
3026        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
3027                           vreg_ofs(s, a->rs2), cpu_env,           \
3028                           s->cfg_ptr->vlen / 8,                   \
3029                           s->cfg_ptr->vlen / 8, data,             \
3030                           fns[s->sew - 1]);                       \
3031        mark_vs_dirty(s);                                          \
3032        gen_set_label(over);                                       \
3033        return true;                                               \
3034    }                                                              \
3035    return false;                                                  \
3036}
3037
3038GEN_OPFV_NARROW_TRANS(vfncvt_f_xu_w, opfxv_narrow_check, vfncvt_f_xu_w,
3039                      RISCV_FRM_DYN)
3040GEN_OPFV_NARROW_TRANS(vfncvt_f_x_w, opfxv_narrow_check, vfncvt_f_x_w,
3041                      RISCV_FRM_DYN)
3042GEN_OPFV_NARROW_TRANS(vfncvt_f_f_w, opffv_narrow_check, vfncvt_f_f_w,
3043                      RISCV_FRM_DYN)
3044/* Reuse the helper function from vfncvt.f.f.w */
3045GEN_OPFV_NARROW_TRANS(vfncvt_rod_f_f_w, opffv_narrow_check, vfncvt_f_f_w,
3046                      RISCV_FRM_ROD)
3047
3048static bool opxfv_narrow_check(DisasContext *s, arg_rmr *a)
3049{
3050    return require_rvv(s) &&
3051           require_scale_rvf(s) &&
3052           vext_check_isa_ill(s) &&
3053           /* OPFV narrowing instructions ignore vs1 check */
3054           vext_check_sd(s, a->rd, a->rs2, a->vm) &&
3055           require_scale_zve32f(s) &&
3056           require_scale_zve64f(s);
3057}
3058
3059#define GEN_OPXFV_NARROW_TRANS(NAME, HELPER, FRM)                  \
3060static bool trans_##NAME(DisasContext *s, arg_rmr *a)              \
3061{                                                                  \
3062    if (opxfv_narrow_check(s, a)) {                                \
3063        if (FRM != RISCV_FRM_DYN) {                                \
3064            gen_set_rm(s, RISCV_FRM_DYN);                          \
3065        }                                                          \
3066                                                                   \
3067        uint32_t data = 0;                                         \
3068        static gen_helper_gvec_3_ptr * const fns[3] = {            \
3069            gen_helper_##HELPER##_b,                               \
3070            gen_helper_##HELPER##_h,                               \
3071            gen_helper_##HELPER##_w,                               \
3072        };                                                         \
3073        TCGLabel *over = gen_new_label();                          \
3074        gen_set_rm(s, FRM);                                        \
3075        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
3076        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
3077                                                                   \
3078        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
3079        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
3080        data = FIELD_DP32(data, VDATA, VTA, s->vta);               \
3081        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
3082        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
3083                           vreg_ofs(s, a->rs2), cpu_env,           \
3084                           s->cfg_ptr->vlen / 8,                   \
3085                           s->cfg_ptr->vlen / 8, data,             \
3086                           fns[s->sew]);                           \
3087        mark_vs_dirty(s);                                          \
3088        gen_set_label(over);                                       \
3089        return true;                                               \
3090    }                                                              \
3091    return false;                                                  \
3092}
3093
3094GEN_OPXFV_NARROW_TRANS(vfncvt_xu_f_w, vfncvt_xu_f_w, RISCV_FRM_DYN)
3095GEN_OPXFV_NARROW_TRANS(vfncvt_x_f_w, vfncvt_x_f_w, RISCV_FRM_DYN)
3096/* Reuse the helper functions from vfncvt.xu.f.w and vfncvt.x.f.w */
3097GEN_OPXFV_NARROW_TRANS(vfncvt_rtz_xu_f_w, vfncvt_xu_f_w, RISCV_FRM_RTZ)
3098GEN_OPXFV_NARROW_TRANS(vfncvt_rtz_x_f_w, vfncvt_x_f_w, RISCV_FRM_RTZ)
3099
3100/*
3101 *** Vector Reduction Operations
3102 */
3103/* Vector Single-Width Integer Reduction Instructions */
3104static bool reduction_check(DisasContext *s, arg_rmrr *a)
3105{
3106    return require_rvv(s) &&
3107           vext_check_isa_ill(s) &&
3108           vext_check_reduction(s, a->rs2);
3109}
3110
3111GEN_OPIVV_TRANS(vredsum_vs, reduction_check)
3112GEN_OPIVV_TRANS(vredmaxu_vs, reduction_check)
3113GEN_OPIVV_TRANS(vredmax_vs, reduction_check)
3114GEN_OPIVV_TRANS(vredminu_vs, reduction_check)
3115GEN_OPIVV_TRANS(vredmin_vs, reduction_check)
3116GEN_OPIVV_TRANS(vredand_vs, reduction_check)
3117GEN_OPIVV_TRANS(vredor_vs, reduction_check)
3118GEN_OPIVV_TRANS(vredxor_vs, reduction_check)
3119
3120/* Vector Widening Integer Reduction Instructions */
3121static bool reduction_widen_check(DisasContext *s, arg_rmrr *a)
3122{
3123    return reduction_check(s, a) && (s->sew < MO_64) &&
3124           ((s->sew + 1) <= (s->cfg_ptr->elen >> 4));
3125}
3126
3127GEN_OPIVV_WIDEN_TRANS(vwredsum_vs, reduction_widen_check)
3128GEN_OPIVV_WIDEN_TRANS(vwredsumu_vs, reduction_widen_check)
3129
3130/* Vector Single-Width Floating-Point Reduction Instructions */
3131static bool freduction_check(DisasContext *s, arg_rmrr *a)
3132{
3133    return reduction_check(s, a) &&
3134           require_rvf(s) &&
3135           require_zve32f(s) &&
3136           require_zve64f(s);
3137}
3138
3139GEN_OPFVV_TRANS(vfredusum_vs, freduction_check)
3140GEN_OPFVV_TRANS(vfredosum_vs, freduction_check)
3141GEN_OPFVV_TRANS(vfredmax_vs, freduction_check)
3142GEN_OPFVV_TRANS(vfredmin_vs, freduction_check)
3143
3144/* Vector Widening Floating-Point Reduction Instructions */
3145static bool freduction_widen_check(DisasContext *s, arg_rmrr *a)
3146{
3147    return reduction_widen_check(s, a) &&
3148           require_scale_rvf(s) &&
3149           (s->sew != MO_8);
3150}
3151
3152GEN_OPFVV_WIDEN_TRANS(vfwredusum_vs, freduction_widen_check)
3153GEN_OPFVV_WIDEN_TRANS(vfwredosum_vs, freduction_widen_check)
3154
3155/*
3156 *** Vector Mask Operations
3157 */
3158
3159/* Vector Mask-Register Logical Instructions */
3160#define GEN_MM_TRANS(NAME)                                         \
3161static bool trans_##NAME(DisasContext *s, arg_r *a)                \
3162{                                                                  \
3163    if (require_rvv(s) &&                                          \
3164        vext_check_isa_ill(s)) {                                   \
3165        uint32_t data = 0;                                         \
3166        gen_helper_gvec_4_ptr *fn = gen_helper_##NAME;             \
3167        TCGLabel *over = gen_new_label();                          \
3168        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
3169        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \
3170                                                                   \
3171        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
3172        data =                                                     \
3173            FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
3174        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),     \
3175                           vreg_ofs(s, a->rs1),                    \
3176                           vreg_ofs(s, a->rs2), cpu_env,           \
3177                           s->cfg_ptr->vlen / 8,                   \
3178                           s->cfg_ptr->vlen / 8, data, fn);        \
3179        mark_vs_dirty(s);                                          \
3180        gen_set_label(over);                                       \
3181        return true;                                               \
3182    }                                                              \
3183    return false;                                                  \
3184}
3185
3186GEN_MM_TRANS(vmand_mm)
3187GEN_MM_TRANS(vmnand_mm)
3188GEN_MM_TRANS(vmandn_mm)
3189GEN_MM_TRANS(vmxor_mm)
3190GEN_MM_TRANS(vmor_mm)
3191GEN_MM_TRANS(vmnor_mm)
3192GEN_MM_TRANS(vmorn_mm)
3193GEN_MM_TRANS(vmxnor_mm)
3194
3195/* Vector count population in mask vcpop */
3196static bool trans_vcpop_m(DisasContext *s, arg_rmr *a)
3197{
3198    if (require_rvv(s) &&
3199        vext_check_isa_ill(s) &&
3200        s->vstart == 0) {
3201        TCGv_ptr src2, mask;
3202        TCGv dst;
3203        TCGv_i32 desc;
3204        uint32_t data = 0;
3205        data = FIELD_DP32(data, VDATA, VM, a->vm);
3206        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3207
3208        mask = tcg_temp_new_ptr();
3209        src2 = tcg_temp_new_ptr();
3210        dst = dest_gpr(s, a->rd);
3211        desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
3212                                          s->cfg_ptr->vlen / 8, data));
3213
3214        tcg_gen_addi_ptr(src2, cpu_env, vreg_ofs(s, a->rs2));
3215        tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
3216
3217        gen_helper_vcpop_m(dst, mask, src2, cpu_env, desc);
3218        gen_set_gpr(s, a->rd, dst);
3219
3220        tcg_temp_free_ptr(mask);
3221        tcg_temp_free_ptr(src2);
3222
3223        return true;
3224    }
3225    return false;
3226}
3227
3228/* vmfirst find-first-set mask bit */
3229static bool trans_vfirst_m(DisasContext *s, arg_rmr *a)
3230{
3231    if (require_rvv(s) &&
3232        vext_check_isa_ill(s) &&
3233        s->vstart == 0) {
3234        TCGv_ptr src2, mask;
3235        TCGv dst;
3236        TCGv_i32 desc;
3237        uint32_t data = 0;
3238        data = FIELD_DP32(data, VDATA, VM, a->vm);
3239        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3240
3241        mask = tcg_temp_new_ptr();
3242        src2 = tcg_temp_new_ptr();
3243        dst = dest_gpr(s, a->rd);
3244        desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8,
3245                                          s->cfg_ptr->vlen / 8, data));
3246
3247        tcg_gen_addi_ptr(src2, cpu_env, vreg_ofs(s, a->rs2));
3248        tcg_gen_addi_ptr(mask, cpu_env, vreg_ofs(s, 0));
3249
3250        gen_helper_vfirst_m(dst, mask, src2, cpu_env, desc);
3251        gen_set_gpr(s, a->rd, dst);
3252
3253        tcg_temp_free_ptr(mask);
3254        tcg_temp_free_ptr(src2);
3255        return true;
3256    }
3257    return false;
3258}
3259
3260/* vmsbf.m set-before-first mask bit */
3261/* vmsif.m set-includ-first mask bit */
3262/* vmsof.m set-only-first mask bit */
3263#define GEN_M_TRANS(NAME)                                          \
3264static bool trans_##NAME(DisasContext *s, arg_rmr *a)              \
3265{                                                                  \
3266    if (require_rvv(s) &&                                          \
3267        vext_check_isa_ill(s) &&                                   \
3268        require_vm(a->vm, a->rd) &&                                \
3269        (a->rd != a->rs2) &&                                       \
3270        (s->vstart == 0)) {                                        \
3271        uint32_t data = 0;                                         \
3272        gen_helper_gvec_3_ptr *fn = gen_helper_##NAME;             \
3273        TCGLabel *over = gen_new_label();                          \
3274        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);          \
3275                                                                   \
3276        data = FIELD_DP32(data, VDATA, VM, a->vm);                 \
3277        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);             \
3278        data =                                                     \
3279            FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\
3280        data = FIELD_DP32(data, VDATA, VMA, s->vma);               \
3281        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd),                     \
3282                           vreg_ofs(s, 0), vreg_ofs(s, a->rs2),    \
3283                           cpu_env, s->cfg_ptr->vlen / 8,          \
3284                           s->cfg_ptr->vlen / 8,                   \
3285                           data, fn);                              \
3286        mark_vs_dirty(s);                                          \
3287        gen_set_label(over);                                       \
3288        return true;                                               \
3289    }                                                              \
3290    return false;                                                  \
3291}
3292
3293GEN_M_TRANS(vmsbf_m)
3294GEN_M_TRANS(vmsif_m)
3295GEN_M_TRANS(vmsof_m)
3296
3297/*
3298 * Vector Iota Instruction
3299 *
3300 * 1. The destination register cannot overlap the source register.
3301 * 2. If masked, cannot overlap the mask register ('v0').
3302 * 3. An illegal instruction exception is raised if vstart is non-zero.
3303 */
3304static bool trans_viota_m(DisasContext *s, arg_viota_m *a)
3305{
3306    if (require_rvv(s) &&
3307        vext_check_isa_ill(s) &&
3308        !is_overlapped(a->rd, 1 << MAX(s->lmul, 0), a->rs2, 1) &&
3309        require_vm(a->vm, a->rd) &&
3310        require_align(a->rd, s->lmul) &&
3311        (s->vstart == 0)) {
3312        uint32_t data = 0;
3313        TCGLabel *over = gen_new_label();
3314        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3315
3316        data = FIELD_DP32(data, VDATA, VM, a->vm);
3317        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3318        data = FIELD_DP32(data, VDATA, VTA, s->vta);
3319        data = FIELD_DP32(data, VDATA, VMA, s->vma);
3320        static gen_helper_gvec_3_ptr * const fns[4] = {
3321            gen_helper_viota_m_b, gen_helper_viota_m_h,
3322            gen_helper_viota_m_w, gen_helper_viota_m_d,
3323        };
3324        tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3325                           vreg_ofs(s, a->rs2), cpu_env,
3326                           s->cfg_ptr->vlen / 8,
3327                           s->cfg_ptr->vlen / 8, data, fns[s->sew]);
3328        mark_vs_dirty(s);
3329        gen_set_label(over);
3330        return true;
3331    }
3332    return false;
3333}
3334
3335/* Vector Element Index Instruction */
3336static bool trans_vid_v(DisasContext *s, arg_vid_v *a)
3337{
3338    if (require_rvv(s) &&
3339        vext_check_isa_ill(s) &&
3340        require_align(a->rd, s->lmul) &&
3341        require_vm(a->vm, a->rd)) {
3342        uint32_t data = 0;
3343        TCGLabel *over = gen_new_label();
3344        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3345        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3346
3347        data = FIELD_DP32(data, VDATA, VM, a->vm);
3348        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3349        data = FIELD_DP32(data, VDATA, VTA, s->vta);
3350        data = FIELD_DP32(data, VDATA, VMA, s->vma);
3351        static gen_helper_gvec_2_ptr * const fns[4] = {
3352            gen_helper_vid_v_b, gen_helper_vid_v_h,
3353            gen_helper_vid_v_w, gen_helper_vid_v_d,
3354        };
3355        tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3356                           cpu_env, s->cfg_ptr->vlen / 8,
3357                           s->cfg_ptr->vlen / 8,
3358                           data, fns[s->sew]);
3359        mark_vs_dirty(s);
3360        gen_set_label(over);
3361        return true;
3362    }
3363    return false;
3364}
3365
3366/*
3367 *** Vector Permutation Instructions
3368 */
3369
3370static void load_element(TCGv_i64 dest, TCGv_ptr base,
3371                         int ofs, int sew, bool sign)
3372{
3373    switch (sew) {
3374    case MO_8:
3375        if (!sign) {
3376            tcg_gen_ld8u_i64(dest, base, ofs);
3377        } else {
3378            tcg_gen_ld8s_i64(dest, base, ofs);
3379        }
3380        break;
3381    case MO_16:
3382        if (!sign) {
3383            tcg_gen_ld16u_i64(dest, base, ofs);
3384        } else {
3385            tcg_gen_ld16s_i64(dest, base, ofs);
3386        }
3387        break;
3388    case MO_32:
3389        if (!sign) {
3390            tcg_gen_ld32u_i64(dest, base, ofs);
3391        } else {
3392            tcg_gen_ld32s_i64(dest, base, ofs);
3393        }
3394        break;
3395    case MO_64:
3396        tcg_gen_ld_i64(dest, base, ofs);
3397        break;
3398    default:
3399        g_assert_not_reached();
3400        break;
3401    }
3402}
3403
3404/* offset of the idx element with base regsiter r */
3405static uint32_t endian_ofs(DisasContext *s, int r, int idx)
3406{
3407#if HOST_BIG_ENDIAN
3408    return vreg_ofs(s, r) + ((idx ^ (7 >> s->sew)) << s->sew);
3409#else
3410    return vreg_ofs(s, r) + (idx << s->sew);
3411#endif
3412}
3413
3414/* adjust the index according to the endian */
3415static void endian_adjust(TCGv_i32 ofs, int sew)
3416{
3417#if HOST_BIG_ENDIAN
3418    tcg_gen_xori_i32(ofs, ofs, 7 >> sew);
3419#endif
3420}
3421
3422/* Load idx >= VLMAX ? 0 : vreg[idx] */
3423static void vec_element_loadx(DisasContext *s, TCGv_i64 dest,
3424                              int vreg, TCGv idx, int vlmax)
3425{
3426    TCGv_i32 ofs = tcg_temp_new_i32();
3427    TCGv_ptr base = tcg_temp_new_ptr();
3428    TCGv_i64 t_idx = tcg_temp_new_i64();
3429    TCGv_i64 t_vlmax, t_zero;
3430
3431    /*
3432     * Mask the index to the length so that we do
3433     * not produce an out-of-range load.
3434     */
3435    tcg_gen_trunc_tl_i32(ofs, idx);
3436    tcg_gen_andi_i32(ofs, ofs, vlmax - 1);
3437
3438    /* Convert the index to an offset. */
3439    endian_adjust(ofs, s->sew);
3440    tcg_gen_shli_i32(ofs, ofs, s->sew);
3441
3442    /* Convert the index to a pointer. */
3443    tcg_gen_ext_i32_ptr(base, ofs);
3444    tcg_gen_add_ptr(base, base, cpu_env);
3445
3446    /* Perform the load. */
3447    load_element(dest, base,
3448                 vreg_ofs(s, vreg), s->sew, false);
3449    tcg_temp_free_ptr(base);
3450    tcg_temp_free_i32(ofs);
3451
3452    /* Flush out-of-range indexing to zero.  */
3453    t_vlmax = tcg_constant_i64(vlmax);
3454    t_zero = tcg_constant_i64(0);
3455    tcg_gen_extu_tl_i64(t_idx, idx);
3456
3457    tcg_gen_movcond_i64(TCG_COND_LTU, dest, t_idx,
3458                        t_vlmax, dest, t_zero);
3459
3460    tcg_temp_free_i64(t_idx);
3461}
3462
3463static void vec_element_loadi(DisasContext *s, TCGv_i64 dest,
3464                              int vreg, int idx, bool sign)
3465{
3466    load_element(dest, cpu_env, endian_ofs(s, vreg, idx), s->sew, sign);
3467}
3468
3469/* Integer Scalar Move Instruction */
3470
3471static void store_element(TCGv_i64 val, TCGv_ptr base,
3472                          int ofs, int sew)
3473{
3474    switch (sew) {
3475    case MO_8:
3476        tcg_gen_st8_i64(val, base, ofs);
3477        break;
3478    case MO_16:
3479        tcg_gen_st16_i64(val, base, ofs);
3480        break;
3481    case MO_32:
3482        tcg_gen_st32_i64(val, base, ofs);
3483        break;
3484    case MO_64:
3485        tcg_gen_st_i64(val, base, ofs);
3486        break;
3487    default:
3488        g_assert_not_reached();
3489        break;
3490    }
3491}
3492
3493/*
3494 * Store vreg[idx] = val.
3495 * The index must be in range of VLMAX.
3496 */
3497static void vec_element_storei(DisasContext *s, int vreg,
3498                               int idx, TCGv_i64 val)
3499{
3500    store_element(val, cpu_env, endian_ofs(s, vreg, idx), s->sew);
3501}
3502
3503/* vmv.x.s rd, vs2 # x[rd] = vs2[0] */
3504static bool trans_vmv_x_s(DisasContext *s, arg_vmv_x_s *a)
3505{
3506    if (require_rvv(s) &&
3507        vext_check_isa_ill(s)) {
3508        TCGv_i64 t1;
3509        TCGv dest;
3510
3511        t1 = tcg_temp_new_i64();
3512        dest = tcg_temp_new();
3513        /*
3514         * load vreg and sign-extend to 64 bits,
3515         * then truncate to XLEN bits before storing to gpr.
3516         */
3517        vec_element_loadi(s, t1, a->rs2, 0, true);
3518        tcg_gen_trunc_i64_tl(dest, t1);
3519        gen_set_gpr(s, a->rd, dest);
3520        tcg_temp_free_i64(t1);
3521        tcg_temp_free(dest);
3522
3523        return true;
3524    }
3525    return false;
3526}
3527
3528/* vmv.s.x vd, rs1 # vd[0] = rs1 */
3529static bool trans_vmv_s_x(DisasContext *s, arg_vmv_s_x *a)
3530{
3531    if (require_rvv(s) &&
3532        vext_check_isa_ill(s)) {
3533        /* This instruction ignores LMUL and vector register groups */
3534        TCGv_i64 t1;
3535        TCGv s1;
3536        TCGLabel *over = gen_new_label();
3537
3538        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3539        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3540
3541        t1 = tcg_temp_new_i64();
3542
3543        /*
3544         * load gpr and sign-extend to 64 bits,
3545         * then truncate to SEW bits when storing to vreg.
3546         */
3547        s1 = get_gpr(s, a->rs1, EXT_NONE);
3548        tcg_gen_ext_tl_i64(t1, s1);
3549        vec_element_storei(s, a->rd, 0, t1);
3550        tcg_temp_free_i64(t1);
3551        mark_vs_dirty(s);
3552        gen_set_label(over);
3553        return true;
3554    }
3555    return false;
3556}
3557
3558/* Floating-Point Scalar Move Instructions */
3559static bool trans_vfmv_f_s(DisasContext *s, arg_vfmv_f_s *a)
3560{
3561    if (require_rvv(s) &&
3562        require_rvf(s) &&
3563        vext_check_isa_ill(s) &&
3564        require_zve32f(s) &&
3565        require_zve64f(s)) {
3566        gen_set_rm(s, RISCV_FRM_DYN);
3567
3568        unsigned int ofs = (8 << s->sew);
3569        unsigned int len = 64 - ofs;
3570        TCGv_i64 t_nan;
3571
3572        vec_element_loadi(s, cpu_fpr[a->rd], a->rs2, 0, false);
3573        /* NaN-box f[rd] as necessary for SEW */
3574        if (len) {
3575            t_nan = tcg_constant_i64(UINT64_MAX);
3576            tcg_gen_deposit_i64(cpu_fpr[a->rd], cpu_fpr[a->rd],
3577                                t_nan, ofs, len);
3578        }
3579
3580        mark_fs_dirty(s);
3581        return true;
3582    }
3583    return false;
3584}
3585
3586/* vfmv.s.f vd, rs1 # vd[0] = rs1 (vs2=0) */
3587static bool trans_vfmv_s_f(DisasContext *s, arg_vfmv_s_f *a)
3588{
3589    if (require_rvv(s) &&
3590        require_rvf(s) &&
3591        vext_check_isa_ill(s) &&
3592        require_zve32f(s) &&
3593        require_zve64f(s)) {
3594        gen_set_rm(s, RISCV_FRM_DYN);
3595
3596        /* The instructions ignore LMUL and vector register group. */
3597        TCGv_i64 t1;
3598        TCGLabel *over = gen_new_label();
3599
3600        /* if vl == 0 or vstart >= vl, skip vector register write back */
3601        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3602        tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3603
3604        /* NaN-box f[rs1] */
3605        t1 = tcg_temp_new_i64();
3606        do_nanbox(s, t1, cpu_fpr[a->rs1]);
3607
3608        vec_element_storei(s, a->rd, 0, t1);
3609        tcg_temp_free_i64(t1);
3610        mark_vs_dirty(s);
3611        gen_set_label(over);
3612        return true;
3613    }
3614    return false;
3615}
3616
3617/* Vector Slide Instructions */
3618static bool slideup_check(DisasContext *s, arg_rmrr *a)
3619{
3620    return require_rvv(s) &&
3621           vext_check_isa_ill(s) &&
3622           vext_check_slide(s, a->rd, a->rs2, a->vm, true);
3623}
3624
3625GEN_OPIVX_TRANS(vslideup_vx, slideup_check)
3626GEN_OPIVX_TRANS(vslide1up_vx, slideup_check)
3627GEN_OPIVI_TRANS(vslideup_vi, IMM_ZX, vslideup_vx, slideup_check)
3628
3629static bool slidedown_check(DisasContext *s, arg_rmrr *a)
3630{
3631    return require_rvv(s) &&
3632           vext_check_isa_ill(s) &&
3633           vext_check_slide(s, a->rd, a->rs2, a->vm, false);
3634}
3635
3636GEN_OPIVX_TRANS(vslidedown_vx, slidedown_check)
3637GEN_OPIVX_TRANS(vslide1down_vx, slidedown_check)
3638GEN_OPIVI_TRANS(vslidedown_vi, IMM_ZX, vslidedown_vx, slidedown_check)
3639
3640/* Vector Floating-Point Slide Instructions */
3641static bool fslideup_check(DisasContext *s, arg_rmrr *a)
3642{
3643    return slideup_check(s, a) &&
3644           require_rvf(s) &&
3645           require_zve32f(s) &&
3646           require_zve64f(s);
3647}
3648
3649static bool fslidedown_check(DisasContext *s, arg_rmrr *a)
3650{
3651    return slidedown_check(s, a) &&
3652           require_rvf(s) &&
3653           require_zve32f(s) &&
3654           require_zve64f(s);
3655}
3656
3657GEN_OPFVF_TRANS(vfslide1up_vf, fslideup_check)
3658GEN_OPFVF_TRANS(vfslide1down_vf, fslidedown_check)
3659
3660/* Vector Register Gather Instruction */
3661static bool vrgather_vv_check(DisasContext *s, arg_rmrr *a)
3662{
3663    return require_rvv(s) &&
3664           vext_check_isa_ill(s) &&
3665           require_align(a->rd, s->lmul) &&
3666           require_align(a->rs1, s->lmul) &&
3667           require_align(a->rs2, s->lmul) &&
3668           (a->rd != a->rs2 && a->rd != a->rs1) &&
3669           require_vm(a->vm, a->rd);
3670}
3671
3672static bool vrgatherei16_vv_check(DisasContext *s, arg_rmrr *a)
3673{
3674    int8_t emul = MO_16 - s->sew + s->lmul;
3675    return require_rvv(s) &&
3676           vext_check_isa_ill(s) &&
3677           (emul >= -3 && emul <= 3) &&
3678           require_align(a->rd, s->lmul) &&
3679           require_align(a->rs1, emul) &&
3680           require_align(a->rs2, s->lmul) &&
3681           (a->rd != a->rs2 && a->rd != a->rs1) &&
3682           !is_overlapped(a->rd, 1 << MAX(s->lmul, 0),
3683                          a->rs1, 1 << MAX(emul, 0)) &&
3684           !is_overlapped(a->rd, 1 << MAX(s->lmul, 0),
3685                          a->rs2, 1 << MAX(s->lmul, 0)) &&
3686           require_vm(a->vm, a->rd);
3687}
3688
3689GEN_OPIVV_TRANS(vrgather_vv, vrgather_vv_check)
3690GEN_OPIVV_TRANS(vrgatherei16_vv, vrgatherei16_vv_check)
3691
3692static bool vrgather_vx_check(DisasContext *s, arg_rmrr *a)
3693{
3694    return require_rvv(s) &&
3695           vext_check_isa_ill(s) &&
3696           require_align(a->rd, s->lmul) &&
3697           require_align(a->rs2, s->lmul) &&
3698           (a->rd != a->rs2) &&
3699           require_vm(a->vm, a->rd);
3700}
3701
3702/* vrgather.vx vd, vs2, rs1, vm # vd[i] = (x[rs1] >= VLMAX) ? 0 : vs2[rs1] */
3703static bool trans_vrgather_vx(DisasContext *s, arg_rmrr *a)
3704{
3705    if (!vrgather_vx_check(s, a)) {
3706        return false;
3707    }
3708
3709    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
3710        int scale = s->lmul - (s->sew + 3);
3711        int vlmax = s->cfg_ptr->vlen >> -scale;
3712        TCGv_i64 dest = tcg_temp_new_i64();
3713
3714        if (a->rs1 == 0) {
3715            vec_element_loadi(s, dest, a->rs2, 0, false);
3716        } else {
3717            vec_element_loadx(s, dest, a->rs2, cpu_gpr[a->rs1], vlmax);
3718        }
3719
3720        tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd),
3721                             MAXSZ(s), MAXSZ(s), dest);
3722        tcg_temp_free_i64(dest);
3723        mark_vs_dirty(s);
3724    } else {
3725        static gen_helper_opivx * const fns[4] = {
3726            gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h,
3727            gen_helper_vrgather_vx_w, gen_helper_vrgather_vx_d
3728        };
3729        return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew], s);
3730    }
3731    return true;
3732}
3733
3734/* vrgather.vi vd, vs2, imm, vm # vd[i] = (imm >= VLMAX) ? 0 : vs2[imm] */
3735static bool trans_vrgather_vi(DisasContext *s, arg_rmrr *a)
3736{
3737    if (!vrgather_vx_check(s, a)) {
3738        return false;
3739    }
3740
3741    if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) {
3742        int scale = s->lmul - (s->sew + 3);
3743        int vlmax = s->cfg_ptr->vlen >> -scale;
3744        if (a->rs1 >= vlmax) {
3745            tcg_gen_gvec_dup_imm(MO_64, vreg_ofs(s, a->rd),
3746                                 MAXSZ(s), MAXSZ(s), 0);
3747        } else {
3748            tcg_gen_gvec_dup_mem(s->sew, vreg_ofs(s, a->rd),
3749                                 endian_ofs(s, a->rs2, a->rs1),
3750                                 MAXSZ(s), MAXSZ(s));
3751        }
3752        mark_vs_dirty(s);
3753    } else {
3754        static gen_helper_opivx * const fns[4] = {
3755            gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h,
3756            gen_helper_vrgather_vx_w, gen_helper_vrgather_vx_d
3757        };
3758        return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, fns[s->sew],
3759                           s, IMM_ZX);
3760    }
3761    return true;
3762}
3763
3764/*
3765 * Vector Compress Instruction
3766 *
3767 * The destination vector register group cannot overlap the
3768 * source vector register group or the source mask register.
3769 */
3770static bool vcompress_vm_check(DisasContext *s, arg_r *a)
3771{
3772    return require_rvv(s) &&
3773           vext_check_isa_ill(s) &&
3774           require_align(a->rd, s->lmul) &&
3775           require_align(a->rs2, s->lmul) &&
3776           (a->rd != a->rs2) &&
3777           !is_overlapped(a->rd, 1 << MAX(s->lmul, 0), a->rs1, 1) &&
3778           (s->vstart == 0);
3779}
3780
3781static bool trans_vcompress_vm(DisasContext *s, arg_r *a)
3782{
3783    if (vcompress_vm_check(s, a)) {
3784        uint32_t data = 0;
3785        static gen_helper_gvec_4_ptr * const fns[4] = {
3786            gen_helper_vcompress_vm_b, gen_helper_vcompress_vm_h,
3787            gen_helper_vcompress_vm_w, gen_helper_vcompress_vm_d,
3788        };
3789        TCGLabel *over = gen_new_label();
3790        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3791
3792        data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3793        data = FIELD_DP32(data, VDATA, VTA, s->vta);
3794        tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3795                           vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2),
3796                           cpu_env, s->cfg_ptr->vlen / 8,
3797                           s->cfg_ptr->vlen / 8, data,
3798                           fns[s->sew]);
3799        mark_vs_dirty(s);
3800        gen_set_label(over);
3801        return true;
3802    }
3803    return false;
3804}
3805
3806/*
3807 * Whole Vector Register Move Instructions ignore vtype and vl setting.
3808 * Thus, we don't need to check vill bit. (Section 16.6)
3809 */
3810#define GEN_VMV_WHOLE_TRANS(NAME, LEN)                             \
3811static bool trans_##NAME(DisasContext *s, arg_##NAME * a)               \
3812{                                                                       \
3813    if (require_rvv(s) &&                                               \
3814        QEMU_IS_ALIGNED(a->rd, LEN) &&                                  \
3815        QEMU_IS_ALIGNED(a->rs2, LEN)) {                                 \
3816        uint32_t maxsz = (s->cfg_ptr->vlen >> 3) * LEN;                 \
3817        if (s->vstart == 0) {                                           \
3818            /* EEW = 8 */                                               \
3819            tcg_gen_gvec_mov(MO_8, vreg_ofs(s, a->rd),                  \
3820                             vreg_ofs(s, a->rs2), maxsz, maxsz);        \
3821            mark_vs_dirty(s);                                           \
3822        } else {                                                        \
3823            TCGLabel *over = gen_new_label();                           \
3824            tcg_gen_brcondi_tl(TCG_COND_GEU, cpu_vstart, maxsz, over);  \
3825            tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), \
3826                               cpu_env, maxsz, maxsz, 0, gen_helper_vmvr_v); \
3827            mark_vs_dirty(s);                                           \
3828            gen_set_label(over);                                        \
3829        }                                                               \
3830        return true;                                                    \
3831    }                                                                   \
3832    return false;                                                       \
3833}
3834
3835GEN_VMV_WHOLE_TRANS(vmv1r_v, 1)
3836GEN_VMV_WHOLE_TRANS(vmv2r_v, 2)
3837GEN_VMV_WHOLE_TRANS(vmv4r_v, 4)
3838GEN_VMV_WHOLE_TRANS(vmv8r_v, 8)
3839
3840static bool int_ext_check(DisasContext *s, arg_rmr *a, uint8_t div)
3841{
3842    uint8_t from = (s->sew + 3) - div;
3843    bool ret = require_rvv(s) &&
3844        (from >= 3 && from <= 8) &&
3845        (a->rd != a->rs2) &&
3846        require_align(a->rd, s->lmul) &&
3847        require_align(a->rs2, s->lmul - div) &&
3848        require_vm(a->vm, a->rd) &&
3849        require_noover(a->rd, s->lmul, a->rs2, s->lmul - div);
3850    return ret;
3851}
3852
3853static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq)
3854{
3855    uint32_t data = 0;
3856    gen_helper_gvec_3_ptr *fn;
3857    TCGLabel *over = gen_new_label();
3858    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over);
3859    tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);
3860
3861    static gen_helper_gvec_3_ptr * const fns[6][4] = {
3862        {
3863            NULL, gen_helper_vzext_vf2_h,
3864            gen_helper_vzext_vf2_w, gen_helper_vzext_vf2_d
3865        },
3866        {
3867            NULL, NULL,
3868            gen_helper_vzext_vf4_w, gen_helper_vzext_vf4_d,
3869        },
3870        {
3871            NULL, NULL,
3872            NULL, gen_helper_vzext_vf8_d
3873        },
3874        {
3875            NULL, gen_helper_vsext_vf2_h,
3876            gen_helper_vsext_vf2_w, gen_helper_vsext_vf2_d
3877        },
3878        {
3879            NULL, NULL,
3880            gen_helper_vsext_vf4_w, gen_helper_vsext_vf4_d,
3881        },
3882        {
3883            NULL, NULL,
3884            NULL, gen_helper_vsext_vf8_d
3885        }
3886    };
3887
3888    fn = fns[seq][s->sew];
3889    if (fn == NULL) {
3890        return false;
3891    }
3892
3893    data = FIELD_DP32(data, VDATA, VM, a->vm);
3894    data = FIELD_DP32(data, VDATA, LMUL, s->lmul);
3895    data = FIELD_DP32(data, VDATA, VTA, s->vta);
3896    data = FIELD_DP32(data, VDATA, VMA, s->vma);
3897
3898    tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0),
3899                       vreg_ofs(s, a->rs2), cpu_env,
3900                       s->cfg_ptr->vlen / 8,
3901                       s->cfg_ptr->vlen / 8, data, fn);
3902
3903    mark_vs_dirty(s);
3904    gen_set_label(over);
3905    return true;
3906}
3907
3908/* Vector Integer Extension */
3909#define GEN_INT_EXT_TRANS(NAME, DIV, SEQ)             \
3910static bool trans_##NAME(DisasContext *s, arg_rmr *a) \
3911{                                                     \
3912    if (int_ext_check(s, a, DIV)) {                   \
3913        return int_ext_op(s, a, SEQ);                 \
3914    }                                                 \
3915    return false;                                     \
3916}
3917
3918GEN_INT_EXT_TRANS(vzext_vf2, 1, 0)
3919GEN_INT_EXT_TRANS(vzext_vf4, 2, 1)
3920GEN_INT_EXT_TRANS(vzext_vf8, 3, 2)
3921GEN_INT_EXT_TRANS(vsext_vf2, 1, 3)
3922GEN_INT_EXT_TRANS(vsext_vf4, 2, 4)
3923GEN_INT_EXT_TRANS(vsext_vf8, 3, 5)
3924