1 /*
2 * AArch64 translation
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20
21 #include "exec/exec-all.h"
22 #include "translate.h"
23 #include "translate-a64.h"
24 #include "qemu/log.h"
25 #include "arm_ldst.h"
26 #include "semihosting/semihost.h"
27 #include "cpregs.h"
28
29 static TCGv_i64 cpu_X[32];
30 static TCGv_i64 cpu_pc;
31
32 /* Load/store exclusive handling */
33 static TCGv_i64 cpu_exclusive_high;
34
35 static const char *regnames[] = {
36 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
37 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
38 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
39 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
40 };
41
42 enum a64_shift_type {
43 A64_SHIFT_TYPE_LSL = 0,
44 A64_SHIFT_TYPE_LSR = 1,
45 A64_SHIFT_TYPE_ASR = 2,
46 A64_SHIFT_TYPE_ROR = 3
47 };
48
49 /*
50 * Helpers for extracting complex instruction fields
51 */
52
53 /*
54 * For load/store with an unsigned 12 bit immediate scaled by the element
55 * size. The input has the immediate field in bits [14:3] and the element
56 * size in [2:0].
57 */
uimm_scaled(DisasContext * s,int x)58 static int uimm_scaled(DisasContext *s, int x)
59 {
60 unsigned imm = x >> 3;
61 unsigned scale = extract32(x, 0, 3);
62 return imm << scale;
63 }
64
65 /* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
scale_by_log2_tag_granule(DisasContext * s,int x)66 static int scale_by_log2_tag_granule(DisasContext *s, int x)
67 {
68 return x << LOG2_TAG_GRANULE;
69 }
70
71 /*
72 * Include the generated decoders.
73 */
74
75 #include "decode-sme-fa64.c.inc"
76 #include "decode-a64.c.inc"
77
78 /* Table based decoder typedefs - used when the relevant bits for decode
79 * are too awkwardly scattered across the instruction (eg SIMD).
80 */
81 typedef void AArch64DecodeFn(DisasContext *s, uint32_t insn);
82
83 typedef struct AArch64DecodeTable {
84 uint32_t pattern;
85 uint32_t mask;
86 AArch64DecodeFn *disas_fn;
87 } AArch64DecodeTable;
88
89 /* initialize TCG globals. */
a64_translate_init(void)90 void a64_translate_init(void)
91 {
92 int i;
93
94 cpu_pc = tcg_global_mem_new_i64(tcg_env,
95 offsetof(CPUARMState, pc),
96 "pc");
97 for (i = 0; i < 32; i++) {
98 cpu_X[i] = tcg_global_mem_new_i64(tcg_env,
99 offsetof(CPUARMState, xregs[i]),
100 regnames[i]);
101 }
102
103 cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env,
104 offsetof(CPUARMState, exclusive_high), "exclusive_high");
105 }
106
107 /*
108 * Return the core mmu_idx to use for A64 load/store insns which
109 * have a "unprivileged load/store" variant. Those insns access
110 * EL0 if executed from an EL which has control over EL0 (usually
111 * EL1) but behave like normal loads and stores if executed from
112 * elsewhere (eg EL3).
113 *
114 * @unpriv : true for the unprivileged encoding; false for the
115 * normal encoding (in which case we will return the same
116 * thing as get_mem_index().
117 */
get_a64_user_mem_index(DisasContext * s,bool unpriv)118 static int get_a64_user_mem_index(DisasContext *s, bool unpriv)
119 {
120 /*
121 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
122 * which is the usual mmu_idx for this cpu state.
123 */
124 ARMMMUIdx useridx = s->mmu_idx;
125
126 if (unpriv && s->unpriv) {
127 /*
128 * We have pre-computed the condition for AccType_UNPRIV.
129 * Therefore we should never get here with a mmu_idx for
130 * which we do not know the corresponding user mmu_idx.
131 */
132 switch (useridx) {
133 case ARMMMUIdx_E10_1:
134 case ARMMMUIdx_E10_1_PAN:
135 useridx = ARMMMUIdx_E10_0;
136 break;
137 case ARMMMUIdx_E20_2:
138 case ARMMMUIdx_E20_2_PAN:
139 useridx = ARMMMUIdx_E20_0;
140 break;
141 default:
142 g_assert_not_reached();
143 }
144 }
145 return arm_to_core_mmu_idx(useridx);
146 }
147
set_btype_raw(int val)148 static void set_btype_raw(int val)
149 {
150 tcg_gen_st_i32(tcg_constant_i32(val), tcg_env,
151 offsetof(CPUARMState, btype));
152 }
153
set_btype(DisasContext * s,int val)154 static void set_btype(DisasContext *s, int val)
155 {
156 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
157 tcg_debug_assert(val >= 1 && val <= 3);
158 set_btype_raw(val);
159 s->btype = -1;
160 }
161
reset_btype(DisasContext * s)162 static void reset_btype(DisasContext *s)
163 {
164 if (s->btype != 0) {
165 set_btype_raw(0);
166 s->btype = 0;
167 }
168 }
169
gen_pc_plus_diff(DisasContext * s,TCGv_i64 dest,target_long diff)170 static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff)
171 {
172 assert(s->pc_save != -1);
173 if (tb_cflags(s->base.tb) & CF_PCREL) {
174 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff);
175 } else {
176 tcg_gen_movi_i64(dest, s->pc_curr + diff);
177 }
178 }
179
gen_a64_update_pc(DisasContext * s,target_long diff)180 void gen_a64_update_pc(DisasContext *s, target_long diff)
181 {
182 gen_pc_plus_diff(s, cpu_pc, diff);
183 s->pc_save = s->pc_curr + diff;
184 }
185
186 /*
187 * Handle Top Byte Ignore (TBI) bits.
188 *
189 * If address tagging is enabled via the TCR TBI bits:
190 * + for EL2 and EL3 there is only one TBI bit, and if it is set
191 * then the address is zero-extended, clearing bits [63:56]
192 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
193 * and TBI1 controls addresses with bit 55 == 1.
194 * If the appropriate TBI bit is set for the address then
195 * the address is sign-extended from bit 55 into bits [63:56]
196 *
197 * Here We have concatenated TBI{1,0} into tbi.
198 */
gen_top_byte_ignore(DisasContext * s,TCGv_i64 dst,TCGv_i64 src,int tbi)199 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
200 TCGv_i64 src, int tbi)
201 {
202 if (tbi == 0) {
203 /* Load unmodified address */
204 tcg_gen_mov_i64(dst, src);
205 } else if (!regime_has_2_ranges(s->mmu_idx)) {
206 /* Force tag byte to all zero */
207 tcg_gen_extract_i64(dst, src, 0, 56);
208 } else {
209 /* Sign-extend from bit 55. */
210 tcg_gen_sextract_i64(dst, src, 0, 56);
211
212 switch (tbi) {
213 case 1:
214 /* tbi0 but !tbi1: only use the extension if positive */
215 tcg_gen_and_i64(dst, dst, src);
216 break;
217 case 2:
218 /* !tbi0 but tbi1: only use the extension if negative */
219 tcg_gen_or_i64(dst, dst, src);
220 break;
221 case 3:
222 /* tbi0 and tbi1: always use the extension */
223 break;
224 default:
225 g_assert_not_reached();
226 }
227 }
228 }
229
gen_a64_set_pc(DisasContext * s,TCGv_i64 src)230 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
231 {
232 /*
233 * If address tagging is enabled for instructions via the TCR TBI bits,
234 * then loading an address into the PC will clear out any tag.
235 */
236 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
237 s->pc_save = -1;
238 }
239
240 /*
241 * Handle MTE and/or TBI.
242 *
243 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
244 * for the tag to be present in the FAR_ELx register. But for user-only
245 * mode we do not have a TLB with which to implement this, so we must
246 * remove the top byte now.
247 *
248 * Always return a fresh temporary that we can increment independently
249 * of the write-back address.
250 */
251
clean_data_tbi(DisasContext * s,TCGv_i64 addr)252 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
253 {
254 TCGv_i64 clean = tcg_temp_new_i64();
255 #ifdef CONFIG_USER_ONLY
256 gen_top_byte_ignore(s, clean, addr, s->tbid);
257 #else
258 tcg_gen_mov_i64(clean, addr);
259 #endif
260 return clean;
261 }
262
263 /* Insert a zero tag into src, with the result at dst. */
gen_address_with_allocation_tag0(TCGv_i64 dst,TCGv_i64 src)264 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
265 {
266 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
267 }
268
gen_probe_access(DisasContext * s,TCGv_i64 ptr,MMUAccessType acc,int log2_size)269 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
270 MMUAccessType acc, int log2_size)
271 {
272 gen_helper_probe_access(tcg_env, ptr,
273 tcg_constant_i32(acc),
274 tcg_constant_i32(get_mem_index(s)),
275 tcg_constant_i32(1 << log2_size));
276 }
277
278 /*
279 * For MTE, check a single logical or atomic access. This probes a single
280 * address, the exact one specified. The size and alignment of the access
281 * is not relevant to MTE, per se, but watchpoints do require the size,
282 * and we want to recognize those before making any other changes to state.
283 */
gen_mte_check1_mmuidx(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop,bool is_unpriv,int core_idx)284 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
285 bool is_write, bool tag_checked,
286 MemOp memop, bool is_unpriv,
287 int core_idx)
288 {
289 if (tag_checked && s->mte_active[is_unpriv]) {
290 TCGv_i64 ret;
291 int desc = 0;
292
293 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
294 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
295 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
296 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
297 desc = FIELD_DP32(desc, MTEDESC, ALIGN, get_alignment_bits(memop));
298 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1);
299
300 ret = tcg_temp_new_i64();
301 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
302
303 return ret;
304 }
305 return clean_data_tbi(s, addr);
306 }
307
gen_mte_check1(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop)308 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
309 bool tag_checked, MemOp memop)
310 {
311 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, memop,
312 false, get_mem_index(s));
313 }
314
315 /*
316 * For MTE, check multiple logical sequential accesses.
317 */
gen_mte_checkN(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,int total_size,MemOp single_mop)318 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
319 bool tag_checked, int total_size, MemOp single_mop)
320 {
321 if (tag_checked && s->mte_active[0]) {
322 TCGv_i64 ret;
323 int desc = 0;
324
325 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
326 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
327 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
328 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
329 desc = FIELD_DP32(desc, MTEDESC, ALIGN, get_alignment_bits(single_mop));
330 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
331
332 ret = tcg_temp_new_i64();
333 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
334
335 return ret;
336 }
337 return clean_data_tbi(s, addr);
338 }
339
340 /*
341 * Generate the special alignment check that applies to AccType_ATOMIC
342 * and AccType_ORDERED insns under FEAT_LSE2: the access need not be
343 * naturally aligned, but it must not cross a 16-byte boundary.
344 * See AArch64.CheckAlignment().
345 */
check_lse2_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)346 static void check_lse2_align(DisasContext *s, int rn, int imm,
347 bool is_write, MemOp mop)
348 {
349 TCGv_i32 tmp;
350 TCGv_i64 addr;
351 TCGLabel *over_label;
352 MMUAccessType type;
353 int mmu_idx;
354
355 tmp = tcg_temp_new_i32();
356 tcg_gen_extrl_i64_i32(tmp, cpu_reg_sp(s, rn));
357 tcg_gen_addi_i32(tmp, tmp, imm & 15);
358 tcg_gen_andi_i32(tmp, tmp, 15);
359 tcg_gen_addi_i32(tmp, tmp, memop_size(mop));
360
361 over_label = gen_new_label();
362 tcg_gen_brcondi_i32(TCG_COND_LEU, tmp, 16, over_label);
363
364 addr = tcg_temp_new_i64();
365 tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm);
366
367 type = is_write ? MMU_DATA_STORE : MMU_DATA_LOAD,
368 mmu_idx = get_mem_index(s);
369 gen_helper_unaligned_access(tcg_env, addr, tcg_constant_i32(type),
370 tcg_constant_i32(mmu_idx));
371
372 gen_set_label(over_label);
373
374 }
375
376 /* Handle the alignment check for AccType_ATOMIC instructions. */
check_atomic_align(DisasContext * s,int rn,MemOp mop)377 static MemOp check_atomic_align(DisasContext *s, int rn, MemOp mop)
378 {
379 MemOp size = mop & MO_SIZE;
380
381 if (size == MO_8) {
382 return mop;
383 }
384
385 /*
386 * If size == MO_128, this is a LDXP, and the operation is single-copy
387 * atomic for each doubleword, not the entire quadword; it still must
388 * be quadword aligned.
389 */
390 if (size == MO_128) {
391 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
392 MO_ATOM_IFALIGN_PAIR);
393 }
394 if (dc_isar_feature(aa64_lse2, s)) {
395 check_lse2_align(s, rn, 0, true, mop);
396 } else {
397 mop |= MO_ALIGN;
398 }
399 return finalize_memop(s, mop);
400 }
401
402 /* Handle the alignment check for AccType_ORDERED instructions. */
check_ordered_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)403 static MemOp check_ordered_align(DisasContext *s, int rn, int imm,
404 bool is_write, MemOp mop)
405 {
406 MemOp size = mop & MO_SIZE;
407
408 if (size == MO_8) {
409 return mop;
410 }
411 if (size == MO_128) {
412 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
413 MO_ATOM_IFALIGN_PAIR);
414 }
415 if (!dc_isar_feature(aa64_lse2, s)) {
416 mop |= MO_ALIGN;
417 } else if (!s->naa) {
418 check_lse2_align(s, rn, imm, is_write, mop);
419 }
420 return finalize_memop(s, mop);
421 }
422
423 typedef struct DisasCompare64 {
424 TCGCond cond;
425 TCGv_i64 value;
426 } DisasCompare64;
427
a64_test_cc(DisasCompare64 * c64,int cc)428 static void a64_test_cc(DisasCompare64 *c64, int cc)
429 {
430 DisasCompare c32;
431
432 arm_test_cc(&c32, cc);
433
434 /*
435 * Sign-extend the 32-bit value so that the GE/LT comparisons work
436 * properly. The NE/EQ comparisons are also fine with this choice.
437 */
438 c64->cond = c32.cond;
439 c64->value = tcg_temp_new_i64();
440 tcg_gen_ext_i32_i64(c64->value, c32.value);
441 }
442
gen_rebuild_hflags(DisasContext * s)443 static void gen_rebuild_hflags(DisasContext *s)
444 {
445 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el));
446 }
447
gen_exception_internal(int excp)448 static void gen_exception_internal(int excp)
449 {
450 assert(excp_is_internal(excp));
451 gen_helper_exception_internal(tcg_env, tcg_constant_i32(excp));
452 }
453
gen_exception_internal_insn(DisasContext * s,int excp)454 static void gen_exception_internal_insn(DisasContext *s, int excp)
455 {
456 gen_a64_update_pc(s, 0);
457 gen_exception_internal(excp);
458 s->base.is_jmp = DISAS_NORETURN;
459 }
460
gen_exception_bkpt_insn(DisasContext * s,uint32_t syndrome)461 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
462 {
463 gen_a64_update_pc(s, 0);
464 gen_helper_exception_bkpt_insn(tcg_env, tcg_constant_i32(syndrome));
465 s->base.is_jmp = DISAS_NORETURN;
466 }
467
gen_step_complete_exception(DisasContext * s)468 static void gen_step_complete_exception(DisasContext *s)
469 {
470 /* We just completed step of an insn. Move from Active-not-pending
471 * to Active-pending, and then also take the swstep exception.
472 * This corresponds to making the (IMPDEF) choice to prioritize
473 * swstep exceptions over asynchronous exceptions taken to an exception
474 * level where debug is disabled. This choice has the advantage that
475 * we do not need to maintain internal state corresponding to the
476 * ISV/EX syndrome bits between completion of the step and generation
477 * of the exception, and our syndrome information is always correct.
478 */
479 gen_ss_advance(s);
480 gen_swstep_exception(s, 1, s->is_ldex);
481 s->base.is_jmp = DISAS_NORETURN;
482 }
483
use_goto_tb(DisasContext * s,uint64_t dest)484 static inline bool use_goto_tb(DisasContext *s, uint64_t dest)
485 {
486 if (s->ss_active) {
487 return false;
488 }
489 return translator_use_goto_tb(&s->base, dest);
490 }
491
gen_goto_tb(DisasContext * s,int n,int64_t diff)492 static void gen_goto_tb(DisasContext *s, int n, int64_t diff)
493 {
494 if (use_goto_tb(s, s->pc_curr + diff)) {
495 /*
496 * For pcrel, the pc must always be up-to-date on entry to
497 * the linked TB, so that it can use simple additions for all
498 * further adjustments. For !pcrel, the linked TB is compiled
499 * to know its full virtual address, so we can delay the
500 * update to pc to the unlinked path. A long chain of links
501 * can thus avoid many updates to the PC.
502 */
503 if (tb_cflags(s->base.tb) & CF_PCREL) {
504 gen_a64_update_pc(s, diff);
505 tcg_gen_goto_tb(n);
506 } else {
507 tcg_gen_goto_tb(n);
508 gen_a64_update_pc(s, diff);
509 }
510 tcg_gen_exit_tb(s->base.tb, n);
511 s->base.is_jmp = DISAS_NORETURN;
512 } else {
513 gen_a64_update_pc(s, diff);
514 if (s->ss_active) {
515 gen_step_complete_exception(s);
516 } else {
517 tcg_gen_lookup_and_goto_ptr();
518 s->base.is_jmp = DISAS_NORETURN;
519 }
520 }
521 }
522
523 /*
524 * Register access functions
525 *
526 * These functions are used for directly accessing a register in where
527 * changes to the final register value are likely to be made. If you
528 * need to use a register for temporary calculation (e.g. index type
529 * operations) use the read_* form.
530 *
531 * B1.2.1 Register mappings
532 *
533 * In instruction register encoding 31 can refer to ZR (zero register) or
534 * the SP (stack pointer) depending on context. In QEMU's case we map SP
535 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
536 * This is the point of the _sp forms.
537 */
cpu_reg(DisasContext * s,int reg)538 TCGv_i64 cpu_reg(DisasContext *s, int reg)
539 {
540 if (reg == 31) {
541 TCGv_i64 t = tcg_temp_new_i64();
542 tcg_gen_movi_i64(t, 0);
543 return t;
544 } else {
545 return cpu_X[reg];
546 }
547 }
548
549 /* register access for when 31 == SP */
cpu_reg_sp(DisasContext * s,int reg)550 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
551 {
552 return cpu_X[reg];
553 }
554
555 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
556 * representing the register contents. This TCGv is an auto-freed
557 * temporary so it need not be explicitly freed, and may be modified.
558 */
read_cpu_reg(DisasContext * s,int reg,int sf)559 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
560 {
561 TCGv_i64 v = tcg_temp_new_i64();
562 if (reg != 31) {
563 if (sf) {
564 tcg_gen_mov_i64(v, cpu_X[reg]);
565 } else {
566 tcg_gen_ext32u_i64(v, cpu_X[reg]);
567 }
568 } else {
569 tcg_gen_movi_i64(v, 0);
570 }
571 return v;
572 }
573
read_cpu_reg_sp(DisasContext * s,int reg,int sf)574 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
575 {
576 TCGv_i64 v = tcg_temp_new_i64();
577 if (sf) {
578 tcg_gen_mov_i64(v, cpu_X[reg]);
579 } else {
580 tcg_gen_ext32u_i64(v, cpu_X[reg]);
581 }
582 return v;
583 }
584
585 /* Return the offset into CPUARMState of a slice (from
586 * the least significant end) of FP register Qn (ie
587 * Dn, Sn, Hn or Bn).
588 * (Note that this is not the same mapping as for A32; see cpu.h)
589 */
fp_reg_offset(DisasContext * s,int regno,MemOp size)590 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
591 {
592 return vec_reg_offset(s, regno, 0, size);
593 }
594
595 /* Offset of the high half of the 128 bit vector Qn */
fp_reg_hi_offset(DisasContext * s,int regno)596 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
597 {
598 return vec_reg_offset(s, regno, 1, MO_64);
599 }
600
601 /* Convenience accessors for reading and writing single and double
602 * FP registers. Writing clears the upper parts of the associated
603 * 128 bit vector register, as required by the architecture.
604 * Note that unlike the GP register accessors, the values returned
605 * by the read functions must be manually freed.
606 */
read_fp_dreg(DisasContext * s,int reg)607 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
608 {
609 TCGv_i64 v = tcg_temp_new_i64();
610
611 tcg_gen_ld_i64(v, tcg_env, fp_reg_offset(s, reg, MO_64));
612 return v;
613 }
614
read_fp_sreg(DisasContext * s,int reg)615 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
616 {
617 TCGv_i32 v = tcg_temp_new_i32();
618
619 tcg_gen_ld_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
620 return v;
621 }
622
read_fp_hreg(DisasContext * s,int reg)623 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
624 {
625 TCGv_i32 v = tcg_temp_new_i32();
626
627 tcg_gen_ld16u_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
628 return v;
629 }
630
631 /* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
632 * If SVE is not enabled, then there are only 128 bits in the vector.
633 */
clear_vec_high(DisasContext * s,bool is_q,int rd)634 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
635 {
636 unsigned ofs = fp_reg_offset(s, rd, MO_64);
637 unsigned vsz = vec_full_reg_size(s);
638
639 /* Nop move, with side effect of clearing the tail. */
640 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
641 }
642
write_fp_dreg(DisasContext * s,int reg,TCGv_i64 v)643 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
644 {
645 unsigned ofs = fp_reg_offset(s, reg, MO_64);
646
647 tcg_gen_st_i64(v, tcg_env, ofs);
648 clear_vec_high(s, false, reg);
649 }
650
write_fp_sreg(DisasContext * s,int reg,TCGv_i32 v)651 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
652 {
653 TCGv_i64 tmp = tcg_temp_new_i64();
654
655 tcg_gen_extu_i32_i64(tmp, v);
656 write_fp_dreg(s, reg, tmp);
657 }
658
659 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn2(DisasContext * s,bool is_q,int rd,int rn,GVecGen2Fn * gvec_fn,int vece)660 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
661 GVecGen2Fn *gvec_fn, int vece)
662 {
663 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
664 is_q ? 16 : 8, vec_full_reg_size(s));
665 }
666
667 /* Expand a 2-operand + immediate AdvSIMD vector operation using
668 * an expander function.
669 */
gen_gvec_fn2i(DisasContext * s,bool is_q,int rd,int rn,int64_t imm,GVecGen2iFn * gvec_fn,int vece)670 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
671 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
672 {
673 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
674 imm, is_q ? 16 : 8, vec_full_reg_size(s));
675 }
676
677 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn3(DisasContext * s,bool is_q,int rd,int rn,int rm,GVecGen3Fn * gvec_fn,int vece)678 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
679 GVecGen3Fn *gvec_fn, int vece)
680 {
681 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
682 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
683 }
684
685 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn4(DisasContext * s,bool is_q,int rd,int rn,int rm,int rx,GVecGen4Fn * gvec_fn,int vece)686 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
687 int rx, GVecGen4Fn *gvec_fn, int vece)
688 {
689 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
690 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
691 is_q ? 16 : 8, vec_full_reg_size(s));
692 }
693
694 /* Expand a 2-operand operation using an out-of-line helper. */
gen_gvec_op2_ool(DisasContext * s,bool is_q,int rd,int rn,int data,gen_helper_gvec_2 * fn)695 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
696 int rn, int data, gen_helper_gvec_2 *fn)
697 {
698 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
699 vec_full_reg_offset(s, rn),
700 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
701 }
702
703 /* Expand a 3-operand operation using an out-of-line helper. */
gen_gvec_op3_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int data,gen_helper_gvec_3 * fn)704 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
705 int rn, int rm, int data, gen_helper_gvec_3 *fn)
706 {
707 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
708 vec_full_reg_offset(s, rn),
709 vec_full_reg_offset(s, rm),
710 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
711 }
712
713 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
714 * an out-of-line helper.
715 */
gen_gvec_op3_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,bool is_fp16,int data,gen_helper_gvec_3_ptr * fn)716 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
717 int rm, bool is_fp16, int data,
718 gen_helper_gvec_3_ptr *fn)
719 {
720 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
721 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
722 vec_full_reg_offset(s, rn),
723 vec_full_reg_offset(s, rm), fpst,
724 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
725 }
726
727 /* Expand a 3-operand + qc + operation using an out-of-line helper. */
gen_gvec_op3_qc(DisasContext * s,bool is_q,int rd,int rn,int rm,gen_helper_gvec_3_ptr * fn)728 static void gen_gvec_op3_qc(DisasContext *s, bool is_q, int rd, int rn,
729 int rm, gen_helper_gvec_3_ptr *fn)
730 {
731 TCGv_ptr qc_ptr = tcg_temp_new_ptr();
732
733 tcg_gen_addi_ptr(qc_ptr, tcg_env, offsetof(CPUARMState, vfp.qc));
734 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
735 vec_full_reg_offset(s, rn),
736 vec_full_reg_offset(s, rm), qc_ptr,
737 is_q ? 16 : 8, vec_full_reg_size(s), 0, fn);
738 }
739
740 /* Expand a 4-operand operation using an out-of-line helper. */
gen_gvec_op4_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,int data,gen_helper_gvec_4 * fn)741 static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
742 int rm, int ra, int data, gen_helper_gvec_4 *fn)
743 {
744 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
745 vec_full_reg_offset(s, rn),
746 vec_full_reg_offset(s, rm),
747 vec_full_reg_offset(s, ra),
748 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
749 }
750
751 /*
752 * Expand a 4-operand + fpstatus pointer + simd data value operation using
753 * an out-of-line helper.
754 */
gen_gvec_op4_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,bool is_fp16,int data,gen_helper_gvec_4_ptr * fn)755 static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
756 int rm, int ra, bool is_fp16, int data,
757 gen_helper_gvec_4_ptr *fn)
758 {
759 TCGv_ptr fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
760 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
761 vec_full_reg_offset(s, rn),
762 vec_full_reg_offset(s, rm),
763 vec_full_reg_offset(s, ra), fpst,
764 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
765 }
766
767 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
768 * than the 32 bit equivalent.
769 */
gen_set_NZ64(TCGv_i64 result)770 static inline void gen_set_NZ64(TCGv_i64 result)
771 {
772 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
773 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
774 }
775
776 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
gen_logic_CC(int sf,TCGv_i64 result)777 static inline void gen_logic_CC(int sf, TCGv_i64 result)
778 {
779 if (sf) {
780 gen_set_NZ64(result);
781 } else {
782 tcg_gen_extrl_i64_i32(cpu_ZF, result);
783 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
784 }
785 tcg_gen_movi_i32(cpu_CF, 0);
786 tcg_gen_movi_i32(cpu_VF, 0);
787 }
788
789 /* dest = T0 + T1; compute C, N, V and Z flags */
gen_add64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)790 static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
791 {
792 TCGv_i64 result, flag, tmp;
793 result = tcg_temp_new_i64();
794 flag = tcg_temp_new_i64();
795 tmp = tcg_temp_new_i64();
796
797 tcg_gen_movi_i64(tmp, 0);
798 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
799
800 tcg_gen_extrl_i64_i32(cpu_CF, flag);
801
802 gen_set_NZ64(result);
803
804 tcg_gen_xor_i64(flag, result, t0);
805 tcg_gen_xor_i64(tmp, t0, t1);
806 tcg_gen_andc_i64(flag, flag, tmp);
807 tcg_gen_extrh_i64_i32(cpu_VF, flag);
808
809 tcg_gen_mov_i64(dest, result);
810 }
811
gen_add32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)812 static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
813 {
814 TCGv_i32 t0_32 = tcg_temp_new_i32();
815 TCGv_i32 t1_32 = tcg_temp_new_i32();
816 TCGv_i32 tmp = tcg_temp_new_i32();
817
818 tcg_gen_movi_i32(tmp, 0);
819 tcg_gen_extrl_i64_i32(t0_32, t0);
820 tcg_gen_extrl_i64_i32(t1_32, t1);
821 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
822 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
823 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
824 tcg_gen_xor_i32(tmp, t0_32, t1_32);
825 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
826 tcg_gen_extu_i32_i64(dest, cpu_NF);
827 }
828
gen_add_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)829 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
830 {
831 if (sf) {
832 gen_add64_CC(dest, t0, t1);
833 } else {
834 gen_add32_CC(dest, t0, t1);
835 }
836 }
837
838 /* dest = T0 - T1; compute C, N, V and Z flags */
gen_sub64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)839 static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
840 {
841 /* 64 bit arithmetic */
842 TCGv_i64 result, flag, tmp;
843
844 result = tcg_temp_new_i64();
845 flag = tcg_temp_new_i64();
846 tcg_gen_sub_i64(result, t0, t1);
847
848 gen_set_NZ64(result);
849
850 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
851 tcg_gen_extrl_i64_i32(cpu_CF, flag);
852
853 tcg_gen_xor_i64(flag, result, t0);
854 tmp = tcg_temp_new_i64();
855 tcg_gen_xor_i64(tmp, t0, t1);
856 tcg_gen_and_i64(flag, flag, tmp);
857 tcg_gen_extrh_i64_i32(cpu_VF, flag);
858 tcg_gen_mov_i64(dest, result);
859 }
860
gen_sub32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)861 static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
862 {
863 /* 32 bit arithmetic */
864 TCGv_i32 t0_32 = tcg_temp_new_i32();
865 TCGv_i32 t1_32 = tcg_temp_new_i32();
866 TCGv_i32 tmp;
867
868 tcg_gen_extrl_i64_i32(t0_32, t0);
869 tcg_gen_extrl_i64_i32(t1_32, t1);
870 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
871 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
872 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
873 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
874 tmp = tcg_temp_new_i32();
875 tcg_gen_xor_i32(tmp, t0_32, t1_32);
876 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
877 tcg_gen_extu_i32_i64(dest, cpu_NF);
878 }
879
gen_sub_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)880 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
881 {
882 if (sf) {
883 gen_sub64_CC(dest, t0, t1);
884 } else {
885 gen_sub32_CC(dest, t0, t1);
886 }
887 }
888
889 /* dest = T0 + T1 + CF; do not compute flags. */
gen_adc(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)890 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
891 {
892 TCGv_i64 flag = tcg_temp_new_i64();
893 tcg_gen_extu_i32_i64(flag, cpu_CF);
894 tcg_gen_add_i64(dest, t0, t1);
895 tcg_gen_add_i64(dest, dest, flag);
896
897 if (!sf) {
898 tcg_gen_ext32u_i64(dest, dest);
899 }
900 }
901
902 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
gen_adc_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)903 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
904 {
905 if (sf) {
906 TCGv_i64 result = tcg_temp_new_i64();
907 TCGv_i64 cf_64 = tcg_temp_new_i64();
908 TCGv_i64 vf_64 = tcg_temp_new_i64();
909 TCGv_i64 tmp = tcg_temp_new_i64();
910 TCGv_i64 zero = tcg_constant_i64(0);
911
912 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
913 tcg_gen_add2_i64(result, cf_64, t0, zero, cf_64, zero);
914 tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, zero);
915 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
916 gen_set_NZ64(result);
917
918 tcg_gen_xor_i64(vf_64, result, t0);
919 tcg_gen_xor_i64(tmp, t0, t1);
920 tcg_gen_andc_i64(vf_64, vf_64, tmp);
921 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
922
923 tcg_gen_mov_i64(dest, result);
924 } else {
925 TCGv_i32 t0_32 = tcg_temp_new_i32();
926 TCGv_i32 t1_32 = tcg_temp_new_i32();
927 TCGv_i32 tmp = tcg_temp_new_i32();
928 TCGv_i32 zero = tcg_constant_i32(0);
929
930 tcg_gen_extrl_i64_i32(t0_32, t0);
931 tcg_gen_extrl_i64_i32(t1_32, t1);
932 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, zero, cpu_CF, zero);
933 tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, zero);
934
935 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
936 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
937 tcg_gen_xor_i32(tmp, t0_32, t1_32);
938 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
939 tcg_gen_extu_i32_i64(dest, cpu_NF);
940 }
941 }
942
943 /*
944 * Load/Store generators
945 */
946
947 /*
948 * Store from GPR register to memory.
949 */
do_gpr_st_memidx(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)950 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
951 TCGv_i64 tcg_addr, MemOp memop, int memidx,
952 bool iss_valid,
953 unsigned int iss_srt,
954 bool iss_sf, bool iss_ar)
955 {
956 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop);
957
958 if (iss_valid) {
959 uint32_t syn;
960
961 syn = syn_data_abort_with_iss(0,
962 (memop & MO_SIZE),
963 false,
964 iss_srt,
965 iss_sf,
966 iss_ar,
967 0, 0, 0, 0, 0, false);
968 disas_set_insn_syndrome(s, syn);
969 }
970 }
971
do_gpr_st(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)972 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
973 TCGv_i64 tcg_addr, MemOp memop,
974 bool iss_valid,
975 unsigned int iss_srt,
976 bool iss_sf, bool iss_ar)
977 {
978 do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
979 iss_valid, iss_srt, iss_sf, iss_ar);
980 }
981
982 /*
983 * Load from memory to GPR register
984 */
do_gpr_ld_memidx(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)985 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
986 MemOp memop, bool extend, int memidx,
987 bool iss_valid, unsigned int iss_srt,
988 bool iss_sf, bool iss_ar)
989 {
990 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
991
992 if (extend && (memop & MO_SIGN)) {
993 g_assert((memop & MO_SIZE) <= MO_32);
994 tcg_gen_ext32u_i64(dest, dest);
995 }
996
997 if (iss_valid) {
998 uint32_t syn;
999
1000 syn = syn_data_abort_with_iss(0,
1001 (memop & MO_SIZE),
1002 (memop & MO_SIGN) != 0,
1003 iss_srt,
1004 iss_sf,
1005 iss_ar,
1006 0, 0, 0, 0, 0, false);
1007 disas_set_insn_syndrome(s, syn);
1008 }
1009 }
1010
do_gpr_ld(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1011 static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1012 MemOp memop, bool extend,
1013 bool iss_valid, unsigned int iss_srt,
1014 bool iss_sf, bool iss_ar)
1015 {
1016 do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s),
1017 iss_valid, iss_srt, iss_sf, iss_ar);
1018 }
1019
1020 /*
1021 * Store from FP register to memory
1022 */
do_fp_st(DisasContext * s,int srcidx,TCGv_i64 tcg_addr,MemOp mop)1023 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, MemOp mop)
1024 {
1025 /* This writes the bottom N bits of a 128 bit wide vector to memory */
1026 TCGv_i64 tmplo = tcg_temp_new_i64();
1027
1028 tcg_gen_ld_i64(tmplo, tcg_env, fp_reg_offset(s, srcidx, MO_64));
1029
1030 if ((mop & MO_SIZE) < MO_128) {
1031 tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1032 } else {
1033 TCGv_i64 tmphi = tcg_temp_new_i64();
1034 TCGv_i128 t16 = tcg_temp_new_i128();
1035
1036 tcg_gen_ld_i64(tmphi, tcg_env, fp_reg_hi_offset(s, srcidx));
1037 tcg_gen_concat_i64_i128(t16, tmplo, tmphi);
1038
1039 tcg_gen_qemu_st_i128(t16, tcg_addr, get_mem_index(s), mop);
1040 }
1041 }
1042
1043 /*
1044 * Load from memory to FP register
1045 */
do_fp_ld(DisasContext * s,int destidx,TCGv_i64 tcg_addr,MemOp mop)1046 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, MemOp mop)
1047 {
1048 /* This always zero-extends and writes to a full 128 bit wide vector */
1049 TCGv_i64 tmplo = tcg_temp_new_i64();
1050 TCGv_i64 tmphi = NULL;
1051
1052 if ((mop & MO_SIZE) < MO_128) {
1053 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1054 } else {
1055 TCGv_i128 t16 = tcg_temp_new_i128();
1056
1057 tcg_gen_qemu_ld_i128(t16, tcg_addr, get_mem_index(s), mop);
1058
1059 tmphi = tcg_temp_new_i64();
1060 tcg_gen_extr_i128_i64(tmplo, tmphi, t16);
1061 }
1062
1063 tcg_gen_st_i64(tmplo, tcg_env, fp_reg_offset(s, destidx, MO_64));
1064
1065 if (tmphi) {
1066 tcg_gen_st_i64(tmphi, tcg_env, fp_reg_hi_offset(s, destidx));
1067 }
1068 clear_vec_high(s, tmphi != NULL, destidx);
1069 }
1070
1071 /*
1072 * Vector load/store helpers.
1073 *
1074 * The principal difference between this and a FP load is that we don't
1075 * zero extend as we are filling a partial chunk of the vector register.
1076 * These functions don't support 128 bit loads/stores, which would be
1077 * normal load/store operations.
1078 *
1079 * The _i32 versions are useful when operating on 32 bit quantities
1080 * (eg for floating point single or using Neon helper functions).
1081 */
1082
1083 /* Get value of an element within a vector register */
read_vec_element(DisasContext * s,TCGv_i64 tcg_dest,int srcidx,int element,MemOp memop)1084 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1085 int element, MemOp memop)
1086 {
1087 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1088 switch ((unsigned)memop) {
1089 case MO_8:
1090 tcg_gen_ld8u_i64(tcg_dest, tcg_env, vect_off);
1091 break;
1092 case MO_16:
1093 tcg_gen_ld16u_i64(tcg_dest, tcg_env, vect_off);
1094 break;
1095 case MO_32:
1096 tcg_gen_ld32u_i64(tcg_dest, tcg_env, vect_off);
1097 break;
1098 case MO_8|MO_SIGN:
1099 tcg_gen_ld8s_i64(tcg_dest, tcg_env, vect_off);
1100 break;
1101 case MO_16|MO_SIGN:
1102 tcg_gen_ld16s_i64(tcg_dest, tcg_env, vect_off);
1103 break;
1104 case MO_32|MO_SIGN:
1105 tcg_gen_ld32s_i64(tcg_dest, tcg_env, vect_off);
1106 break;
1107 case MO_64:
1108 case MO_64|MO_SIGN:
1109 tcg_gen_ld_i64(tcg_dest, tcg_env, vect_off);
1110 break;
1111 default:
1112 g_assert_not_reached();
1113 }
1114 }
1115
read_vec_element_i32(DisasContext * s,TCGv_i32 tcg_dest,int srcidx,int element,MemOp memop)1116 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1117 int element, MemOp memop)
1118 {
1119 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1120 switch (memop) {
1121 case MO_8:
1122 tcg_gen_ld8u_i32(tcg_dest, tcg_env, vect_off);
1123 break;
1124 case MO_16:
1125 tcg_gen_ld16u_i32(tcg_dest, tcg_env, vect_off);
1126 break;
1127 case MO_8|MO_SIGN:
1128 tcg_gen_ld8s_i32(tcg_dest, tcg_env, vect_off);
1129 break;
1130 case MO_16|MO_SIGN:
1131 tcg_gen_ld16s_i32(tcg_dest, tcg_env, vect_off);
1132 break;
1133 case MO_32:
1134 case MO_32|MO_SIGN:
1135 tcg_gen_ld_i32(tcg_dest, tcg_env, vect_off);
1136 break;
1137 default:
1138 g_assert_not_reached();
1139 }
1140 }
1141
1142 /* Set value of an element within a vector register */
write_vec_element(DisasContext * s,TCGv_i64 tcg_src,int destidx,int element,MemOp memop)1143 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1144 int element, MemOp memop)
1145 {
1146 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1147 switch (memop) {
1148 case MO_8:
1149 tcg_gen_st8_i64(tcg_src, tcg_env, vect_off);
1150 break;
1151 case MO_16:
1152 tcg_gen_st16_i64(tcg_src, tcg_env, vect_off);
1153 break;
1154 case MO_32:
1155 tcg_gen_st32_i64(tcg_src, tcg_env, vect_off);
1156 break;
1157 case MO_64:
1158 tcg_gen_st_i64(tcg_src, tcg_env, vect_off);
1159 break;
1160 default:
1161 g_assert_not_reached();
1162 }
1163 }
1164
write_vec_element_i32(DisasContext * s,TCGv_i32 tcg_src,int destidx,int element,MemOp memop)1165 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1166 int destidx, int element, MemOp memop)
1167 {
1168 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1169 switch (memop) {
1170 case MO_8:
1171 tcg_gen_st8_i32(tcg_src, tcg_env, vect_off);
1172 break;
1173 case MO_16:
1174 tcg_gen_st16_i32(tcg_src, tcg_env, vect_off);
1175 break;
1176 case MO_32:
1177 tcg_gen_st_i32(tcg_src, tcg_env, vect_off);
1178 break;
1179 default:
1180 g_assert_not_reached();
1181 }
1182 }
1183
1184 /* Store from vector register to memory */
do_vec_st(DisasContext * s,int srcidx,int element,TCGv_i64 tcg_addr,MemOp mop)1185 static void do_vec_st(DisasContext *s, int srcidx, int element,
1186 TCGv_i64 tcg_addr, MemOp mop)
1187 {
1188 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1189
1190 read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE);
1191 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1192 }
1193
1194 /* Load from memory to vector register */
do_vec_ld(DisasContext * s,int destidx,int element,TCGv_i64 tcg_addr,MemOp mop)1195 static void do_vec_ld(DisasContext *s, int destidx, int element,
1196 TCGv_i64 tcg_addr, MemOp mop)
1197 {
1198 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1199
1200 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1201 write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE);
1202 }
1203
1204 /* Check that FP/Neon access is enabled. If it is, return
1205 * true. If not, emit code to generate an appropriate exception,
1206 * and return false; the caller should not emit any code for
1207 * the instruction. Note that this check must happen after all
1208 * unallocated-encoding checks (otherwise the syndrome information
1209 * for the resulting exception will be incorrect).
1210 */
fp_access_check_only(DisasContext * s)1211 static bool fp_access_check_only(DisasContext *s)
1212 {
1213 if (s->fp_excp_el) {
1214 assert(!s->fp_access_checked);
1215 s->fp_access_checked = true;
1216
1217 gen_exception_insn_el(s, 0, EXCP_UDEF,
1218 syn_fp_access_trap(1, 0xe, false, 0),
1219 s->fp_excp_el);
1220 return false;
1221 }
1222 s->fp_access_checked = true;
1223 return true;
1224 }
1225
fp_access_check(DisasContext * s)1226 static bool fp_access_check(DisasContext *s)
1227 {
1228 if (!fp_access_check_only(s)) {
1229 return false;
1230 }
1231 if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
1232 gen_exception_insn(s, 0, EXCP_UDEF,
1233 syn_smetrap(SME_ET_Streaming, false));
1234 return false;
1235 }
1236 return true;
1237 }
1238
1239 /*
1240 * Check that SVE access is enabled. If it is, return true.
1241 * If not, emit code to generate an appropriate exception and return false.
1242 * This function corresponds to CheckSVEEnabled().
1243 */
sve_access_check(DisasContext * s)1244 bool sve_access_check(DisasContext *s)
1245 {
1246 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) {
1247 assert(dc_isar_feature(aa64_sme, s));
1248 if (!sme_sm_enabled_check(s)) {
1249 goto fail_exit;
1250 }
1251 } else if (s->sve_excp_el) {
1252 gen_exception_insn_el(s, 0, EXCP_UDEF,
1253 syn_sve_access_trap(), s->sve_excp_el);
1254 goto fail_exit;
1255 }
1256 s->sve_access_checked = true;
1257 return fp_access_check(s);
1258
1259 fail_exit:
1260 /* Assert that we only raise one exception per instruction. */
1261 assert(!s->sve_access_checked);
1262 s->sve_access_checked = true;
1263 return false;
1264 }
1265
1266 /*
1267 * Check that SME access is enabled, raise an exception if not.
1268 * Note that this function corresponds to CheckSMEAccess and is
1269 * only used directly for cpregs.
1270 */
sme_access_check(DisasContext * s)1271 static bool sme_access_check(DisasContext *s)
1272 {
1273 if (s->sme_excp_el) {
1274 gen_exception_insn_el(s, 0, EXCP_UDEF,
1275 syn_smetrap(SME_ET_AccessTrap, false),
1276 s->sme_excp_el);
1277 return false;
1278 }
1279 return true;
1280 }
1281
1282 /* This function corresponds to CheckSMEEnabled. */
sme_enabled_check(DisasContext * s)1283 bool sme_enabled_check(DisasContext *s)
1284 {
1285 /*
1286 * Note that unlike sve_excp_el, we have not constrained sme_excp_el
1287 * to be zero when fp_excp_el has priority. This is because we need
1288 * sme_excp_el by itself for cpregs access checks.
1289 */
1290 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) {
1291 s->fp_access_checked = true;
1292 return sme_access_check(s);
1293 }
1294 return fp_access_check_only(s);
1295 }
1296
1297 /* Common subroutine for CheckSMEAnd*Enabled. */
sme_enabled_check_with_svcr(DisasContext * s,unsigned req)1298 bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
1299 {
1300 if (!sme_enabled_check(s)) {
1301 return false;
1302 }
1303 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
1304 gen_exception_insn(s, 0, EXCP_UDEF,
1305 syn_smetrap(SME_ET_NotStreaming, false));
1306 return false;
1307 }
1308 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
1309 gen_exception_insn(s, 0, EXCP_UDEF,
1310 syn_smetrap(SME_ET_InactiveZA, false));
1311 return false;
1312 }
1313 return true;
1314 }
1315
1316 /*
1317 * This utility function is for doing register extension with an
1318 * optional shift. You will likely want to pass a temporary for the
1319 * destination register. See DecodeRegExtend() in the ARM ARM.
1320 */
ext_and_shift_reg(TCGv_i64 tcg_out,TCGv_i64 tcg_in,int option,unsigned int shift)1321 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1322 int option, unsigned int shift)
1323 {
1324 int extsize = extract32(option, 0, 2);
1325 bool is_signed = extract32(option, 2, 1);
1326
1327 tcg_gen_ext_i64(tcg_out, tcg_in, extsize | (is_signed ? MO_SIGN : 0));
1328 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1329 }
1330
gen_check_sp_alignment(DisasContext * s)1331 static inline void gen_check_sp_alignment(DisasContext *s)
1332 {
1333 /* The AArch64 architecture mandates that (if enabled via PSTATE
1334 * or SCTLR bits) there is a check that SP is 16-aligned on every
1335 * SP-relative load or store (with an exception generated if it is not).
1336 * In line with general QEMU practice regarding misaligned accesses,
1337 * we omit these checks for the sake of guest program performance.
1338 * This function is provided as a hook so we can more easily add these
1339 * checks in future (possibly as a "favour catching guest program bugs
1340 * over speed" user selectable option).
1341 */
1342 }
1343
1344 /*
1345 * This provides a simple table based table lookup decoder. It is
1346 * intended to be used when the relevant bits for decode are too
1347 * awkwardly placed and switch/if based logic would be confusing and
1348 * deeply nested. Since it's a linear search through the table, tables
1349 * should be kept small.
1350 *
1351 * It returns the first handler where insn & mask == pattern, or
1352 * NULL if there is no match.
1353 * The table is terminated by an empty mask (i.e. 0)
1354 */
lookup_disas_fn(const AArch64DecodeTable * table,uint32_t insn)1355 static inline AArch64DecodeFn *lookup_disas_fn(const AArch64DecodeTable *table,
1356 uint32_t insn)
1357 {
1358 const AArch64DecodeTable *tptr = table;
1359
1360 while (tptr->mask) {
1361 if ((insn & tptr->mask) == tptr->pattern) {
1362 return tptr->disas_fn;
1363 }
1364 tptr++;
1365 }
1366 return NULL;
1367 }
1368
1369 /*
1370 * The instruction disassembly implemented here matches
1371 * the instruction encoding classifications in chapter C4
1372 * of the ARM Architecture Reference Manual (DDI0487B_a);
1373 * classification names and decode diagrams here should generally
1374 * match up with those in the manual.
1375 */
1376
trans_B(DisasContext * s,arg_i * a)1377 static bool trans_B(DisasContext *s, arg_i *a)
1378 {
1379 reset_btype(s);
1380 gen_goto_tb(s, 0, a->imm);
1381 return true;
1382 }
1383
trans_BL(DisasContext * s,arg_i * a)1384 static bool trans_BL(DisasContext *s, arg_i *a)
1385 {
1386 gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s));
1387 reset_btype(s);
1388 gen_goto_tb(s, 0, a->imm);
1389 return true;
1390 }
1391
1392
trans_CBZ(DisasContext * s,arg_cbz * a)1393 static bool trans_CBZ(DisasContext *s, arg_cbz *a)
1394 {
1395 DisasLabel match;
1396 TCGv_i64 tcg_cmp;
1397
1398 tcg_cmp = read_cpu_reg(s, a->rt, a->sf);
1399 reset_btype(s);
1400
1401 match = gen_disas_label(s);
1402 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1403 tcg_cmp, 0, match.label);
1404 gen_goto_tb(s, 0, 4);
1405 set_disas_label(s, match);
1406 gen_goto_tb(s, 1, a->imm);
1407 return true;
1408 }
1409
trans_TBZ(DisasContext * s,arg_tbz * a)1410 static bool trans_TBZ(DisasContext *s, arg_tbz *a)
1411 {
1412 DisasLabel match;
1413 TCGv_i64 tcg_cmp;
1414
1415 tcg_cmp = tcg_temp_new_i64();
1416 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos);
1417
1418 reset_btype(s);
1419
1420 match = gen_disas_label(s);
1421 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1422 tcg_cmp, 0, match.label);
1423 gen_goto_tb(s, 0, 4);
1424 set_disas_label(s, match);
1425 gen_goto_tb(s, 1, a->imm);
1426 return true;
1427 }
1428
trans_B_cond(DisasContext * s,arg_B_cond * a)1429 static bool trans_B_cond(DisasContext *s, arg_B_cond *a)
1430 {
1431 /* BC.cond is only present with FEAT_HBC */
1432 if (a->c && !dc_isar_feature(aa64_hbc, s)) {
1433 return false;
1434 }
1435 reset_btype(s);
1436 if (a->cond < 0x0e) {
1437 /* genuinely conditional branches */
1438 DisasLabel match = gen_disas_label(s);
1439 arm_gen_test_cc(a->cond, match.label);
1440 gen_goto_tb(s, 0, 4);
1441 set_disas_label(s, match);
1442 gen_goto_tb(s, 1, a->imm);
1443 } else {
1444 /* 0xe and 0xf are both "always" conditions */
1445 gen_goto_tb(s, 0, a->imm);
1446 }
1447 return true;
1448 }
1449
set_btype_for_br(DisasContext * s,int rn)1450 static void set_btype_for_br(DisasContext *s, int rn)
1451 {
1452 if (dc_isar_feature(aa64_bti, s)) {
1453 /* BR to {x16,x17} or !guard -> 1, else 3. */
1454 set_btype(s, rn == 16 || rn == 17 || !s->guarded_page ? 1 : 3);
1455 }
1456 }
1457
set_btype_for_blr(DisasContext * s)1458 static void set_btype_for_blr(DisasContext *s)
1459 {
1460 if (dc_isar_feature(aa64_bti, s)) {
1461 /* BLR sets BTYPE to 2, regardless of source guarded page. */
1462 set_btype(s, 2);
1463 }
1464 }
1465
trans_BR(DisasContext * s,arg_r * a)1466 static bool trans_BR(DisasContext *s, arg_r *a)
1467 {
1468 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1469 set_btype_for_br(s, a->rn);
1470 s->base.is_jmp = DISAS_JUMP;
1471 return true;
1472 }
1473
trans_BLR(DisasContext * s,arg_r * a)1474 static bool trans_BLR(DisasContext *s, arg_r *a)
1475 {
1476 TCGv_i64 dst = cpu_reg(s, a->rn);
1477 TCGv_i64 lr = cpu_reg(s, 30);
1478 if (dst == lr) {
1479 TCGv_i64 tmp = tcg_temp_new_i64();
1480 tcg_gen_mov_i64(tmp, dst);
1481 dst = tmp;
1482 }
1483 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1484 gen_a64_set_pc(s, dst);
1485 set_btype_for_blr(s);
1486 s->base.is_jmp = DISAS_JUMP;
1487 return true;
1488 }
1489
trans_RET(DisasContext * s,arg_r * a)1490 static bool trans_RET(DisasContext *s, arg_r *a)
1491 {
1492 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1493 s->base.is_jmp = DISAS_JUMP;
1494 return true;
1495 }
1496
auth_branch_target(DisasContext * s,TCGv_i64 dst,TCGv_i64 modifier,bool use_key_a)1497 static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,
1498 TCGv_i64 modifier, bool use_key_a)
1499 {
1500 TCGv_i64 truedst;
1501 /*
1502 * Return the branch target for a BRAA/RETA/etc, which is either
1503 * just the destination dst, or that value with the pauth check
1504 * done and the code removed from the high bits.
1505 */
1506 if (!s->pauth_active) {
1507 return dst;
1508 }
1509
1510 truedst = tcg_temp_new_i64();
1511 if (use_key_a) {
1512 gen_helper_autia_combined(truedst, tcg_env, dst, modifier);
1513 } else {
1514 gen_helper_autib_combined(truedst, tcg_env, dst, modifier);
1515 }
1516 return truedst;
1517 }
1518
trans_BRAZ(DisasContext * s,arg_braz * a)1519 static bool trans_BRAZ(DisasContext *s, arg_braz *a)
1520 {
1521 TCGv_i64 dst;
1522
1523 if (!dc_isar_feature(aa64_pauth, s)) {
1524 return false;
1525 }
1526
1527 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1528 gen_a64_set_pc(s, dst);
1529 set_btype_for_br(s, a->rn);
1530 s->base.is_jmp = DISAS_JUMP;
1531 return true;
1532 }
1533
trans_BLRAZ(DisasContext * s,arg_braz * a)1534 static bool trans_BLRAZ(DisasContext *s, arg_braz *a)
1535 {
1536 TCGv_i64 dst, lr;
1537
1538 if (!dc_isar_feature(aa64_pauth, s)) {
1539 return false;
1540 }
1541
1542 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1543 lr = cpu_reg(s, 30);
1544 if (dst == lr) {
1545 TCGv_i64 tmp = tcg_temp_new_i64();
1546 tcg_gen_mov_i64(tmp, dst);
1547 dst = tmp;
1548 }
1549 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1550 gen_a64_set_pc(s, dst);
1551 set_btype_for_blr(s);
1552 s->base.is_jmp = DISAS_JUMP;
1553 return true;
1554 }
1555
trans_RETA(DisasContext * s,arg_reta * a)1556 static bool trans_RETA(DisasContext *s, arg_reta *a)
1557 {
1558 TCGv_i64 dst;
1559
1560 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m);
1561 gen_a64_set_pc(s, dst);
1562 s->base.is_jmp = DISAS_JUMP;
1563 return true;
1564 }
1565
trans_BRA(DisasContext * s,arg_bra * a)1566 static bool trans_BRA(DisasContext *s, arg_bra *a)
1567 {
1568 TCGv_i64 dst;
1569
1570 if (!dc_isar_feature(aa64_pauth, s)) {
1571 return false;
1572 }
1573 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m);
1574 gen_a64_set_pc(s, dst);
1575 set_btype_for_br(s, a->rn);
1576 s->base.is_jmp = DISAS_JUMP;
1577 return true;
1578 }
1579
trans_BLRA(DisasContext * s,arg_bra * a)1580 static bool trans_BLRA(DisasContext *s, arg_bra *a)
1581 {
1582 TCGv_i64 dst, lr;
1583
1584 if (!dc_isar_feature(aa64_pauth, s)) {
1585 return false;
1586 }
1587 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m);
1588 lr = cpu_reg(s, 30);
1589 if (dst == lr) {
1590 TCGv_i64 tmp = tcg_temp_new_i64();
1591 tcg_gen_mov_i64(tmp, dst);
1592 dst = tmp;
1593 }
1594 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1595 gen_a64_set_pc(s, dst);
1596 set_btype_for_blr(s);
1597 s->base.is_jmp = DISAS_JUMP;
1598 return true;
1599 }
1600
trans_ERET(DisasContext * s,arg_ERET * a)1601 static bool trans_ERET(DisasContext *s, arg_ERET *a)
1602 {
1603 TCGv_i64 dst;
1604
1605 if (s->current_el == 0) {
1606 return false;
1607 }
1608 if (s->trap_eret) {
1609 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(0), 2);
1610 return true;
1611 }
1612 dst = tcg_temp_new_i64();
1613 tcg_gen_ld_i64(dst, tcg_env,
1614 offsetof(CPUARMState, elr_el[s->current_el]));
1615
1616 translator_io_start(&s->base);
1617
1618 gen_helper_exception_return(tcg_env, dst);
1619 /* Must exit loop to check un-masked IRQs */
1620 s->base.is_jmp = DISAS_EXIT;
1621 return true;
1622 }
1623
trans_ERETA(DisasContext * s,arg_reta * a)1624 static bool trans_ERETA(DisasContext *s, arg_reta *a)
1625 {
1626 TCGv_i64 dst;
1627
1628 if (!dc_isar_feature(aa64_pauth, s)) {
1629 return false;
1630 }
1631 if (s->current_el == 0) {
1632 return false;
1633 }
1634 /* The FGT trap takes precedence over an auth trap. */
1635 if (s->trap_eret) {
1636 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2);
1637 return true;
1638 }
1639 dst = tcg_temp_new_i64();
1640 tcg_gen_ld_i64(dst, tcg_env,
1641 offsetof(CPUARMState, elr_el[s->current_el]));
1642
1643 dst = auth_branch_target(s, dst, cpu_X[31], !a->m);
1644
1645 translator_io_start(&s->base);
1646
1647 gen_helper_exception_return(tcg_env, dst);
1648 /* Must exit loop to check un-masked IRQs */
1649 s->base.is_jmp = DISAS_EXIT;
1650 return true;
1651 }
1652
trans_NOP(DisasContext * s,arg_NOP * a)1653 static bool trans_NOP(DisasContext *s, arg_NOP *a)
1654 {
1655 return true;
1656 }
1657
trans_YIELD(DisasContext * s,arg_YIELD * a)1658 static bool trans_YIELD(DisasContext *s, arg_YIELD *a)
1659 {
1660 /*
1661 * When running in MTTCG we don't generate jumps to the yield and
1662 * WFE helpers as it won't affect the scheduling of other vCPUs.
1663 * If we wanted to more completely model WFE/SEV so we don't busy
1664 * spin unnecessarily we would need to do something more involved.
1665 */
1666 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1667 s->base.is_jmp = DISAS_YIELD;
1668 }
1669 return true;
1670 }
1671
trans_WFI(DisasContext * s,arg_WFI * a)1672 static bool trans_WFI(DisasContext *s, arg_WFI *a)
1673 {
1674 s->base.is_jmp = DISAS_WFI;
1675 return true;
1676 }
1677
trans_WFE(DisasContext * s,arg_WFI * a)1678 static bool trans_WFE(DisasContext *s, arg_WFI *a)
1679 {
1680 /*
1681 * When running in MTTCG we don't generate jumps to the yield and
1682 * WFE helpers as it won't affect the scheduling of other vCPUs.
1683 * If we wanted to more completely model WFE/SEV so we don't busy
1684 * spin unnecessarily we would need to do something more involved.
1685 */
1686 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1687 s->base.is_jmp = DISAS_WFE;
1688 }
1689 return true;
1690 }
1691
trans_XPACLRI(DisasContext * s,arg_XPACLRI * a)1692 static bool trans_XPACLRI(DisasContext *s, arg_XPACLRI *a)
1693 {
1694 if (s->pauth_active) {
1695 gen_helper_xpaci(cpu_X[30], tcg_env, cpu_X[30]);
1696 }
1697 return true;
1698 }
1699
trans_PACIA1716(DisasContext * s,arg_PACIA1716 * a)1700 static bool trans_PACIA1716(DisasContext *s, arg_PACIA1716 *a)
1701 {
1702 if (s->pauth_active) {
1703 gen_helper_pacia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1704 }
1705 return true;
1706 }
1707
trans_PACIB1716(DisasContext * s,arg_PACIB1716 * a)1708 static bool trans_PACIB1716(DisasContext *s, arg_PACIB1716 *a)
1709 {
1710 if (s->pauth_active) {
1711 gen_helper_pacib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1712 }
1713 return true;
1714 }
1715
trans_AUTIA1716(DisasContext * s,arg_AUTIA1716 * a)1716 static bool trans_AUTIA1716(DisasContext *s, arg_AUTIA1716 *a)
1717 {
1718 if (s->pauth_active) {
1719 gen_helper_autia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1720 }
1721 return true;
1722 }
1723
trans_AUTIB1716(DisasContext * s,arg_AUTIB1716 * a)1724 static bool trans_AUTIB1716(DisasContext *s, arg_AUTIB1716 *a)
1725 {
1726 if (s->pauth_active) {
1727 gen_helper_autib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
1728 }
1729 return true;
1730 }
1731
trans_ESB(DisasContext * s,arg_ESB * a)1732 static bool trans_ESB(DisasContext *s, arg_ESB *a)
1733 {
1734 /* Without RAS, we must implement this as NOP. */
1735 if (dc_isar_feature(aa64_ras, s)) {
1736 /*
1737 * QEMU does not have a source of physical SErrors,
1738 * so we are only concerned with virtual SErrors.
1739 * The pseudocode in the ARM for this case is
1740 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
1741 * AArch64.vESBOperation();
1742 * Most of the condition can be evaluated at translation time.
1743 * Test for EL2 present, and defer test for SEL2 to runtime.
1744 */
1745 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) {
1746 gen_helper_vesb(tcg_env);
1747 }
1748 }
1749 return true;
1750 }
1751
trans_PACIAZ(DisasContext * s,arg_PACIAZ * a)1752 static bool trans_PACIAZ(DisasContext *s, arg_PACIAZ *a)
1753 {
1754 if (s->pauth_active) {
1755 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1756 }
1757 return true;
1758 }
1759
trans_PACIASP(DisasContext * s,arg_PACIASP * a)1760 static bool trans_PACIASP(DisasContext *s, arg_PACIASP *a)
1761 {
1762 if (s->pauth_active) {
1763 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1764 }
1765 return true;
1766 }
1767
trans_PACIBZ(DisasContext * s,arg_PACIBZ * a)1768 static bool trans_PACIBZ(DisasContext *s, arg_PACIBZ *a)
1769 {
1770 if (s->pauth_active) {
1771 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1772 }
1773 return true;
1774 }
1775
trans_PACIBSP(DisasContext * s,arg_PACIBSP * a)1776 static bool trans_PACIBSP(DisasContext *s, arg_PACIBSP *a)
1777 {
1778 if (s->pauth_active) {
1779 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1780 }
1781 return true;
1782 }
1783
trans_AUTIAZ(DisasContext * s,arg_AUTIAZ * a)1784 static bool trans_AUTIAZ(DisasContext *s, arg_AUTIAZ *a)
1785 {
1786 if (s->pauth_active) {
1787 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1788 }
1789 return true;
1790 }
1791
trans_AUTIASP(DisasContext * s,arg_AUTIASP * a)1792 static bool trans_AUTIASP(DisasContext *s, arg_AUTIASP *a)
1793 {
1794 if (s->pauth_active) {
1795 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1796 }
1797 return true;
1798 }
1799
trans_AUTIBZ(DisasContext * s,arg_AUTIBZ * a)1800 static bool trans_AUTIBZ(DisasContext *s, arg_AUTIBZ *a)
1801 {
1802 if (s->pauth_active) {
1803 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
1804 }
1805 return true;
1806 }
1807
trans_AUTIBSP(DisasContext * s,arg_AUTIBSP * a)1808 static bool trans_AUTIBSP(DisasContext *s, arg_AUTIBSP *a)
1809 {
1810 if (s->pauth_active) {
1811 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
1812 }
1813 return true;
1814 }
1815
trans_CLREX(DisasContext * s,arg_CLREX * a)1816 static bool trans_CLREX(DisasContext *s, arg_CLREX *a)
1817 {
1818 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
1819 return true;
1820 }
1821
trans_DSB_DMB(DisasContext * s,arg_DSB_DMB * a)1822 static bool trans_DSB_DMB(DisasContext *s, arg_DSB_DMB *a)
1823 {
1824 /* We handle DSB and DMB the same way */
1825 TCGBar bar;
1826
1827 switch (a->types) {
1828 case 1: /* MBReqTypes_Reads */
1829 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
1830 break;
1831 case 2: /* MBReqTypes_Writes */
1832 bar = TCG_BAR_SC | TCG_MO_ST_ST;
1833 break;
1834 default: /* MBReqTypes_All */
1835 bar = TCG_BAR_SC | TCG_MO_ALL;
1836 break;
1837 }
1838 tcg_gen_mb(bar);
1839 return true;
1840 }
1841
trans_ISB(DisasContext * s,arg_ISB * a)1842 static bool trans_ISB(DisasContext *s, arg_ISB *a)
1843 {
1844 /*
1845 * We need to break the TB after this insn to execute
1846 * self-modifying code correctly and also to take
1847 * any pending interrupts immediately.
1848 */
1849 reset_btype(s);
1850 gen_goto_tb(s, 0, 4);
1851 return true;
1852 }
1853
trans_SB(DisasContext * s,arg_SB * a)1854 static bool trans_SB(DisasContext *s, arg_SB *a)
1855 {
1856 if (!dc_isar_feature(aa64_sb, s)) {
1857 return false;
1858 }
1859 /*
1860 * TODO: There is no speculation barrier opcode for TCG;
1861 * MB and end the TB instead.
1862 */
1863 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
1864 gen_goto_tb(s, 0, 4);
1865 return true;
1866 }
1867
trans_CFINV(DisasContext * s,arg_CFINV * a)1868 static bool trans_CFINV(DisasContext *s, arg_CFINV *a)
1869 {
1870 if (!dc_isar_feature(aa64_condm_4, s)) {
1871 return false;
1872 }
1873 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
1874 return true;
1875 }
1876
trans_XAFLAG(DisasContext * s,arg_XAFLAG * a)1877 static bool trans_XAFLAG(DisasContext *s, arg_XAFLAG *a)
1878 {
1879 TCGv_i32 z;
1880
1881 if (!dc_isar_feature(aa64_condm_5, s)) {
1882 return false;
1883 }
1884
1885 z = tcg_temp_new_i32();
1886
1887 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
1888
1889 /*
1890 * (!C & !Z) << 31
1891 * (!(C | Z)) << 31
1892 * ~((C | Z) << 31)
1893 * ~-(C | Z)
1894 * (C | Z) - 1
1895 */
1896 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
1897 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
1898
1899 /* !(Z & C) */
1900 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
1901 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
1902
1903 /* (!C & Z) << 31 -> -(Z & ~C) */
1904 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
1905 tcg_gen_neg_i32(cpu_VF, cpu_VF);
1906
1907 /* C | Z */
1908 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
1909
1910 return true;
1911 }
1912
trans_AXFLAG(DisasContext * s,arg_AXFLAG * a)1913 static bool trans_AXFLAG(DisasContext *s, arg_AXFLAG *a)
1914 {
1915 if (!dc_isar_feature(aa64_condm_5, s)) {
1916 return false;
1917 }
1918
1919 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
1920 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
1921
1922 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
1923 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
1924
1925 tcg_gen_movi_i32(cpu_NF, 0);
1926 tcg_gen_movi_i32(cpu_VF, 0);
1927
1928 return true;
1929 }
1930
trans_MSR_i_UAO(DisasContext * s,arg_i * a)1931 static bool trans_MSR_i_UAO(DisasContext *s, arg_i *a)
1932 {
1933 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
1934 return false;
1935 }
1936 if (a->imm & 1) {
1937 set_pstate_bits(PSTATE_UAO);
1938 } else {
1939 clear_pstate_bits(PSTATE_UAO);
1940 }
1941 gen_rebuild_hflags(s);
1942 s->base.is_jmp = DISAS_TOO_MANY;
1943 return true;
1944 }
1945
trans_MSR_i_PAN(DisasContext * s,arg_i * a)1946 static bool trans_MSR_i_PAN(DisasContext *s, arg_i *a)
1947 {
1948 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
1949 return false;
1950 }
1951 if (a->imm & 1) {
1952 set_pstate_bits(PSTATE_PAN);
1953 } else {
1954 clear_pstate_bits(PSTATE_PAN);
1955 }
1956 gen_rebuild_hflags(s);
1957 s->base.is_jmp = DISAS_TOO_MANY;
1958 return true;
1959 }
1960
trans_MSR_i_SPSEL(DisasContext * s,arg_i * a)1961 static bool trans_MSR_i_SPSEL(DisasContext *s, arg_i *a)
1962 {
1963 if (s->current_el == 0) {
1964 return false;
1965 }
1966 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP));
1967 s->base.is_jmp = DISAS_TOO_MANY;
1968 return true;
1969 }
1970
trans_MSR_i_SBSS(DisasContext * s,arg_i * a)1971 static bool trans_MSR_i_SBSS(DisasContext *s, arg_i *a)
1972 {
1973 if (!dc_isar_feature(aa64_ssbs, s)) {
1974 return false;
1975 }
1976 if (a->imm & 1) {
1977 set_pstate_bits(PSTATE_SSBS);
1978 } else {
1979 clear_pstate_bits(PSTATE_SSBS);
1980 }
1981 /* Don't need to rebuild hflags since SSBS is a nop */
1982 s->base.is_jmp = DISAS_TOO_MANY;
1983 return true;
1984 }
1985
trans_MSR_i_DIT(DisasContext * s,arg_i * a)1986 static bool trans_MSR_i_DIT(DisasContext *s, arg_i *a)
1987 {
1988 if (!dc_isar_feature(aa64_dit, s)) {
1989 return false;
1990 }
1991 if (a->imm & 1) {
1992 set_pstate_bits(PSTATE_DIT);
1993 } else {
1994 clear_pstate_bits(PSTATE_DIT);
1995 }
1996 /* There's no need to rebuild hflags because DIT is a nop */
1997 s->base.is_jmp = DISAS_TOO_MANY;
1998 return true;
1999 }
2000
trans_MSR_i_TCO(DisasContext * s,arg_i * a)2001 static bool trans_MSR_i_TCO(DisasContext *s, arg_i *a)
2002 {
2003 if (dc_isar_feature(aa64_mte, s)) {
2004 /* Full MTE is enabled -- set the TCO bit as directed. */
2005 if (a->imm & 1) {
2006 set_pstate_bits(PSTATE_TCO);
2007 } else {
2008 clear_pstate_bits(PSTATE_TCO);
2009 }
2010 gen_rebuild_hflags(s);
2011 /* Many factors, including TCO, go into MTE_ACTIVE. */
2012 s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
2013 return true;
2014 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
2015 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2016 return true;
2017 } else {
2018 /* Insn not present */
2019 return false;
2020 }
2021 }
2022
trans_MSR_i_DAIFSET(DisasContext * s,arg_i * a)2023 static bool trans_MSR_i_DAIFSET(DisasContext *s, arg_i *a)
2024 {
2025 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm));
2026 s->base.is_jmp = DISAS_TOO_MANY;
2027 return true;
2028 }
2029
trans_MSR_i_DAIFCLEAR(DisasContext * s,arg_i * a)2030 static bool trans_MSR_i_DAIFCLEAR(DisasContext *s, arg_i *a)
2031 {
2032 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm));
2033 /* Exit the cpu loop to re-evaluate pending IRQs. */
2034 s->base.is_jmp = DISAS_UPDATE_EXIT;
2035 return true;
2036 }
2037
trans_MSR_i_ALLINT(DisasContext * s,arg_i * a)2038 static bool trans_MSR_i_ALLINT(DisasContext *s, arg_i *a)
2039 {
2040 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) {
2041 return false;
2042 }
2043
2044 if (a->imm == 0) {
2045 clear_pstate_bits(PSTATE_ALLINT);
2046 } else if (s->current_el > 1) {
2047 set_pstate_bits(PSTATE_ALLINT);
2048 } else {
2049 gen_helper_msr_set_allint_el1(tcg_env);
2050 }
2051
2052 /* Exit the cpu loop to re-evaluate pending IRQs. */
2053 s->base.is_jmp = DISAS_UPDATE_EXIT;
2054 return true;
2055 }
2056
trans_MSR_i_SVCR(DisasContext * s,arg_MSR_i_SVCR * a)2057 static bool trans_MSR_i_SVCR(DisasContext *s, arg_MSR_i_SVCR *a)
2058 {
2059 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) {
2060 return false;
2061 }
2062 if (sme_access_check(s)) {
2063 int old = s->pstate_sm | (s->pstate_za << 1);
2064 int new = a->imm * 3;
2065
2066 if ((old ^ new) & a->mask) {
2067 /* At least one bit changes. */
2068 gen_helper_set_svcr(tcg_env, tcg_constant_i32(new),
2069 tcg_constant_i32(a->mask));
2070 s->base.is_jmp = DISAS_TOO_MANY;
2071 }
2072 }
2073 return true;
2074 }
2075
gen_get_nzcv(TCGv_i64 tcg_rt)2076 static void gen_get_nzcv(TCGv_i64 tcg_rt)
2077 {
2078 TCGv_i32 tmp = tcg_temp_new_i32();
2079 TCGv_i32 nzcv = tcg_temp_new_i32();
2080
2081 /* build bit 31, N */
2082 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
2083 /* build bit 30, Z */
2084 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
2085 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
2086 /* build bit 29, C */
2087 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
2088 /* build bit 28, V */
2089 tcg_gen_shri_i32(tmp, cpu_VF, 31);
2090 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
2091 /* generate result */
2092 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
2093 }
2094
gen_set_nzcv(TCGv_i64 tcg_rt)2095 static void gen_set_nzcv(TCGv_i64 tcg_rt)
2096 {
2097 TCGv_i32 nzcv = tcg_temp_new_i32();
2098
2099 /* take NZCV from R[t] */
2100 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
2101
2102 /* bit 31, N */
2103 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
2104 /* bit 30, Z */
2105 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
2106 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
2107 /* bit 29, C */
2108 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
2109 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
2110 /* bit 28, V */
2111 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
2112 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
2113 }
2114
gen_sysreg_undef(DisasContext * s,bool isread,uint8_t op0,uint8_t op1,uint8_t op2,uint8_t crn,uint8_t crm,uint8_t rt)2115 static void gen_sysreg_undef(DisasContext *s, bool isread,
2116 uint8_t op0, uint8_t op1, uint8_t op2,
2117 uint8_t crn, uint8_t crm, uint8_t rt)
2118 {
2119 /*
2120 * Generate code to emit an UNDEF with correct syndrome
2121 * information for a failed system register access.
2122 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2123 * but if FEAT_IDST is implemented then read accesses to registers
2124 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2125 * syndrome.
2126 */
2127 uint32_t syndrome;
2128
2129 if (isread && dc_isar_feature(aa64_ids, s) &&
2130 arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) {
2131 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2132 } else {
2133 syndrome = syn_uncategorized();
2134 }
2135 gen_exception_insn(s, 0, EXCP_UDEF, syndrome);
2136 }
2137
2138 /* MRS - move from system register
2139 * MSR (register) - move to system register
2140 * SYS
2141 * SYSL
2142 * These are all essentially the same insn in 'read' and 'write'
2143 * versions, with varying op0 fields.
2144 */
handle_sys(DisasContext * s,bool isread,unsigned int op0,unsigned int op1,unsigned int op2,unsigned int crn,unsigned int crm,unsigned int rt)2145 static void handle_sys(DisasContext *s, bool isread,
2146 unsigned int op0, unsigned int op1, unsigned int op2,
2147 unsigned int crn, unsigned int crm, unsigned int rt)
2148 {
2149 uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2150 crn, crm, op0, op1, op2);
2151 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2152 bool need_exit_tb = false;
2153 bool nv_trap_to_el2 = false;
2154 bool nv_redirect_reg = false;
2155 bool skip_fp_access_checks = false;
2156 bool nv2_mem_redirect = false;
2157 TCGv_ptr tcg_ri = NULL;
2158 TCGv_i64 tcg_rt;
2159 uint32_t syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2160
2161 if (crn == 11 || crn == 15) {
2162 /*
2163 * Check for TIDCP trap, which must take precedence over
2164 * the UNDEF for "no such register" etc.
2165 */
2166 switch (s->current_el) {
2167 case 0:
2168 if (dc_isar_feature(aa64_tidcp1, s)) {
2169 gen_helper_tidcp_el0(tcg_env, tcg_constant_i32(syndrome));
2170 }
2171 break;
2172 case 1:
2173 gen_helper_tidcp_el1(tcg_env, tcg_constant_i32(syndrome));
2174 break;
2175 }
2176 }
2177
2178 if (!ri) {
2179 /* Unknown register; this might be a guest error or a QEMU
2180 * unimplemented feature.
2181 */
2182 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
2183 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2184 isread ? "read" : "write", op0, op1, crn, crm, op2);
2185 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2186 return;
2187 }
2188
2189 if (s->nv2 && ri->nv2_redirect_offset) {
2190 /*
2191 * Some registers always redirect to memory; some only do so if
2192 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in
2193 * pairs which share an offset; see the table in R_CSRPQ).
2194 */
2195 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) {
2196 nv2_mem_redirect = s->nv1;
2197 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) {
2198 nv2_mem_redirect = !s->nv1;
2199 } else {
2200 nv2_mem_redirect = true;
2201 }
2202 }
2203
2204 /* Check access permissions */
2205 if (!cp_access_ok(s->current_el, ri, isread)) {
2206 /*
2207 * FEAT_NV/NV2 handling does not do the usual FP access checks
2208 * for registers only accessible at EL2 (though it *does* do them
2209 * for registers accessible at EL1).
2210 */
2211 skip_fp_access_checks = true;
2212 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) {
2213 /*
2214 * This is one of the few EL2 registers which should redirect
2215 * to the equivalent EL1 register. We do that after running
2216 * the EL2 register's accessfn.
2217 */
2218 nv_redirect_reg = true;
2219 assert(!nv2_mem_redirect);
2220 } else if (nv2_mem_redirect) {
2221 /*
2222 * NV2 redirect-to-memory takes precedence over trap to EL2 or
2223 * UNDEF to EL1.
2224 */
2225 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) {
2226 /*
2227 * This register / instruction exists and is an EL2 register, so
2228 * we must trap to EL2 if accessed in nested virtualization EL1
2229 * instead of UNDEFing. We'll do that after the usual access checks.
2230 * (This makes a difference only for a couple of registers like
2231 * VSTTBR_EL2 where the "UNDEF if NonSecure" should take priority
2232 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have
2233 * an accessfn which does nothing when called from EL1, because
2234 * the trap-to-EL3 controls which would apply to that register
2235 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.)
2236 */
2237 nv_trap_to_el2 = true;
2238 } else {
2239 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2240 return;
2241 }
2242 }
2243
2244 if (ri->accessfn || (ri->fgt && s->fgt_active)) {
2245 /* Emit code to perform further access permissions checks at
2246 * runtime; this may result in an exception.
2247 */
2248 gen_a64_update_pc(s, 0);
2249 tcg_ri = tcg_temp_new_ptr();
2250 gen_helper_access_check_cp_reg(tcg_ri, tcg_env,
2251 tcg_constant_i32(key),
2252 tcg_constant_i32(syndrome),
2253 tcg_constant_i32(isread));
2254 } else if (ri->type & ARM_CP_RAISES_EXC) {
2255 /*
2256 * The readfn or writefn might raise an exception;
2257 * synchronize the CPU state in case it does.
2258 */
2259 gen_a64_update_pc(s, 0);
2260 }
2261
2262 if (!skip_fp_access_checks) {
2263 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
2264 return;
2265 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
2266 return;
2267 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
2268 return;
2269 }
2270 }
2271
2272 if (nv_trap_to_el2) {
2273 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2274 return;
2275 }
2276
2277 if (nv_redirect_reg) {
2278 /*
2279 * FEAT_NV2 redirection of an EL2 register to an EL1 register.
2280 * Conveniently in all cases the encoding of the EL1 register is
2281 * identical to the EL2 register except that opc1 is 0.
2282 * Get the reginfo for the EL1 register to use for the actual access.
2283 * We don't use the EL1 register's access function, and
2284 * fine-grained-traps on EL1 also do not apply here.
2285 */
2286 key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2287 crn, crm, op0, 0, op2);
2288 ri = get_arm_cp_reginfo(s->cp_regs, key);
2289 assert(ri);
2290 assert(cp_access_ok(s->current_el, ri, isread));
2291 /*
2292 * We might not have done an update_pc earlier, so check we don't
2293 * need it. We could support this in future if necessary.
2294 */
2295 assert(!(ri->type & ARM_CP_RAISES_EXC));
2296 }
2297
2298 if (nv2_mem_redirect) {
2299 /*
2300 * This system register is being redirected into an EL2 memory access.
2301 * This means it is not an IO operation, doesn't change hflags,
2302 * and need not end the TB, because it has no side effects.
2303 *
2304 * The access is 64-bit single copy atomic, guaranteed aligned because
2305 * of the definition of VCNR_EL2. Its endianness depends on
2306 * SCTLR_EL2.EE, not on the data endianness of EL1.
2307 * It is done under either the EL2 translation regime or the EL2&0
2308 * translation regime, depending on HCR_EL2.E2H. It behaves as if
2309 * PSTATE.PAN is 0.
2310 */
2311 TCGv_i64 ptr = tcg_temp_new_i64();
2312 MemOp mop = MO_64 | MO_ALIGN | MO_ATOM_IFALIGN;
2313 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
2314 int memidx = arm_to_core_mmu_idx(armmemidx);
2315 uint32_t syn;
2316
2317 mop |= (s->nv2_mem_be ? MO_BE : MO_LE);
2318
2319 tcg_gen_ld_i64(ptr, tcg_env, offsetof(CPUARMState, cp15.vncr_el2));
2320 tcg_gen_addi_i64(ptr, ptr,
2321 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK));
2322 tcg_rt = cpu_reg(s, rt);
2323
2324 syn = syn_data_abort_vncr(0, !isread, 0);
2325 disas_set_insn_syndrome(s, syn);
2326 if (isread) {
2327 tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
2328 } else {
2329 tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
2330 }
2331 return;
2332 }
2333
2334 /* Handle special cases first */
2335 switch (ri->type & ARM_CP_SPECIAL_MASK) {
2336 case 0:
2337 break;
2338 case ARM_CP_NOP:
2339 return;
2340 case ARM_CP_NZCV:
2341 tcg_rt = cpu_reg(s, rt);
2342 if (isread) {
2343 gen_get_nzcv(tcg_rt);
2344 } else {
2345 gen_set_nzcv(tcg_rt);
2346 }
2347 return;
2348 case ARM_CP_CURRENTEL:
2349 {
2350 /*
2351 * Reads as current EL value from pstate, which is
2352 * guaranteed to be constant by the tb flags.
2353 * For nested virt we should report EL2.
2354 */
2355 int el = s->nv ? 2 : s->current_el;
2356 tcg_rt = cpu_reg(s, rt);
2357 tcg_gen_movi_i64(tcg_rt, el << 2);
2358 return;
2359 }
2360 case ARM_CP_DC_ZVA:
2361 /* Writes clear the aligned block of memory which rt points into. */
2362 if (s->mte_active[0]) {
2363 int desc = 0;
2364
2365 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
2366 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
2367 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
2368
2369 tcg_rt = tcg_temp_new_i64();
2370 gen_helper_mte_check_zva(tcg_rt, tcg_env,
2371 tcg_constant_i32(desc), cpu_reg(s, rt));
2372 } else {
2373 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
2374 }
2375 gen_helper_dc_zva(tcg_env, tcg_rt);
2376 return;
2377 case ARM_CP_DC_GVA:
2378 {
2379 TCGv_i64 clean_addr, tag;
2380
2381 /*
2382 * DC_GVA, like DC_ZVA, requires that we supply the original
2383 * pointer for an invalid page. Probe that address first.
2384 */
2385 tcg_rt = cpu_reg(s, rt);
2386 clean_addr = clean_data_tbi(s, tcg_rt);
2387 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
2388
2389 if (s->ata[0]) {
2390 /* Extract the tag from the register to match STZGM. */
2391 tag = tcg_temp_new_i64();
2392 tcg_gen_shri_i64(tag, tcg_rt, 56);
2393 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2394 }
2395 }
2396 return;
2397 case ARM_CP_DC_GZVA:
2398 {
2399 TCGv_i64 clean_addr, tag;
2400
2401 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
2402 tcg_rt = cpu_reg(s, rt);
2403 clean_addr = clean_data_tbi(s, tcg_rt);
2404 gen_helper_dc_zva(tcg_env, clean_addr);
2405
2406 if (s->ata[0]) {
2407 /* Extract the tag from the register to match STZGM. */
2408 tag = tcg_temp_new_i64();
2409 tcg_gen_shri_i64(tag, tcg_rt, 56);
2410 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2411 }
2412 }
2413 return;
2414 default:
2415 g_assert_not_reached();
2416 }
2417
2418 if (ri->type & ARM_CP_IO) {
2419 /* I/O operations must end the TB here (whether read or write) */
2420 need_exit_tb = translator_io_start(&s->base);
2421 }
2422
2423 tcg_rt = cpu_reg(s, rt);
2424
2425 if (isread) {
2426 if (ri->type & ARM_CP_CONST) {
2427 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
2428 } else if (ri->readfn) {
2429 if (!tcg_ri) {
2430 tcg_ri = gen_lookup_cp_reg(key);
2431 }
2432 gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
2433 } else {
2434 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
2435 }
2436 } else {
2437 if (ri->type & ARM_CP_CONST) {
2438 /* If not forbidden by access permissions, treat as WI */
2439 return;
2440 } else if (ri->writefn) {
2441 if (!tcg_ri) {
2442 tcg_ri = gen_lookup_cp_reg(key);
2443 }
2444 gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
2445 } else {
2446 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
2447 }
2448 }
2449
2450 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
2451 /*
2452 * A write to any coprocessor register that ends a TB
2453 * must rebuild the hflags for the next TB.
2454 */
2455 gen_rebuild_hflags(s);
2456 /*
2457 * We default to ending the TB on a coprocessor register write,
2458 * but allow this to be suppressed by the register definition
2459 * (usually only necessary to work around guest bugs).
2460 */
2461 need_exit_tb = true;
2462 }
2463 if (need_exit_tb) {
2464 s->base.is_jmp = DISAS_UPDATE_EXIT;
2465 }
2466 }
2467
trans_SYS(DisasContext * s,arg_SYS * a)2468 static bool trans_SYS(DisasContext *s, arg_SYS *a)
2469 {
2470 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
2471 return true;
2472 }
2473
trans_SVC(DisasContext * s,arg_i * a)2474 static bool trans_SVC(DisasContext *s, arg_i *a)
2475 {
2476 /*
2477 * For SVC, HVC and SMC we advance the single-step state
2478 * machine before taking the exception. This is architecturally
2479 * mandated, to ensure that single-stepping a system call
2480 * instruction works properly.
2481 */
2482 uint32_t syndrome = syn_aa64_svc(a->imm);
2483 if (s->fgt_svc) {
2484 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2485 return true;
2486 }
2487 gen_ss_advance(s);
2488 gen_exception_insn(s, 4, EXCP_SWI, syndrome);
2489 return true;
2490 }
2491
trans_HVC(DisasContext * s,arg_i * a)2492 static bool trans_HVC(DisasContext *s, arg_i *a)
2493 {
2494 int target_el = s->current_el == 3 ? 3 : 2;
2495
2496 if (s->current_el == 0) {
2497 unallocated_encoding(s);
2498 return true;
2499 }
2500 /*
2501 * The pre HVC helper handles cases when HVC gets trapped
2502 * as an undefined insn by runtime configuration.
2503 */
2504 gen_a64_update_pc(s, 0);
2505 gen_helper_pre_hvc(tcg_env);
2506 /* Architecture requires ss advance before we do the actual work */
2507 gen_ss_advance(s);
2508 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el);
2509 return true;
2510 }
2511
trans_SMC(DisasContext * s,arg_i * a)2512 static bool trans_SMC(DisasContext *s, arg_i *a)
2513 {
2514 if (s->current_el == 0) {
2515 unallocated_encoding(s);
2516 return true;
2517 }
2518 gen_a64_update_pc(s, 0);
2519 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm)));
2520 /* Architecture requires ss advance before we do the actual work */
2521 gen_ss_advance(s);
2522 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3);
2523 return true;
2524 }
2525
trans_BRK(DisasContext * s,arg_i * a)2526 static bool trans_BRK(DisasContext *s, arg_i *a)
2527 {
2528 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm));
2529 return true;
2530 }
2531
trans_HLT(DisasContext * s,arg_i * a)2532 static bool trans_HLT(DisasContext *s, arg_i *a)
2533 {
2534 /*
2535 * HLT. This has two purposes.
2536 * Architecturally, it is an external halting debug instruction.
2537 * Since QEMU doesn't implement external debug, we treat this as
2538 * it is required for halting debug disabled: it will UNDEF.
2539 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2540 */
2541 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) {
2542 gen_exception_internal_insn(s, EXCP_SEMIHOST);
2543 } else {
2544 unallocated_encoding(s);
2545 }
2546 return true;
2547 }
2548
2549 /*
2550 * Load/Store exclusive instructions are implemented by remembering
2551 * the value/address loaded, and seeing if these are the same
2552 * when the store is performed. This is not actually the architecturally
2553 * mandated semantics, but it works for typical guest code sequences
2554 * and avoids having to monitor regular stores.
2555 *
2556 * The store exclusive uses the atomic cmpxchg primitives to avoid
2557 * races in multi-threaded linux-user and when MTTCG softmmu is
2558 * enabled.
2559 */
gen_load_exclusive(DisasContext * s,int rt,int rt2,int rn,int size,bool is_pair)2560 static void gen_load_exclusive(DisasContext *s, int rt, int rt2, int rn,
2561 int size, bool is_pair)
2562 {
2563 int idx = get_mem_index(s);
2564 TCGv_i64 dirty_addr, clean_addr;
2565 MemOp memop = check_atomic_align(s, rn, size + is_pair);
2566
2567 s->is_ldex = true;
2568 dirty_addr = cpu_reg_sp(s, rn);
2569 clean_addr = gen_mte_check1(s, dirty_addr, false, rn != 31, memop);
2570
2571 g_assert(size <= 3);
2572 if (is_pair) {
2573 g_assert(size >= 2);
2574 if (size == 2) {
2575 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2576 if (s->be_data == MO_LE) {
2577 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2578 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2579 } else {
2580 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2581 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2582 }
2583 } else {
2584 TCGv_i128 t16 = tcg_temp_new_i128();
2585
2586 tcg_gen_qemu_ld_i128(t16, clean_addr, idx, memop);
2587
2588 if (s->be_data == MO_LE) {
2589 tcg_gen_extr_i128_i64(cpu_exclusive_val,
2590 cpu_exclusive_high, t16);
2591 } else {
2592 tcg_gen_extr_i128_i64(cpu_exclusive_high,
2593 cpu_exclusive_val, t16);
2594 }
2595 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2596 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2597 }
2598 } else {
2599 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2600 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2601 }
2602 tcg_gen_mov_i64(cpu_exclusive_addr, clean_addr);
2603 }
2604
gen_store_exclusive(DisasContext * s,int rd,int rt,int rt2,int rn,int size,int is_pair)2605 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2606 int rn, int size, int is_pair)
2607 {
2608 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2609 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2610 * [addr] = {Rt};
2611 * if (is_pair) {
2612 * [addr + datasize] = {Rt2};
2613 * }
2614 * {Rd} = 0;
2615 * } else {
2616 * {Rd} = 1;
2617 * }
2618 * env->exclusive_addr = -1;
2619 */
2620 TCGLabel *fail_label = gen_new_label();
2621 TCGLabel *done_label = gen_new_label();
2622 TCGv_i64 tmp, clean_addr;
2623 MemOp memop;
2624
2625 /*
2626 * FIXME: We are out of spec here. We have recorded only the address
2627 * from load_exclusive, not the entire range, and we assume that the
2628 * size of the access on both sides match. The architecture allows the
2629 * store to be smaller than the load, so long as the stored bytes are
2630 * within the range recorded by the load.
2631 */
2632
2633 /* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
2634 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2635 tcg_gen_brcond_i64(TCG_COND_NE, clean_addr, cpu_exclusive_addr, fail_label);
2636
2637 /*
2638 * The write, and any associated faults, only happen if the virtual
2639 * and physical addresses pass the exclusive monitor check. These
2640 * faults are exceedingly unlikely, because normally the guest uses
2641 * the exact same address register for the load_exclusive, and we
2642 * would have recognized these faults there.
2643 *
2644 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an
2645 * unaligned 4-byte write within the range of an aligned 8-byte load.
2646 * With LSE2, the store would need to cross a 16-byte boundary when the
2647 * load did not, which would mean the store is outside the range
2648 * recorded for the monitor, which would have failed a corrected monitor
2649 * check above. For now, we assume no size change and retain the
2650 * MO_ALIGN to let tcg know what we checked in the load_exclusive.
2651 *
2652 * It is possible to trigger an MTE fault, by performing the load with
2653 * a virtual address with a valid tag and performing the store with the
2654 * same virtual address and a different invalid tag.
2655 */
2656 memop = size + is_pair;
2657 if (memop == MO_128 || !dc_isar_feature(aa64_lse2, s)) {
2658 memop |= MO_ALIGN;
2659 }
2660 memop = finalize_memop(s, memop);
2661 gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2662
2663 tmp = tcg_temp_new_i64();
2664 if (is_pair) {
2665 if (size == 2) {
2666 if (s->be_data == MO_LE) {
2667 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2668 } else {
2669 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2670 }
2671 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2672 cpu_exclusive_val, tmp,
2673 get_mem_index(s), memop);
2674 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2675 } else {
2676 TCGv_i128 t16 = tcg_temp_new_i128();
2677 TCGv_i128 c16 = tcg_temp_new_i128();
2678 TCGv_i64 a, b;
2679
2680 if (s->be_data == MO_LE) {
2681 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
2682 tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
2683 cpu_exclusive_high);
2684 } else {
2685 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
2686 tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
2687 cpu_exclusive_val);
2688 }
2689
2690 tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
2691 get_mem_index(s), memop);
2692
2693 a = tcg_temp_new_i64();
2694 b = tcg_temp_new_i64();
2695 if (s->be_data == MO_LE) {
2696 tcg_gen_extr_i128_i64(a, b, t16);
2697 } else {
2698 tcg_gen_extr_i128_i64(b, a, t16);
2699 }
2700
2701 tcg_gen_xor_i64(a, a, cpu_exclusive_val);
2702 tcg_gen_xor_i64(b, b, cpu_exclusive_high);
2703 tcg_gen_or_i64(tmp, a, b);
2704
2705 tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
2706 }
2707 } else {
2708 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
2709 cpu_reg(s, rt), get_mem_index(s), memop);
2710 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2711 }
2712 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
2713 tcg_gen_br(done_label);
2714
2715 gen_set_label(fail_label);
2716 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
2717 gen_set_label(done_label);
2718 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2719 }
2720
gen_compare_and_swap(DisasContext * s,int rs,int rt,int rn,int size)2721 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
2722 int rn, int size)
2723 {
2724 TCGv_i64 tcg_rs = cpu_reg(s, rs);
2725 TCGv_i64 tcg_rt = cpu_reg(s, rt);
2726 int memidx = get_mem_index(s);
2727 TCGv_i64 clean_addr;
2728 MemOp memop;
2729
2730 if (rn == 31) {
2731 gen_check_sp_alignment(s);
2732 }
2733 memop = check_atomic_align(s, rn, size);
2734 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2735 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt,
2736 memidx, memop);
2737 }
2738
gen_compare_and_swap_pair(DisasContext * s,int rs,int rt,int rn,int size)2739 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
2740 int rn, int size)
2741 {
2742 TCGv_i64 s1 = cpu_reg(s, rs);
2743 TCGv_i64 s2 = cpu_reg(s, rs + 1);
2744 TCGv_i64 t1 = cpu_reg(s, rt);
2745 TCGv_i64 t2 = cpu_reg(s, rt + 1);
2746 TCGv_i64 clean_addr;
2747 int memidx = get_mem_index(s);
2748 MemOp memop;
2749
2750 if (rn == 31) {
2751 gen_check_sp_alignment(s);
2752 }
2753
2754 /* This is a single atomic access, despite the "pair". */
2755 memop = check_atomic_align(s, rn, size + 1);
2756 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2757
2758 if (size == 2) {
2759 TCGv_i64 cmp = tcg_temp_new_i64();
2760 TCGv_i64 val = tcg_temp_new_i64();
2761
2762 if (s->be_data == MO_LE) {
2763 tcg_gen_concat32_i64(val, t1, t2);
2764 tcg_gen_concat32_i64(cmp, s1, s2);
2765 } else {
2766 tcg_gen_concat32_i64(val, t2, t1);
2767 tcg_gen_concat32_i64(cmp, s2, s1);
2768 }
2769
2770 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, memop);
2771
2772 if (s->be_data == MO_LE) {
2773 tcg_gen_extr32_i64(s1, s2, cmp);
2774 } else {
2775 tcg_gen_extr32_i64(s2, s1, cmp);
2776 }
2777 } else {
2778 TCGv_i128 cmp = tcg_temp_new_i128();
2779 TCGv_i128 val = tcg_temp_new_i128();
2780
2781 if (s->be_data == MO_LE) {
2782 tcg_gen_concat_i64_i128(val, t1, t2);
2783 tcg_gen_concat_i64_i128(cmp, s1, s2);
2784 } else {
2785 tcg_gen_concat_i64_i128(val, t2, t1);
2786 tcg_gen_concat_i64_i128(cmp, s2, s1);
2787 }
2788
2789 tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, memop);
2790
2791 if (s->be_data == MO_LE) {
2792 tcg_gen_extr_i128_i64(s1, s2, cmp);
2793 } else {
2794 tcg_gen_extr_i128_i64(s2, s1, cmp);
2795 }
2796 }
2797 }
2798
2799 /*
2800 * Compute the ISS.SF bit for syndrome information if an exception
2801 * is taken on a load or store. This indicates whether the instruction
2802 * is accessing a 32-bit or 64-bit register. This logic is derived
2803 * from the ARMv8 specs for LDR (Shared decode for all encodings).
2804 */
ldst_iss_sf(int size,bool sign,bool ext)2805 static bool ldst_iss_sf(int size, bool sign, bool ext)
2806 {
2807
2808 if (sign) {
2809 /*
2810 * Signed loads are 64 bit results if we are not going to
2811 * do a zero-extend from 32 to 64 after the load.
2812 * (For a store, sign and ext are always false.)
2813 */
2814 return !ext;
2815 } else {
2816 /* Unsigned loads/stores work at the specified size */
2817 return size == MO_64;
2818 }
2819 }
2820
trans_STXR(DisasContext * s,arg_stxr * a)2821 static bool trans_STXR(DisasContext *s, arg_stxr *a)
2822 {
2823 if (a->rn == 31) {
2824 gen_check_sp_alignment(s);
2825 }
2826 if (a->lasr) {
2827 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2828 }
2829 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false);
2830 return true;
2831 }
2832
trans_LDXR(DisasContext * s,arg_stxr * a)2833 static bool trans_LDXR(DisasContext *s, arg_stxr *a)
2834 {
2835 if (a->rn == 31) {
2836 gen_check_sp_alignment(s);
2837 }
2838 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false);
2839 if (a->lasr) {
2840 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2841 }
2842 return true;
2843 }
2844
trans_STLR(DisasContext * s,arg_stlr * a)2845 static bool trans_STLR(DisasContext *s, arg_stlr *a)
2846 {
2847 TCGv_i64 clean_addr;
2848 MemOp memop;
2849 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2850
2851 /*
2852 * StoreLORelease is the same as Store-Release for QEMU, but
2853 * needs the feature-test.
2854 */
2855 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2856 return false;
2857 }
2858 /* Generate ISS for non-exclusive accesses including LASR. */
2859 if (a->rn == 31) {
2860 gen_check_sp_alignment(s);
2861 }
2862 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2863 memop = check_ordered_align(s, a->rn, 0, true, a->sz);
2864 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
2865 true, a->rn != 31, memop);
2866 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt,
2867 iss_sf, a->lasr);
2868 return true;
2869 }
2870
trans_LDAR(DisasContext * s,arg_stlr * a)2871 static bool trans_LDAR(DisasContext *s, arg_stlr *a)
2872 {
2873 TCGv_i64 clean_addr;
2874 MemOp memop;
2875 bool iss_sf = ldst_iss_sf(a->sz, false, false);
2876
2877 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
2878 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
2879 return false;
2880 }
2881 /* Generate ISS for non-exclusive accesses including LASR. */
2882 if (a->rn == 31) {
2883 gen_check_sp_alignment(s);
2884 }
2885 memop = check_ordered_align(s, a->rn, 0, false, a->sz);
2886 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
2887 false, a->rn != 31, memop);
2888 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true,
2889 a->rt, iss_sf, a->lasr);
2890 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2891 return true;
2892 }
2893
trans_STXP(DisasContext * s,arg_stxr * a)2894 static bool trans_STXP(DisasContext *s, arg_stxr *a)
2895 {
2896 if (a->rn == 31) {
2897 gen_check_sp_alignment(s);
2898 }
2899 if (a->lasr) {
2900 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
2901 }
2902 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true);
2903 return true;
2904 }
2905
trans_LDXP(DisasContext * s,arg_stxr * a)2906 static bool trans_LDXP(DisasContext *s, arg_stxr *a)
2907 {
2908 if (a->rn == 31) {
2909 gen_check_sp_alignment(s);
2910 }
2911 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true);
2912 if (a->lasr) {
2913 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
2914 }
2915 return true;
2916 }
2917
trans_CASP(DisasContext * s,arg_CASP * a)2918 static bool trans_CASP(DisasContext *s, arg_CASP *a)
2919 {
2920 if (!dc_isar_feature(aa64_atomics, s)) {
2921 return false;
2922 }
2923 if (((a->rt | a->rs) & 1) != 0) {
2924 return false;
2925 }
2926
2927 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz);
2928 return true;
2929 }
2930
trans_CAS(DisasContext * s,arg_CAS * a)2931 static bool trans_CAS(DisasContext *s, arg_CAS *a)
2932 {
2933 if (!dc_isar_feature(aa64_atomics, s)) {
2934 return false;
2935 }
2936 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz);
2937 return true;
2938 }
2939
trans_LD_lit(DisasContext * s,arg_ldlit * a)2940 static bool trans_LD_lit(DisasContext *s, arg_ldlit *a)
2941 {
2942 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
2943 TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
2944 TCGv_i64 clean_addr = tcg_temp_new_i64();
2945 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
2946
2947 gen_pc_plus_diff(s, clean_addr, a->imm);
2948 do_gpr_ld(s, tcg_rt, clean_addr, memop,
2949 false, true, a->rt, iss_sf, false);
2950 return true;
2951 }
2952
trans_LD_lit_v(DisasContext * s,arg_ldlit * a)2953 static bool trans_LD_lit_v(DisasContext *s, arg_ldlit *a)
2954 {
2955 /* Load register (literal), vector version */
2956 TCGv_i64 clean_addr;
2957 MemOp memop;
2958
2959 if (!fp_access_check(s)) {
2960 return true;
2961 }
2962 memop = finalize_memop_asimd(s, a->sz);
2963 clean_addr = tcg_temp_new_i64();
2964 gen_pc_plus_diff(s, clean_addr, a->imm);
2965 do_fp_ld(s, a->rt, clean_addr, memop);
2966 return true;
2967 }
2968
op_addr_ldstpair_pre(DisasContext * s,arg_ldstpair * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)2969 static void op_addr_ldstpair_pre(DisasContext *s, arg_ldstpair *a,
2970 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
2971 uint64_t offset, bool is_store, MemOp mop)
2972 {
2973 if (a->rn == 31) {
2974 gen_check_sp_alignment(s);
2975 }
2976
2977 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
2978 if (!a->p) {
2979 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
2980 }
2981
2982 *clean_addr = gen_mte_checkN(s, *dirty_addr, is_store,
2983 (a->w || a->rn != 31), 2 << a->sz, mop);
2984 }
2985
op_addr_ldstpair_post(DisasContext * s,arg_ldstpair * a,TCGv_i64 dirty_addr,uint64_t offset)2986 static void op_addr_ldstpair_post(DisasContext *s, arg_ldstpair *a,
2987 TCGv_i64 dirty_addr, uint64_t offset)
2988 {
2989 if (a->w) {
2990 if (a->p) {
2991 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
2992 }
2993 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
2994 }
2995 }
2996
trans_STP(DisasContext * s,arg_ldstpair * a)2997 static bool trans_STP(DisasContext *s, arg_ldstpair *a)
2998 {
2999 uint64_t offset = a->imm << a->sz;
3000 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3001 MemOp mop = finalize_memop(s, a->sz);
3002
3003 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3004 tcg_rt = cpu_reg(s, a->rt);
3005 tcg_rt2 = cpu_reg(s, a->rt2);
3006 /*
3007 * We built mop above for the single logical access -- rebuild it
3008 * now for the paired operation.
3009 *
3010 * With LSE2, non-sign-extending pairs are treated atomically if
3011 * aligned, and if unaligned one of the pair will be completely
3012 * within a 16-byte block and that element will be atomic.
3013 * Otherwise each element is separately atomic.
3014 * In all cases, issue one operation with the correct atomicity.
3015 */
3016 mop = a->sz + 1;
3017 if (s->align_mem) {
3018 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3019 }
3020 mop = finalize_memop_pair(s, mop);
3021 if (a->sz == 2) {
3022 TCGv_i64 tmp = tcg_temp_new_i64();
3023
3024 if (s->be_data == MO_LE) {
3025 tcg_gen_concat32_i64(tmp, tcg_rt, tcg_rt2);
3026 } else {
3027 tcg_gen_concat32_i64(tmp, tcg_rt2, tcg_rt);
3028 }
3029 tcg_gen_qemu_st_i64(tmp, clean_addr, get_mem_index(s), mop);
3030 } else {
3031 TCGv_i128 tmp = tcg_temp_new_i128();
3032
3033 if (s->be_data == MO_LE) {
3034 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3035 } else {
3036 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3037 }
3038 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3039 }
3040 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3041 return true;
3042 }
3043
trans_LDP(DisasContext * s,arg_ldstpair * a)3044 static bool trans_LDP(DisasContext *s, arg_ldstpair *a)
3045 {
3046 uint64_t offset = a->imm << a->sz;
3047 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3048 MemOp mop = finalize_memop(s, a->sz);
3049
3050 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3051 tcg_rt = cpu_reg(s, a->rt);
3052 tcg_rt2 = cpu_reg(s, a->rt2);
3053
3054 /*
3055 * We built mop above for the single logical access -- rebuild it
3056 * now for the paired operation.
3057 *
3058 * With LSE2, non-sign-extending pairs are treated atomically if
3059 * aligned, and if unaligned one of the pair will be completely
3060 * within a 16-byte block and that element will be atomic.
3061 * Otherwise each element is separately atomic.
3062 * In all cases, issue one operation with the correct atomicity.
3063 *
3064 * This treats sign-extending loads like zero-extending loads,
3065 * since that reuses the most code below.
3066 */
3067 mop = a->sz + 1;
3068 if (s->align_mem) {
3069 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3070 }
3071 mop = finalize_memop_pair(s, mop);
3072 if (a->sz == 2) {
3073 int o2 = s->be_data == MO_LE ? 32 : 0;
3074 int o1 = o2 ^ 32;
3075
3076 tcg_gen_qemu_ld_i64(tcg_rt, clean_addr, get_mem_index(s), mop);
3077 if (a->sign) {
3078 tcg_gen_sextract_i64(tcg_rt2, tcg_rt, o2, 32);
3079 tcg_gen_sextract_i64(tcg_rt, tcg_rt, o1, 32);
3080 } else {
3081 tcg_gen_extract_i64(tcg_rt2, tcg_rt, o2, 32);
3082 tcg_gen_extract_i64(tcg_rt, tcg_rt, o1, 32);
3083 }
3084 } else {
3085 TCGv_i128 tmp = tcg_temp_new_i128();
3086
3087 tcg_gen_qemu_ld_i128(tmp, clean_addr, get_mem_index(s), mop);
3088 if (s->be_data == MO_LE) {
3089 tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, tmp);
3090 } else {
3091 tcg_gen_extr_i128_i64(tcg_rt2, tcg_rt, tmp);
3092 }
3093 }
3094 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3095 return true;
3096 }
3097
trans_STP_v(DisasContext * s,arg_ldstpair * a)3098 static bool trans_STP_v(DisasContext *s, arg_ldstpair *a)
3099 {
3100 uint64_t offset = a->imm << a->sz;
3101 TCGv_i64 clean_addr, dirty_addr;
3102 MemOp mop;
3103
3104 if (!fp_access_check(s)) {
3105 return true;
3106 }
3107
3108 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3109 mop = finalize_memop_asimd(s, a->sz);
3110 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3111 do_fp_st(s, a->rt, clean_addr, mop);
3112 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3113 do_fp_st(s, a->rt2, clean_addr, mop);
3114 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3115 return true;
3116 }
3117
trans_LDP_v(DisasContext * s,arg_ldstpair * a)3118 static bool trans_LDP_v(DisasContext *s, arg_ldstpair *a)
3119 {
3120 uint64_t offset = a->imm << a->sz;
3121 TCGv_i64 clean_addr, dirty_addr;
3122 MemOp mop;
3123
3124 if (!fp_access_check(s)) {
3125 return true;
3126 }
3127
3128 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3129 mop = finalize_memop_asimd(s, a->sz);
3130 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3131 do_fp_ld(s, a->rt, clean_addr, mop);
3132 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3133 do_fp_ld(s, a->rt2, clean_addr, mop);
3134 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3135 return true;
3136 }
3137
trans_STGP(DisasContext * s,arg_ldstpair * a)3138 static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
3139 {
3140 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3141 uint64_t offset = a->imm << LOG2_TAG_GRANULE;
3142 MemOp mop;
3143 TCGv_i128 tmp;
3144
3145 /* STGP only comes in one size. */
3146 tcg_debug_assert(a->sz == MO_64);
3147
3148 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3149 return false;
3150 }
3151
3152 if (a->rn == 31) {
3153 gen_check_sp_alignment(s);
3154 }
3155
3156 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3157 if (!a->p) {
3158 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3159 }
3160
3161 clean_addr = clean_data_tbi(s, dirty_addr);
3162 tcg_rt = cpu_reg(s, a->rt);
3163 tcg_rt2 = cpu_reg(s, a->rt2);
3164
3165 /*
3166 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3167 * and one tag operation. We implement it as one single aligned 16-byte
3168 * memory operation for convenience. Note that the alignment ensures
3169 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3170 */
3171 mop = finalize_memop_atom(s, MO_128 | MO_ALIGN, MO_ATOM_IFALIGN_PAIR);
3172
3173 tmp = tcg_temp_new_i128();
3174 if (s->be_data == MO_LE) {
3175 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3176 } else {
3177 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3178 }
3179 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3180
3181 /* Perform the tag store, if tag access enabled. */
3182 if (s->ata[0]) {
3183 if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3184 gen_helper_stg_parallel(tcg_env, dirty_addr, dirty_addr);
3185 } else {
3186 gen_helper_stg(tcg_env, dirty_addr, dirty_addr);
3187 }
3188 }
3189
3190 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3191 return true;
3192 }
3193
op_addr_ldst_imm_pre(DisasContext * s,arg_ldst_imm * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)3194 static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a,
3195 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3196 uint64_t offset, bool is_store, MemOp mop)
3197 {
3198 int memidx;
3199
3200 if (a->rn == 31) {
3201 gen_check_sp_alignment(s);
3202 }
3203
3204 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3205 if (!a->p) {
3206 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3207 }
3208 memidx = get_a64_user_mem_index(s, a->unpriv);
3209 *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store,
3210 a->w || a->rn != 31,
3211 mop, a->unpriv, memidx);
3212 }
3213
op_addr_ldst_imm_post(DisasContext * s,arg_ldst_imm * a,TCGv_i64 dirty_addr,uint64_t offset)3214 static void op_addr_ldst_imm_post(DisasContext *s, arg_ldst_imm *a,
3215 TCGv_i64 dirty_addr, uint64_t offset)
3216 {
3217 if (a->w) {
3218 if (a->p) {
3219 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3220 }
3221 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3222 }
3223 }
3224
trans_STR_i(DisasContext * s,arg_ldst_imm * a)3225 static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a)
3226 {
3227 bool iss_sf, iss_valid = !a->w;
3228 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3229 int memidx = get_a64_user_mem_index(s, a->unpriv);
3230 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3231
3232 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3233
3234 tcg_rt = cpu_reg(s, a->rt);
3235 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3236
3237 do_gpr_st_memidx(s, tcg_rt, clean_addr, mop, memidx,
3238 iss_valid, a->rt, iss_sf, false);
3239 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3240 return true;
3241 }
3242
trans_LDR_i(DisasContext * s,arg_ldst_imm * a)3243 static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a)
3244 {
3245 bool iss_sf, iss_valid = !a->w;
3246 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3247 int memidx = get_a64_user_mem_index(s, a->unpriv);
3248 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3249
3250 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3251
3252 tcg_rt = cpu_reg(s, a->rt);
3253 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3254
3255 do_gpr_ld_memidx(s, tcg_rt, clean_addr, mop,
3256 a->ext, memidx, iss_valid, a->rt, iss_sf, false);
3257 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3258 return true;
3259 }
3260
trans_STR_v_i(DisasContext * s,arg_ldst_imm * a)3261 static bool trans_STR_v_i(DisasContext *s, arg_ldst_imm *a)
3262 {
3263 TCGv_i64 clean_addr, dirty_addr;
3264 MemOp mop;
3265
3266 if (!fp_access_check(s)) {
3267 return true;
3268 }
3269 mop = finalize_memop_asimd(s, a->sz);
3270 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3271 do_fp_st(s, a->rt, clean_addr, mop);
3272 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3273 return true;
3274 }
3275
trans_LDR_v_i(DisasContext * s,arg_ldst_imm * a)3276 static bool trans_LDR_v_i(DisasContext *s, arg_ldst_imm *a)
3277 {
3278 TCGv_i64 clean_addr, dirty_addr;
3279 MemOp mop;
3280
3281 if (!fp_access_check(s)) {
3282 return true;
3283 }
3284 mop = finalize_memop_asimd(s, a->sz);
3285 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3286 do_fp_ld(s, a->rt, clean_addr, mop);
3287 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3288 return true;
3289 }
3290
op_addr_ldst_pre(DisasContext * s,arg_ldst * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,bool is_store,MemOp memop)3291 static void op_addr_ldst_pre(DisasContext *s, arg_ldst *a,
3292 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3293 bool is_store, MemOp memop)
3294 {
3295 TCGv_i64 tcg_rm;
3296
3297 if (a->rn == 31) {
3298 gen_check_sp_alignment(s);
3299 }
3300 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3301
3302 tcg_rm = read_cpu_reg(s, a->rm, 1);
3303 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0);
3304
3305 tcg_gen_add_i64(*dirty_addr, *dirty_addr, tcg_rm);
3306 *clean_addr = gen_mte_check1(s, *dirty_addr, is_store, true, memop);
3307 }
3308
trans_LDR(DisasContext * s,arg_ldst * a)3309 static bool trans_LDR(DisasContext *s, arg_ldst *a)
3310 {
3311 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3312 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3313 MemOp memop;
3314
3315 if (extract32(a->opt, 1, 1) == 0) {
3316 return false;
3317 }
3318
3319 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3320 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3321 tcg_rt = cpu_reg(s, a->rt);
3322 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3323 a->ext, true, a->rt, iss_sf, false);
3324 return true;
3325 }
3326
trans_STR(DisasContext * s,arg_ldst * a)3327 static bool trans_STR(DisasContext *s, arg_ldst *a)
3328 {
3329 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3330 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3331 MemOp memop;
3332
3333 if (extract32(a->opt, 1, 1) == 0) {
3334 return false;
3335 }
3336
3337 memop = finalize_memop(s, a->sz);
3338 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3339 tcg_rt = cpu_reg(s, a->rt);
3340 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false);
3341 return true;
3342 }
3343
trans_LDR_v(DisasContext * s,arg_ldst * a)3344 static bool trans_LDR_v(DisasContext *s, arg_ldst *a)
3345 {
3346 TCGv_i64 clean_addr, dirty_addr;
3347 MemOp memop;
3348
3349 if (extract32(a->opt, 1, 1) == 0) {
3350 return false;
3351 }
3352
3353 if (!fp_access_check(s)) {
3354 return true;
3355 }
3356
3357 memop = finalize_memop_asimd(s, a->sz);
3358 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3359 do_fp_ld(s, a->rt, clean_addr, memop);
3360 return true;
3361 }
3362
trans_STR_v(DisasContext * s,arg_ldst * a)3363 static bool trans_STR_v(DisasContext *s, arg_ldst *a)
3364 {
3365 TCGv_i64 clean_addr, dirty_addr;
3366 MemOp memop;
3367
3368 if (extract32(a->opt, 1, 1) == 0) {
3369 return false;
3370 }
3371
3372 if (!fp_access_check(s)) {
3373 return true;
3374 }
3375
3376 memop = finalize_memop_asimd(s, a->sz);
3377 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3378 do_fp_st(s, a->rt, clean_addr, memop);
3379 return true;
3380 }
3381
3382
do_atomic_ld(DisasContext * s,arg_atomic * a,AtomicThreeOpFn * fn,int sign,bool invert)3383 static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn,
3384 int sign, bool invert)
3385 {
3386 MemOp mop = a->sz | sign;
3387 TCGv_i64 clean_addr, tcg_rs, tcg_rt;
3388
3389 if (a->rn == 31) {
3390 gen_check_sp_alignment(s);
3391 }
3392 mop = check_atomic_align(s, a->rn, mop);
3393 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3394 a->rn != 31, mop);
3395 tcg_rs = read_cpu_reg(s, a->rs, true);
3396 tcg_rt = cpu_reg(s, a->rt);
3397 if (invert) {
3398 tcg_gen_not_i64(tcg_rs, tcg_rs);
3399 }
3400 /*
3401 * The tcg atomic primitives are all full barriers. Therefore we
3402 * can ignore the Acquire and Release bits of this instruction.
3403 */
3404 fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
3405
3406 if (mop & MO_SIGN) {
3407 switch (a->sz) {
3408 case MO_8:
3409 tcg_gen_ext8u_i64(tcg_rt, tcg_rt);
3410 break;
3411 case MO_16:
3412 tcg_gen_ext16u_i64(tcg_rt, tcg_rt);
3413 break;
3414 case MO_32:
3415 tcg_gen_ext32u_i64(tcg_rt, tcg_rt);
3416 break;
3417 case MO_64:
3418 break;
3419 default:
3420 g_assert_not_reached();
3421 }
3422 }
3423 return true;
3424 }
3425
3426 TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false)
3427 TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true)
3428 TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false)
3429 TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false)
TRANS_FEAT(LDSMAX,aa64_atomics,do_atomic_ld,a,tcg_gen_atomic_fetch_smax_i64,MO_SIGN,false)3430 TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false)
3431 TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false)
3432 TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false)
3433 TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false)
3434 TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false)
3435
3436 static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a)
3437 {
3438 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3439 TCGv_i64 clean_addr;
3440 MemOp mop;
3441
3442 if (!dc_isar_feature(aa64_atomics, s) ||
3443 !dc_isar_feature(aa64_rcpc_8_3, s)) {
3444 return false;
3445 }
3446 if (a->rn == 31) {
3447 gen_check_sp_alignment(s);
3448 }
3449 mop = check_atomic_align(s, a->rn, a->sz);
3450 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3451 a->rn != 31, mop);
3452 /*
3453 * LDAPR* are a special case because they are a simple load, not a
3454 * fetch-and-do-something op.
3455 * The architectural consistency requirements here are weaker than
3456 * full load-acquire (we only need "load-acquire processor consistent"),
3457 * but we choose to implement them as full LDAQ.
3458 */
3459 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false,
3460 true, a->rt, iss_sf, true);
3461 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3462 return true;
3463 }
3464
trans_LDRA(DisasContext * s,arg_LDRA * a)3465 static bool trans_LDRA(DisasContext *s, arg_LDRA *a)
3466 {
3467 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3468 MemOp memop;
3469
3470 /* Load with pointer authentication */
3471 if (!dc_isar_feature(aa64_pauth, s)) {
3472 return false;
3473 }
3474
3475 if (a->rn == 31) {
3476 gen_check_sp_alignment(s);
3477 }
3478 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3479
3480 if (s->pauth_active) {
3481 if (!a->m) {
3482 gen_helper_autda_combined(dirty_addr, tcg_env, dirty_addr,
3483 tcg_constant_i64(0));
3484 } else {
3485 gen_helper_autdb_combined(dirty_addr, tcg_env, dirty_addr,
3486 tcg_constant_i64(0));
3487 }
3488 }
3489
3490 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3491
3492 memop = finalize_memop(s, MO_64);
3493
3494 /* Note that "clean" and "dirty" here refer to TBI not PAC. */
3495 clean_addr = gen_mte_check1(s, dirty_addr, false,
3496 a->w || a->rn != 31, memop);
3497
3498 tcg_rt = cpu_reg(s, a->rt);
3499 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3500 /* extend */ false, /* iss_valid */ !a->w,
3501 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false);
3502
3503 if (a->w) {
3504 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3505 }
3506 return true;
3507 }
3508
trans_LDAPR_i(DisasContext * s,arg_ldapr_stlr_i * a)3509 static bool trans_LDAPR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3510 {
3511 TCGv_i64 clean_addr, dirty_addr;
3512 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0);
3513 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3514
3515 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3516 return false;
3517 }
3518
3519 if (a->rn == 31) {
3520 gen_check_sp_alignment(s);
3521 }
3522
3523 mop = check_ordered_align(s, a->rn, a->imm, false, mop);
3524 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3525 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3526 clean_addr = clean_data_tbi(s, dirty_addr);
3527
3528 /*
3529 * Load-AcquirePC semantics; we implement as the slightly more
3530 * restrictive Load-Acquire.
3531 */
3532 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true,
3533 a->rt, iss_sf, true);
3534 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3535 return true;
3536 }
3537
trans_STLR_i(DisasContext * s,arg_ldapr_stlr_i * a)3538 static bool trans_STLR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3539 {
3540 TCGv_i64 clean_addr, dirty_addr;
3541 MemOp mop = a->sz;
3542 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3543
3544 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3545 return false;
3546 }
3547
3548 /* TODO: ARMv8.4-LSE SCTLR.nAA */
3549
3550 if (a->rn == 31) {
3551 gen_check_sp_alignment(s);
3552 }
3553
3554 mop = check_ordered_align(s, a->rn, a->imm, true, mop);
3555 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3556 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3557 clean_addr = clean_data_tbi(s, dirty_addr);
3558
3559 /* Store-Release semantics */
3560 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3561 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true);
3562 return true;
3563 }
3564
trans_LD_mult(DisasContext * s,arg_ldst_mult * a)3565 static bool trans_LD_mult(DisasContext *s, arg_ldst_mult *a)
3566 {
3567 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3568 MemOp endian, align, mop;
3569
3570 int total; /* total bytes */
3571 int elements; /* elements per vector */
3572 int r;
3573 int size = a->sz;
3574
3575 if (!a->p && a->rm != 0) {
3576 /* For non-postindexed accesses the Rm field must be 0 */
3577 return false;
3578 }
3579 if (size == 3 && !a->q && a->selem != 1) {
3580 return false;
3581 }
3582 if (!fp_access_check(s)) {
3583 return true;
3584 }
3585
3586 if (a->rn == 31) {
3587 gen_check_sp_alignment(s);
3588 }
3589
3590 /* For our purposes, bytes are always little-endian. */
3591 endian = s->be_data;
3592 if (size == 0) {
3593 endian = MO_LE;
3594 }
3595
3596 total = a->rpt * a->selem * (a->q ? 16 : 8);
3597 tcg_rn = cpu_reg_sp(s, a->rn);
3598
3599 /*
3600 * Issue the MTE check vs the logical repeat count, before we
3601 * promote consecutive little-endian elements below.
3602 */
3603 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total,
3604 finalize_memop_asimd(s, size));
3605
3606 /*
3607 * Consecutive little-endian elements from a single register
3608 * can be promoted to a larger little-endian operation.
3609 */
3610 align = MO_ALIGN;
3611 if (a->selem == 1 && endian == MO_LE) {
3612 align = pow2_align(size);
3613 size = 3;
3614 }
3615 if (!s->align_mem) {
3616 align = 0;
3617 }
3618 mop = endian | size | align;
3619
3620 elements = (a->q ? 16 : 8) >> size;
3621 tcg_ebytes = tcg_constant_i64(1 << size);
3622 for (r = 0; r < a->rpt; r++) {
3623 int e;
3624 for (e = 0; e < elements; e++) {
3625 int xs;
3626 for (xs = 0; xs < a->selem; xs++) {
3627 int tt = (a->rt + r + xs) % 32;
3628 do_vec_ld(s, tt, e, clean_addr, mop);
3629 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3630 }
3631 }
3632 }
3633
3634 /*
3635 * For non-quad operations, setting a slice of the low 64 bits of
3636 * the register clears the high 64 bits (in the ARM ARM pseudocode
3637 * this is implicit in the fact that 'rval' is a 64 bit wide
3638 * variable). For quad operations, we might still need to zero
3639 * the high bits of SVE.
3640 */
3641 for (r = 0; r < a->rpt * a->selem; r++) {
3642 int tt = (a->rt + r) % 32;
3643 clear_vec_high(s, a->q, tt);
3644 }
3645
3646 if (a->p) {
3647 if (a->rm == 31) {
3648 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3649 } else {
3650 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3651 }
3652 }
3653 return true;
3654 }
3655
trans_ST_mult(DisasContext * s,arg_ldst_mult * a)3656 static bool trans_ST_mult(DisasContext *s, arg_ldst_mult *a)
3657 {
3658 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3659 MemOp endian, align, mop;
3660
3661 int total; /* total bytes */
3662 int elements; /* elements per vector */
3663 int r;
3664 int size = a->sz;
3665
3666 if (!a->p && a->rm != 0) {
3667 /* For non-postindexed accesses the Rm field must be 0 */
3668 return false;
3669 }
3670 if (size == 3 && !a->q && a->selem != 1) {
3671 return false;
3672 }
3673 if (!fp_access_check(s)) {
3674 return true;
3675 }
3676
3677 if (a->rn == 31) {
3678 gen_check_sp_alignment(s);
3679 }
3680
3681 /* For our purposes, bytes are always little-endian. */
3682 endian = s->be_data;
3683 if (size == 0) {
3684 endian = MO_LE;
3685 }
3686
3687 total = a->rpt * a->selem * (a->q ? 16 : 8);
3688 tcg_rn = cpu_reg_sp(s, a->rn);
3689
3690 /*
3691 * Issue the MTE check vs the logical repeat count, before we
3692 * promote consecutive little-endian elements below.
3693 */
3694 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total,
3695 finalize_memop_asimd(s, size));
3696
3697 /*
3698 * Consecutive little-endian elements from a single register
3699 * can be promoted to a larger little-endian operation.
3700 */
3701 align = MO_ALIGN;
3702 if (a->selem == 1 && endian == MO_LE) {
3703 align = pow2_align(size);
3704 size = 3;
3705 }
3706 if (!s->align_mem) {
3707 align = 0;
3708 }
3709 mop = endian | size | align;
3710
3711 elements = (a->q ? 16 : 8) >> size;
3712 tcg_ebytes = tcg_constant_i64(1 << size);
3713 for (r = 0; r < a->rpt; r++) {
3714 int e;
3715 for (e = 0; e < elements; e++) {
3716 int xs;
3717 for (xs = 0; xs < a->selem; xs++) {
3718 int tt = (a->rt + r + xs) % 32;
3719 do_vec_st(s, tt, e, clean_addr, mop);
3720 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3721 }
3722 }
3723 }
3724
3725 if (a->p) {
3726 if (a->rm == 31) {
3727 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3728 } else {
3729 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3730 }
3731 }
3732 return true;
3733 }
3734
trans_ST_single(DisasContext * s,arg_ldst_single * a)3735 static bool trans_ST_single(DisasContext *s, arg_ldst_single *a)
3736 {
3737 int xs, total, rt;
3738 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3739 MemOp mop;
3740
3741 if (!a->p && a->rm != 0) {
3742 return false;
3743 }
3744 if (!fp_access_check(s)) {
3745 return true;
3746 }
3747
3748 if (a->rn == 31) {
3749 gen_check_sp_alignment(s);
3750 }
3751
3752 total = a->selem << a->scale;
3753 tcg_rn = cpu_reg_sp(s, a->rn);
3754
3755 mop = finalize_memop_asimd(s, a->scale);
3756 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31,
3757 total, mop);
3758
3759 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3760 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3761 do_vec_st(s, rt, a->index, clean_addr, mop);
3762 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3763 }
3764
3765 if (a->p) {
3766 if (a->rm == 31) {
3767 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3768 } else {
3769 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3770 }
3771 }
3772 return true;
3773 }
3774
trans_LD_single(DisasContext * s,arg_ldst_single * a)3775 static bool trans_LD_single(DisasContext *s, arg_ldst_single *a)
3776 {
3777 int xs, total, rt;
3778 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3779 MemOp mop;
3780
3781 if (!a->p && a->rm != 0) {
3782 return false;
3783 }
3784 if (!fp_access_check(s)) {
3785 return true;
3786 }
3787
3788 if (a->rn == 31) {
3789 gen_check_sp_alignment(s);
3790 }
3791
3792 total = a->selem << a->scale;
3793 tcg_rn = cpu_reg_sp(s, a->rn);
3794
3795 mop = finalize_memop_asimd(s, a->scale);
3796 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3797 total, mop);
3798
3799 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3800 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3801 do_vec_ld(s, rt, a->index, clean_addr, mop);
3802 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3803 }
3804
3805 if (a->p) {
3806 if (a->rm == 31) {
3807 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3808 } else {
3809 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3810 }
3811 }
3812 return true;
3813 }
3814
trans_LD_single_repl(DisasContext * s,arg_LD_single_repl * a)3815 static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a)
3816 {
3817 int xs, total, rt;
3818 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3819 MemOp mop;
3820
3821 if (!a->p && a->rm != 0) {
3822 return false;
3823 }
3824 if (!fp_access_check(s)) {
3825 return true;
3826 }
3827
3828 if (a->rn == 31) {
3829 gen_check_sp_alignment(s);
3830 }
3831
3832 total = a->selem << a->scale;
3833 tcg_rn = cpu_reg_sp(s, a->rn);
3834
3835 mop = finalize_memop_asimd(s, a->scale);
3836 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
3837 total, mop);
3838
3839 tcg_ebytes = tcg_constant_i64(1 << a->scale);
3840 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
3841 /* Load and replicate to all elements */
3842 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
3843
3844 tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop);
3845 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt),
3846 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp);
3847 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3848 }
3849
3850 if (a->p) {
3851 if (a->rm == 31) {
3852 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3853 } else {
3854 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3855 }
3856 }
3857 return true;
3858 }
3859
trans_STZGM(DisasContext * s,arg_ldst_tag * a)3860 static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
3861 {
3862 TCGv_i64 addr, clean_addr, tcg_rt;
3863 int size = 4 << s->dcz_blocksize;
3864
3865 if (!dc_isar_feature(aa64_mte, s)) {
3866 return false;
3867 }
3868 if (s->current_el == 0) {
3869 return false;
3870 }
3871
3872 if (a->rn == 31) {
3873 gen_check_sp_alignment(s);
3874 }
3875
3876 addr = read_cpu_reg_sp(s, a->rn, true);
3877 tcg_gen_addi_i64(addr, addr, a->imm);
3878 tcg_rt = cpu_reg(s, a->rt);
3879
3880 if (s->ata[0]) {
3881 gen_helper_stzgm_tags(tcg_env, addr, tcg_rt);
3882 }
3883 /*
3884 * The non-tags portion of STZGM is mostly like DC_ZVA,
3885 * except the alignment happens before the access.
3886 */
3887 clean_addr = clean_data_tbi(s, addr);
3888 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3889 gen_helper_dc_zva(tcg_env, clean_addr);
3890 return true;
3891 }
3892
trans_STGM(DisasContext * s,arg_ldst_tag * a)3893 static bool trans_STGM(DisasContext *s, arg_ldst_tag *a)
3894 {
3895 TCGv_i64 addr, clean_addr, tcg_rt;
3896
3897 if (!dc_isar_feature(aa64_mte, s)) {
3898 return false;
3899 }
3900 if (s->current_el == 0) {
3901 return false;
3902 }
3903
3904 if (a->rn == 31) {
3905 gen_check_sp_alignment(s);
3906 }
3907
3908 addr = read_cpu_reg_sp(s, a->rn, true);
3909 tcg_gen_addi_i64(addr, addr, a->imm);
3910 tcg_rt = cpu_reg(s, a->rt);
3911
3912 if (s->ata[0]) {
3913 gen_helper_stgm(tcg_env, addr, tcg_rt);
3914 } else {
3915 MMUAccessType acc = MMU_DATA_STORE;
3916 int size = 4 << s->gm_blocksize;
3917
3918 clean_addr = clean_data_tbi(s, addr);
3919 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3920 gen_probe_access(s, clean_addr, acc, size);
3921 }
3922 return true;
3923 }
3924
trans_LDGM(DisasContext * s,arg_ldst_tag * a)3925 static bool trans_LDGM(DisasContext *s, arg_ldst_tag *a)
3926 {
3927 TCGv_i64 addr, clean_addr, tcg_rt;
3928
3929 if (!dc_isar_feature(aa64_mte, s)) {
3930 return false;
3931 }
3932 if (s->current_el == 0) {
3933 return false;
3934 }
3935
3936 if (a->rn == 31) {
3937 gen_check_sp_alignment(s);
3938 }
3939
3940 addr = read_cpu_reg_sp(s, a->rn, true);
3941 tcg_gen_addi_i64(addr, addr, a->imm);
3942 tcg_rt = cpu_reg(s, a->rt);
3943
3944 if (s->ata[0]) {
3945 gen_helper_ldgm(tcg_rt, tcg_env, addr);
3946 } else {
3947 MMUAccessType acc = MMU_DATA_LOAD;
3948 int size = 4 << s->gm_blocksize;
3949
3950 clean_addr = clean_data_tbi(s, addr);
3951 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
3952 gen_probe_access(s, clean_addr, acc, size);
3953 /* The result tags are zeros. */
3954 tcg_gen_movi_i64(tcg_rt, 0);
3955 }
3956 return true;
3957 }
3958
trans_LDG(DisasContext * s,arg_ldst_tag * a)3959 static bool trans_LDG(DisasContext *s, arg_ldst_tag *a)
3960 {
3961 TCGv_i64 addr, clean_addr, tcg_rt;
3962
3963 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3964 return false;
3965 }
3966
3967 if (a->rn == 31) {
3968 gen_check_sp_alignment(s);
3969 }
3970
3971 addr = read_cpu_reg_sp(s, a->rn, true);
3972 if (!a->p) {
3973 /* pre-index or signed offset */
3974 tcg_gen_addi_i64(addr, addr, a->imm);
3975 }
3976
3977 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE);
3978 tcg_rt = cpu_reg(s, a->rt);
3979 if (s->ata[0]) {
3980 gen_helper_ldg(tcg_rt, tcg_env, addr, tcg_rt);
3981 } else {
3982 /*
3983 * Tag access disabled: we must check for aborts on the load
3984 * load from [rn+offset], and then insert a 0 tag into rt.
3985 */
3986 clean_addr = clean_data_tbi(s, addr);
3987 gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
3988 gen_address_with_allocation_tag0(tcg_rt, tcg_rt);
3989 }
3990
3991 if (a->w) {
3992 /* pre-index or post-index */
3993 if (a->p) {
3994 /* post-index */
3995 tcg_gen_addi_i64(addr, addr, a->imm);
3996 }
3997 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
3998 }
3999 return true;
4000 }
4001
do_STG(DisasContext * s,arg_ldst_tag * a,bool is_zero,bool is_pair)4002 static bool do_STG(DisasContext *s, arg_ldst_tag *a, bool is_zero, bool is_pair)
4003 {
4004 TCGv_i64 addr, tcg_rt;
4005
4006 if (a->rn == 31) {
4007 gen_check_sp_alignment(s);
4008 }
4009
4010 addr = read_cpu_reg_sp(s, a->rn, true);
4011 if (!a->p) {
4012 /* pre-index or signed offset */
4013 tcg_gen_addi_i64(addr, addr, a->imm);
4014 }
4015 tcg_rt = cpu_reg_sp(s, a->rt);
4016 if (!s->ata[0]) {
4017 /*
4018 * For STG and ST2G, we need to check alignment and probe memory.
4019 * TODO: For STZG and STZ2G, we could rely on the stores below,
4020 * at least for system mode; user-only won't enforce alignment.
4021 */
4022 if (is_pair) {
4023 gen_helper_st2g_stub(tcg_env, addr);
4024 } else {
4025 gen_helper_stg_stub(tcg_env, addr);
4026 }
4027 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
4028 if (is_pair) {
4029 gen_helper_st2g_parallel(tcg_env, addr, tcg_rt);
4030 } else {
4031 gen_helper_stg_parallel(tcg_env, addr, tcg_rt);
4032 }
4033 } else {
4034 if (is_pair) {
4035 gen_helper_st2g(tcg_env, addr, tcg_rt);
4036 } else {
4037 gen_helper_stg(tcg_env, addr, tcg_rt);
4038 }
4039 }
4040
4041 if (is_zero) {
4042 TCGv_i64 clean_addr = clean_data_tbi(s, addr);
4043 TCGv_i64 zero64 = tcg_constant_i64(0);
4044 TCGv_i128 zero128 = tcg_temp_new_i128();
4045 int mem_index = get_mem_index(s);
4046 MemOp mop = finalize_memop(s, MO_128 | MO_ALIGN);
4047
4048 tcg_gen_concat_i64_i128(zero128, zero64, zero64);
4049
4050 /* This is 1 or 2 atomic 16-byte operations. */
4051 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4052 if (is_pair) {
4053 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4054 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4055 }
4056 }
4057
4058 if (a->w) {
4059 /* pre-index or post-index */
4060 if (a->p) {
4061 /* post-index */
4062 tcg_gen_addi_i64(addr, addr, a->imm);
4063 }
4064 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4065 }
4066 return true;
4067 }
4068
4069 TRANS_FEAT(STG, aa64_mte_insn_reg, do_STG, a, false, false)
4070 TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false)
4071 TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true)
4072 TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true)
4073
4074 typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32);
4075
do_SET(DisasContext * s,arg_set * a,bool is_epilogue,bool is_setg,SetFn fn)4076 static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue,
4077 bool is_setg, SetFn fn)
4078 {
4079 int memidx;
4080 uint32_t syndrome, desc = 0;
4081
4082 if (is_setg && !dc_isar_feature(aa64_mte, s)) {
4083 return false;
4084 }
4085
4086 /*
4087 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4088 * us to pull this check before the CheckMOPSEnabled() test
4089 * (which we do in the helper function)
4090 */
4091 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4092 a->rd == 31 || a->rn == 31) {
4093 return false;
4094 }
4095
4096 memidx = get_a64_user_mem_index(s, a->unpriv);
4097
4098 /*
4099 * We pass option_a == true, matching our implementation;
4100 * we pass wrong_option == false: helper function may set that bit.
4101 */
4102 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv,
4103 is_epilogue, false, true, a->rd, a->rs, a->rn);
4104
4105 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) {
4106 /* We may need to do MTE tag checking, so assemble the descriptor */
4107 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
4108 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
4109 desc = FIELD_DP32(desc, MTEDESC, WRITE, true);
4110 /* SIZEM1 and ALIGN we leave 0 (byte write) */
4111 }
4112 /* The helper function always needs the memidx even with MTE disabled */
4113 desc = FIELD_DP32(desc, MTEDESC, MIDX, memidx);
4114
4115 /*
4116 * The helper needs the register numbers, but since they're in
4117 * the syndrome anyway, we let it extract them from there rather
4118 * than passing in an extra three integer arguments.
4119 */
4120 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(desc));
4121 return true;
4122 }
4123
4124 TRANS_FEAT(SETP, aa64_mops, do_SET, a, false, false, gen_helper_setp)
4125 TRANS_FEAT(SETM, aa64_mops, do_SET, a, false, false, gen_helper_setm)
4126 TRANS_FEAT(SETE, aa64_mops, do_SET, a, true, false, gen_helper_sete)
4127 TRANS_FEAT(SETGP, aa64_mops, do_SET, a, false, true, gen_helper_setgp)
4128 TRANS_FEAT(SETGM, aa64_mops, do_SET, a, false, true, gen_helper_setgm)
4129 TRANS_FEAT(SETGE, aa64_mops, do_SET, a, true, true, gen_helper_setge)
4130
4131 typedef void CpyFn(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32);
4132
do_CPY(DisasContext * s,arg_cpy * a,bool is_epilogue,CpyFn fn)4133 static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn)
4134 {
4135 int rmemidx, wmemidx;
4136 uint32_t syndrome, rdesc = 0, wdesc = 0;
4137 bool wunpriv = extract32(a->options, 0, 1);
4138 bool runpriv = extract32(a->options, 1, 1);
4139
4140 /*
4141 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4142 * us to pull this check before the CheckMOPSEnabled() test
4143 * (which we do in the helper function)
4144 */
4145 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4146 a->rd == 31 || a->rs == 31 || a->rn == 31) {
4147 return false;
4148 }
4149
4150 rmemidx = get_a64_user_mem_index(s, runpriv);
4151 wmemidx = get_a64_user_mem_index(s, wunpriv);
4152
4153 /*
4154 * We pass option_a == true, matching our implementation;
4155 * we pass wrong_option == false: helper function may set that bit.
4156 */
4157 syndrome = syn_mop(false, false, a->options, is_epilogue,
4158 false, true, a->rd, a->rs, a->rn);
4159
4160 /* If we need to do MTE tag checking, assemble the descriptors */
4161 if (s->mte_active[runpriv]) {
4162 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid);
4163 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma);
4164 }
4165 if (s->mte_active[wunpriv]) {
4166 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid);
4167 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma);
4168 wdesc = FIELD_DP32(wdesc, MTEDESC, WRITE, true);
4169 }
4170 /* The helper function needs these parts of the descriptor regardless */
4171 rdesc = FIELD_DP32(rdesc, MTEDESC, MIDX, rmemidx);
4172 wdesc = FIELD_DP32(wdesc, MTEDESC, MIDX, wmemidx);
4173
4174 /*
4175 * The helper needs the register numbers, but since they're in
4176 * the syndrome anyway, we let it extract them from there rather
4177 * than passing in an extra three integer arguments.
4178 */
4179 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(wdesc),
4180 tcg_constant_i32(rdesc));
4181 return true;
4182 }
4183
4184 TRANS_FEAT(CPYP, aa64_mops, do_CPY, a, false, gen_helper_cpyp)
4185 TRANS_FEAT(CPYM, aa64_mops, do_CPY, a, false, gen_helper_cpym)
4186 TRANS_FEAT(CPYE, aa64_mops, do_CPY, a, true, gen_helper_cpye)
4187 TRANS_FEAT(CPYFP, aa64_mops, do_CPY, a, false, gen_helper_cpyfp)
4188 TRANS_FEAT(CPYFM, aa64_mops, do_CPY, a, false, gen_helper_cpyfm)
4189 TRANS_FEAT(CPYFE, aa64_mops, do_CPY, a, true, gen_helper_cpyfe)
4190
4191 typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64);
4192
gen_rri(DisasContext * s,arg_rri_sf * a,bool rd_sp,bool rn_sp,ArithTwoOp * fn)4193 static bool gen_rri(DisasContext *s, arg_rri_sf *a,
4194 bool rd_sp, bool rn_sp, ArithTwoOp *fn)
4195 {
4196 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn);
4197 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd);
4198 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm);
4199
4200 fn(tcg_rd, tcg_rn, tcg_imm);
4201 if (!a->sf) {
4202 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4203 }
4204 return true;
4205 }
4206
4207 /*
4208 * PC-rel. addressing
4209 */
4210
trans_ADR(DisasContext * s,arg_ri * a)4211 static bool trans_ADR(DisasContext *s, arg_ri *a)
4212 {
4213 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm);
4214 return true;
4215 }
4216
trans_ADRP(DisasContext * s,arg_ri * a)4217 static bool trans_ADRP(DisasContext *s, arg_ri *a)
4218 {
4219 int64_t offset = (int64_t)a->imm << 12;
4220
4221 /* The page offset is ok for CF_PCREL. */
4222 offset -= s->pc_curr & 0xfff;
4223 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset);
4224 return true;
4225 }
4226
4227 /*
4228 * Add/subtract (immediate)
4229 */
4230 TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64)
4231 TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64)
4232 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4233 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4234
4235 /*
4236 * Add/subtract (immediate, with tags)
4237 */
4238
gen_add_sub_imm_with_tags(DisasContext * s,arg_rri_tag * a,bool sub_op)4239 static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a,
4240 bool sub_op)
4241 {
4242 TCGv_i64 tcg_rn, tcg_rd;
4243 int imm;
4244
4245 imm = a->uimm6 << LOG2_TAG_GRANULE;
4246 if (sub_op) {
4247 imm = -imm;
4248 }
4249
4250 tcg_rn = cpu_reg_sp(s, a->rn);
4251 tcg_rd = cpu_reg_sp(s, a->rd);
4252
4253 if (s->ata[0]) {
4254 gen_helper_addsubg(tcg_rd, tcg_env, tcg_rn,
4255 tcg_constant_i32(imm),
4256 tcg_constant_i32(a->uimm4));
4257 } else {
4258 tcg_gen_addi_i64(tcg_rd, tcg_rn, imm);
4259 gen_address_with_allocation_tag0(tcg_rd, tcg_rd);
4260 }
4261 return true;
4262 }
4263
TRANS_FEAT(ADDG_i,aa64_mte_insn_reg,gen_add_sub_imm_with_tags,a,false)4264 TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false)
4265 TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true)
4266
4267 /* The input should be a value in the bottom e bits (with higher
4268 * bits zero); returns that value replicated into every element
4269 * of size e in a 64 bit integer.
4270 */
4271 static uint64_t bitfield_replicate(uint64_t mask, unsigned int e)
4272 {
4273 assert(e != 0);
4274 while (e < 64) {
4275 mask |= mask << e;
4276 e *= 2;
4277 }
4278 return mask;
4279 }
4280
4281 /*
4282 * Logical (immediate)
4283 */
4284
4285 /*
4286 * Simplified variant of pseudocode DecodeBitMasks() for the case where we
4287 * only require the wmask. Returns false if the imms/immr/immn are a reserved
4288 * value (ie should cause a guest UNDEF exception), and true if they are
4289 * valid, in which case the decoded bit pattern is written to result.
4290 */
logic_imm_decode_wmask(uint64_t * result,unsigned int immn,unsigned int imms,unsigned int immr)4291 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
4292 unsigned int imms, unsigned int immr)
4293 {
4294 uint64_t mask;
4295 unsigned e, levels, s, r;
4296 int len;
4297
4298 assert(immn < 2 && imms < 64 && immr < 64);
4299
4300 /* The bit patterns we create here are 64 bit patterns which
4301 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or
4302 * 64 bits each. Each element contains the same value: a run
4303 * of between 1 and e-1 non-zero bits, rotated within the
4304 * element by between 0 and e-1 bits.
4305 *
4306 * The element size and run length are encoded into immn (1 bit)
4307 * and imms (6 bits) as follows:
4308 * 64 bit elements: immn = 1, imms = <length of run - 1>
4309 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1>
4310 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1>
4311 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1>
4312 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1>
4313 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1>
4314 * Notice that immn = 0, imms = 11111x is the only combination
4315 * not covered by one of the above options; this is reserved.
4316 * Further, <length of run - 1> all-ones is a reserved pattern.
4317 *
4318 * In all cases the rotation is by immr % e (and immr is 6 bits).
4319 */
4320
4321 /* First determine the element size */
4322 len = 31 - clz32((immn << 6) | (~imms & 0x3f));
4323 if (len < 1) {
4324 /* This is the immn == 0, imms == 0x11111x case */
4325 return false;
4326 }
4327 e = 1 << len;
4328
4329 levels = e - 1;
4330 s = imms & levels;
4331 r = immr & levels;
4332
4333 if (s == levels) {
4334 /* <length of run - 1> mustn't be all-ones. */
4335 return false;
4336 }
4337
4338 /* Create the value of one element: s+1 set bits rotated
4339 * by r within the element (which is e bits wide)...
4340 */
4341 mask = MAKE_64BIT_MASK(0, s + 1);
4342 if (r) {
4343 mask = (mask >> r) | (mask << (e - r));
4344 mask &= MAKE_64BIT_MASK(0, e);
4345 }
4346 /* ...then replicate the element over the whole 64 bit value */
4347 mask = bitfield_replicate(mask, e);
4348 *result = mask;
4349 return true;
4350 }
4351
gen_rri_log(DisasContext * s,arg_rri_log * a,bool set_cc,void (* fn)(TCGv_i64,TCGv_i64,int64_t))4352 static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc,
4353 void (*fn)(TCGv_i64, TCGv_i64, int64_t))
4354 {
4355 TCGv_i64 tcg_rd, tcg_rn;
4356 uint64_t imm;
4357
4358 /* Some immediate field values are reserved. */
4359 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
4360 extract32(a->dbm, 0, 6),
4361 extract32(a->dbm, 6, 6))) {
4362 return false;
4363 }
4364 if (!a->sf) {
4365 imm &= 0xffffffffull;
4366 }
4367
4368 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd);
4369 tcg_rn = cpu_reg(s, a->rn);
4370
4371 fn(tcg_rd, tcg_rn, imm);
4372 if (set_cc) {
4373 gen_logic_CC(a->sf, tcg_rd);
4374 }
4375 if (!a->sf) {
4376 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4377 }
4378 return true;
4379 }
4380
TRANS(AND_i,gen_rri_log,a,false,tcg_gen_andi_i64)4381 TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64)
4382 TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64)
4383 TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64)
4384 TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64)
4385
4386 /*
4387 * Move wide (immediate)
4388 */
4389
4390 static bool trans_MOVZ(DisasContext *s, arg_movw *a)
4391 {
4392 int pos = a->hw << 4;
4393 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos);
4394 return true;
4395 }
4396
trans_MOVN(DisasContext * s,arg_movw * a)4397 static bool trans_MOVN(DisasContext *s, arg_movw *a)
4398 {
4399 int pos = a->hw << 4;
4400 uint64_t imm = a->imm;
4401
4402 imm = ~(imm << pos);
4403 if (!a->sf) {
4404 imm = (uint32_t)imm;
4405 }
4406 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm);
4407 return true;
4408 }
4409
trans_MOVK(DisasContext * s,arg_movw * a)4410 static bool trans_MOVK(DisasContext *s, arg_movw *a)
4411 {
4412 int pos = a->hw << 4;
4413 TCGv_i64 tcg_rd, tcg_im;
4414
4415 tcg_rd = cpu_reg(s, a->rd);
4416 tcg_im = tcg_constant_i64(a->imm);
4417 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16);
4418 if (!a->sf) {
4419 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4420 }
4421 return true;
4422 }
4423
4424 /*
4425 * Bitfield
4426 */
4427
trans_SBFM(DisasContext * s,arg_SBFM * a)4428 static bool trans_SBFM(DisasContext *s, arg_SBFM *a)
4429 {
4430 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4431 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4432 unsigned int bitsize = a->sf ? 64 : 32;
4433 unsigned int ri = a->immr;
4434 unsigned int si = a->imms;
4435 unsigned int pos, len;
4436
4437 if (si >= ri) {
4438 /* Wd<s-r:0> = Wn<s:r> */
4439 len = (si - ri) + 1;
4440 tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len);
4441 if (!a->sf) {
4442 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4443 }
4444 } else {
4445 /* Wd<32+s-r,32-r> = Wn<s:0> */
4446 len = si + 1;
4447 pos = (bitsize - ri) & (bitsize - 1);
4448
4449 if (len < ri) {
4450 /*
4451 * Sign extend the destination field from len to fill the
4452 * balance of the word. Let the deposit below insert all
4453 * of those sign bits.
4454 */
4455 tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len);
4456 len = ri;
4457 }
4458
4459 /*
4460 * We start with zero, and we haven't modified any bits outside
4461 * bitsize, therefore no final zero-extension is unneeded for !sf.
4462 */
4463 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4464 }
4465 return true;
4466 }
4467
trans_UBFM(DisasContext * s,arg_UBFM * a)4468 static bool trans_UBFM(DisasContext *s, arg_UBFM *a)
4469 {
4470 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4471 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4472 unsigned int bitsize = a->sf ? 64 : 32;
4473 unsigned int ri = a->immr;
4474 unsigned int si = a->imms;
4475 unsigned int pos, len;
4476
4477 tcg_rd = cpu_reg(s, a->rd);
4478 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4479
4480 if (si >= ri) {
4481 /* Wd<s-r:0> = Wn<s:r> */
4482 len = (si - ri) + 1;
4483 tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len);
4484 } else {
4485 /* Wd<32+s-r,32-r> = Wn<s:0> */
4486 len = si + 1;
4487 pos = (bitsize - ri) & (bitsize - 1);
4488 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4489 }
4490 return true;
4491 }
4492
trans_BFM(DisasContext * s,arg_BFM * a)4493 static bool trans_BFM(DisasContext *s, arg_BFM *a)
4494 {
4495 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4496 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4497 unsigned int bitsize = a->sf ? 64 : 32;
4498 unsigned int ri = a->immr;
4499 unsigned int si = a->imms;
4500 unsigned int pos, len;
4501
4502 tcg_rd = cpu_reg(s, a->rd);
4503 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4504
4505 if (si >= ri) {
4506 /* Wd<s-r:0> = Wn<s:r> */
4507 tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri);
4508 len = (si - ri) + 1;
4509 pos = 0;
4510 } else {
4511 /* Wd<32+s-r,32-r> = Wn<s:0> */
4512 len = si + 1;
4513 pos = (bitsize - ri) & (bitsize - 1);
4514 }
4515
4516 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len);
4517 if (!a->sf) {
4518 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4519 }
4520 return true;
4521 }
4522
trans_EXTR(DisasContext * s,arg_extract * a)4523 static bool trans_EXTR(DisasContext *s, arg_extract *a)
4524 {
4525 TCGv_i64 tcg_rd, tcg_rm, tcg_rn;
4526
4527 tcg_rd = cpu_reg(s, a->rd);
4528
4529 if (unlikely(a->imm == 0)) {
4530 /*
4531 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
4532 * so an extract from bit 0 is a special case.
4533 */
4534 if (a->sf) {
4535 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm));
4536 } else {
4537 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm));
4538 }
4539 } else {
4540 tcg_rm = cpu_reg(s, a->rm);
4541 tcg_rn = cpu_reg(s, a->rn);
4542
4543 if (a->sf) {
4544 /* Specialization to ROR happens in EXTRACT2. */
4545 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm);
4546 } else {
4547 TCGv_i32 t0 = tcg_temp_new_i32();
4548
4549 tcg_gen_extrl_i64_i32(t0, tcg_rm);
4550 if (a->rm == a->rn) {
4551 tcg_gen_rotri_i32(t0, t0, a->imm);
4552 } else {
4553 TCGv_i32 t1 = tcg_temp_new_i32();
4554 tcg_gen_extrl_i64_i32(t1, tcg_rn);
4555 tcg_gen_extract2_i32(t0, t0, t1, a->imm);
4556 }
4557 tcg_gen_extu_i32_i64(tcg_rd, t0);
4558 }
4559 }
4560 return true;
4561 }
4562
4563 /* Shift a TCGv src by TCGv shift_amount, put result in dst.
4564 * Note that it is the caller's responsibility to ensure that the
4565 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM
4566 * mandated semantics for out of range shifts.
4567 */
shift_reg(TCGv_i64 dst,TCGv_i64 src,int sf,enum a64_shift_type shift_type,TCGv_i64 shift_amount)4568 static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
4569 enum a64_shift_type shift_type, TCGv_i64 shift_amount)
4570 {
4571 switch (shift_type) {
4572 case A64_SHIFT_TYPE_LSL:
4573 tcg_gen_shl_i64(dst, src, shift_amount);
4574 break;
4575 case A64_SHIFT_TYPE_LSR:
4576 tcg_gen_shr_i64(dst, src, shift_amount);
4577 break;
4578 case A64_SHIFT_TYPE_ASR:
4579 if (!sf) {
4580 tcg_gen_ext32s_i64(dst, src);
4581 }
4582 tcg_gen_sar_i64(dst, sf ? src : dst, shift_amount);
4583 break;
4584 case A64_SHIFT_TYPE_ROR:
4585 if (sf) {
4586 tcg_gen_rotr_i64(dst, src, shift_amount);
4587 } else {
4588 TCGv_i32 t0, t1;
4589 t0 = tcg_temp_new_i32();
4590 t1 = tcg_temp_new_i32();
4591 tcg_gen_extrl_i64_i32(t0, src);
4592 tcg_gen_extrl_i64_i32(t1, shift_amount);
4593 tcg_gen_rotr_i32(t0, t0, t1);
4594 tcg_gen_extu_i32_i64(dst, t0);
4595 }
4596 break;
4597 default:
4598 assert(FALSE); /* all shift types should be handled */
4599 break;
4600 }
4601
4602 if (!sf) { /* zero extend final result */
4603 tcg_gen_ext32u_i64(dst, dst);
4604 }
4605 }
4606
4607 /* Shift a TCGv src by immediate, put result in dst.
4608 * The shift amount must be in range (this should always be true as the
4609 * relevant instructions will UNDEF on bad shift immediates).
4610 */
shift_reg_imm(TCGv_i64 dst,TCGv_i64 src,int sf,enum a64_shift_type shift_type,unsigned int shift_i)4611 static void shift_reg_imm(TCGv_i64 dst, TCGv_i64 src, int sf,
4612 enum a64_shift_type shift_type, unsigned int shift_i)
4613 {
4614 assert(shift_i < (sf ? 64 : 32));
4615
4616 if (shift_i == 0) {
4617 tcg_gen_mov_i64(dst, src);
4618 } else {
4619 shift_reg(dst, src, sf, shift_type, tcg_constant_i64(shift_i));
4620 }
4621 }
4622
4623 /* Logical (shifted register)
4624 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
4625 * +----+-----+-----------+-------+---+------+--------+------+------+
4626 * | sf | opc | 0 1 0 1 0 | shift | N | Rm | imm6 | Rn | Rd |
4627 * +----+-----+-----------+-------+---+------+--------+------+------+
4628 */
disas_logic_reg(DisasContext * s,uint32_t insn)4629 static void disas_logic_reg(DisasContext *s, uint32_t insn)
4630 {
4631 TCGv_i64 tcg_rd, tcg_rn, tcg_rm;
4632 unsigned int sf, opc, shift_type, invert, rm, shift_amount, rn, rd;
4633
4634 sf = extract32(insn, 31, 1);
4635 opc = extract32(insn, 29, 2);
4636 shift_type = extract32(insn, 22, 2);
4637 invert = extract32(insn, 21, 1);
4638 rm = extract32(insn, 16, 5);
4639 shift_amount = extract32(insn, 10, 6);
4640 rn = extract32(insn, 5, 5);
4641 rd = extract32(insn, 0, 5);
4642
4643 if (!sf && (shift_amount & (1 << 5))) {
4644 unallocated_encoding(s);
4645 return;
4646 }
4647
4648 tcg_rd = cpu_reg(s, rd);
4649
4650 if (opc == 1 && shift_amount == 0 && shift_type == 0 && rn == 31) {
4651 /* Unshifted ORR and ORN with WZR/XZR is the standard encoding for
4652 * register-register MOV and MVN, so it is worth special casing.
4653 */
4654 tcg_rm = cpu_reg(s, rm);
4655 if (invert) {
4656 tcg_gen_not_i64(tcg_rd, tcg_rm);
4657 if (!sf) {
4658 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4659 }
4660 } else {
4661 if (sf) {
4662 tcg_gen_mov_i64(tcg_rd, tcg_rm);
4663 } else {
4664 tcg_gen_ext32u_i64(tcg_rd, tcg_rm);
4665 }
4666 }
4667 return;
4668 }
4669
4670 tcg_rm = read_cpu_reg(s, rm, sf);
4671
4672 if (shift_amount) {
4673 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, shift_amount);
4674 }
4675
4676 tcg_rn = cpu_reg(s, rn);
4677
4678 switch (opc | (invert << 2)) {
4679 case 0: /* AND */
4680 case 3: /* ANDS */
4681 tcg_gen_and_i64(tcg_rd, tcg_rn, tcg_rm);
4682 break;
4683 case 1: /* ORR */
4684 tcg_gen_or_i64(tcg_rd, tcg_rn, tcg_rm);
4685 break;
4686 case 2: /* EOR */
4687 tcg_gen_xor_i64(tcg_rd, tcg_rn, tcg_rm);
4688 break;
4689 case 4: /* BIC */
4690 case 7: /* BICS */
4691 tcg_gen_andc_i64(tcg_rd, tcg_rn, tcg_rm);
4692 break;
4693 case 5: /* ORN */
4694 tcg_gen_orc_i64(tcg_rd, tcg_rn, tcg_rm);
4695 break;
4696 case 6: /* EON */
4697 tcg_gen_eqv_i64(tcg_rd, tcg_rn, tcg_rm);
4698 break;
4699 default:
4700 assert(FALSE);
4701 break;
4702 }
4703
4704 if (!sf) {
4705 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4706 }
4707
4708 if (opc == 3) {
4709 gen_logic_CC(sf, tcg_rd);
4710 }
4711 }
4712
4713 /*
4714 * Add/subtract (extended register)
4715 *
4716 * 31|30|29|28 24|23 22|21|20 16|15 13|12 10|9 5|4 0|
4717 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
4718 * |sf|op| S| 0 1 0 1 1 | opt | 1| Rm |option| imm3 | Rn | Rd |
4719 * +--+--+--+-----------+-----+--+-------+------+------+----+----+
4720 *
4721 * sf: 0 -> 32bit, 1 -> 64bit
4722 * op: 0 -> add , 1 -> sub
4723 * S: 1 -> set flags
4724 * opt: 00
4725 * option: extension type (see DecodeRegExtend)
4726 * imm3: optional shift to Rm
4727 *
4728 * Rd = Rn + LSL(extend(Rm), amount)
4729 */
disas_add_sub_ext_reg(DisasContext * s,uint32_t insn)4730 static void disas_add_sub_ext_reg(DisasContext *s, uint32_t insn)
4731 {
4732 int rd = extract32(insn, 0, 5);
4733 int rn = extract32(insn, 5, 5);
4734 int imm3 = extract32(insn, 10, 3);
4735 int option = extract32(insn, 13, 3);
4736 int rm = extract32(insn, 16, 5);
4737 int opt = extract32(insn, 22, 2);
4738 bool setflags = extract32(insn, 29, 1);
4739 bool sub_op = extract32(insn, 30, 1);
4740 bool sf = extract32(insn, 31, 1);
4741
4742 TCGv_i64 tcg_rm, tcg_rn; /* temps */
4743 TCGv_i64 tcg_rd;
4744 TCGv_i64 tcg_result;
4745
4746 if (imm3 > 4 || opt != 0) {
4747 unallocated_encoding(s);
4748 return;
4749 }
4750
4751 /* non-flag setting ops may use SP */
4752 if (!setflags) {
4753 tcg_rd = cpu_reg_sp(s, rd);
4754 } else {
4755 tcg_rd = cpu_reg(s, rd);
4756 }
4757 tcg_rn = read_cpu_reg_sp(s, rn, sf);
4758
4759 tcg_rm = read_cpu_reg(s, rm, sf);
4760 ext_and_shift_reg(tcg_rm, tcg_rm, option, imm3);
4761
4762 tcg_result = tcg_temp_new_i64();
4763
4764 if (!setflags) {
4765 if (sub_op) {
4766 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
4767 } else {
4768 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
4769 }
4770 } else {
4771 if (sub_op) {
4772 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
4773 } else {
4774 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
4775 }
4776 }
4777
4778 if (sf) {
4779 tcg_gen_mov_i64(tcg_rd, tcg_result);
4780 } else {
4781 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
4782 }
4783 }
4784
4785 /*
4786 * Add/subtract (shifted register)
4787 *
4788 * 31 30 29 28 24 23 22 21 20 16 15 10 9 5 4 0
4789 * +--+--+--+-----------+-----+--+-------+---------+------+------+
4790 * |sf|op| S| 0 1 0 1 1 |shift| 0| Rm | imm6 | Rn | Rd |
4791 * +--+--+--+-----------+-----+--+-------+---------+------+------+
4792 *
4793 * sf: 0 -> 32bit, 1 -> 64bit
4794 * op: 0 -> add , 1 -> sub
4795 * S: 1 -> set flags
4796 * shift: 00 -> LSL, 01 -> LSR, 10 -> ASR, 11 -> RESERVED
4797 * imm6: Shift amount to apply to Rm before the add/sub
4798 */
disas_add_sub_reg(DisasContext * s,uint32_t insn)4799 static void disas_add_sub_reg(DisasContext *s, uint32_t insn)
4800 {
4801 int rd = extract32(insn, 0, 5);
4802 int rn = extract32(insn, 5, 5);
4803 int imm6 = extract32(insn, 10, 6);
4804 int rm = extract32(insn, 16, 5);
4805 int shift_type = extract32(insn, 22, 2);
4806 bool setflags = extract32(insn, 29, 1);
4807 bool sub_op = extract32(insn, 30, 1);
4808 bool sf = extract32(insn, 31, 1);
4809
4810 TCGv_i64 tcg_rd = cpu_reg(s, rd);
4811 TCGv_i64 tcg_rn, tcg_rm;
4812 TCGv_i64 tcg_result;
4813
4814 if ((shift_type == 3) || (!sf && (imm6 > 31))) {
4815 unallocated_encoding(s);
4816 return;
4817 }
4818
4819 tcg_rn = read_cpu_reg(s, rn, sf);
4820 tcg_rm = read_cpu_reg(s, rm, sf);
4821
4822 shift_reg_imm(tcg_rm, tcg_rm, sf, shift_type, imm6);
4823
4824 tcg_result = tcg_temp_new_i64();
4825
4826 if (!setflags) {
4827 if (sub_op) {
4828 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
4829 } else {
4830 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
4831 }
4832 } else {
4833 if (sub_op) {
4834 gen_sub_CC(sf, tcg_result, tcg_rn, tcg_rm);
4835 } else {
4836 gen_add_CC(sf, tcg_result, tcg_rn, tcg_rm);
4837 }
4838 }
4839
4840 if (sf) {
4841 tcg_gen_mov_i64(tcg_rd, tcg_result);
4842 } else {
4843 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
4844 }
4845 }
4846
4847 /* Data-processing (3 source)
4848 *
4849 * 31 30 29 28 24 23 21 20 16 15 14 10 9 5 4 0
4850 * +--+------+-----------+------+------+----+------+------+------+
4851 * |sf| op54 | 1 1 0 1 1 | op31 | Rm | o0 | Ra | Rn | Rd |
4852 * +--+------+-----------+------+------+----+------+------+------+
4853 */
disas_data_proc_3src(DisasContext * s,uint32_t insn)4854 static void disas_data_proc_3src(DisasContext *s, uint32_t insn)
4855 {
4856 int rd = extract32(insn, 0, 5);
4857 int rn = extract32(insn, 5, 5);
4858 int ra = extract32(insn, 10, 5);
4859 int rm = extract32(insn, 16, 5);
4860 int op_id = (extract32(insn, 29, 3) << 4) |
4861 (extract32(insn, 21, 3) << 1) |
4862 extract32(insn, 15, 1);
4863 bool sf = extract32(insn, 31, 1);
4864 bool is_sub = extract32(op_id, 0, 1);
4865 bool is_high = extract32(op_id, 2, 1);
4866 bool is_signed = false;
4867 TCGv_i64 tcg_op1;
4868 TCGv_i64 tcg_op2;
4869 TCGv_i64 tcg_tmp;
4870
4871 /* Note that op_id is sf:op54:op31:o0 so it includes the 32/64 size flag */
4872 switch (op_id) {
4873 case 0x42: /* SMADDL */
4874 case 0x43: /* SMSUBL */
4875 case 0x44: /* SMULH */
4876 is_signed = true;
4877 break;
4878 case 0x0: /* MADD (32bit) */
4879 case 0x1: /* MSUB (32bit) */
4880 case 0x40: /* MADD (64bit) */
4881 case 0x41: /* MSUB (64bit) */
4882 case 0x4a: /* UMADDL */
4883 case 0x4b: /* UMSUBL */
4884 case 0x4c: /* UMULH */
4885 break;
4886 default:
4887 unallocated_encoding(s);
4888 return;
4889 }
4890
4891 if (is_high) {
4892 TCGv_i64 low_bits = tcg_temp_new_i64(); /* low bits discarded */
4893 TCGv_i64 tcg_rd = cpu_reg(s, rd);
4894 TCGv_i64 tcg_rn = cpu_reg(s, rn);
4895 TCGv_i64 tcg_rm = cpu_reg(s, rm);
4896
4897 if (is_signed) {
4898 tcg_gen_muls2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
4899 } else {
4900 tcg_gen_mulu2_i64(low_bits, tcg_rd, tcg_rn, tcg_rm);
4901 }
4902 return;
4903 }
4904
4905 tcg_op1 = tcg_temp_new_i64();
4906 tcg_op2 = tcg_temp_new_i64();
4907 tcg_tmp = tcg_temp_new_i64();
4908
4909 if (op_id < 0x42) {
4910 tcg_gen_mov_i64(tcg_op1, cpu_reg(s, rn));
4911 tcg_gen_mov_i64(tcg_op2, cpu_reg(s, rm));
4912 } else {
4913 if (is_signed) {
4914 tcg_gen_ext32s_i64(tcg_op1, cpu_reg(s, rn));
4915 tcg_gen_ext32s_i64(tcg_op2, cpu_reg(s, rm));
4916 } else {
4917 tcg_gen_ext32u_i64(tcg_op1, cpu_reg(s, rn));
4918 tcg_gen_ext32u_i64(tcg_op2, cpu_reg(s, rm));
4919 }
4920 }
4921
4922 if (ra == 31 && !is_sub) {
4923 /* Special-case MADD with rA == XZR; it is the standard MUL alias */
4924 tcg_gen_mul_i64(cpu_reg(s, rd), tcg_op1, tcg_op2);
4925 } else {
4926 tcg_gen_mul_i64(tcg_tmp, tcg_op1, tcg_op2);
4927 if (is_sub) {
4928 tcg_gen_sub_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
4929 } else {
4930 tcg_gen_add_i64(cpu_reg(s, rd), cpu_reg(s, ra), tcg_tmp);
4931 }
4932 }
4933
4934 if (!sf) {
4935 tcg_gen_ext32u_i64(cpu_reg(s, rd), cpu_reg(s, rd));
4936 }
4937 }
4938
4939 /* Add/subtract (with carry)
4940 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 10 9 5 4 0
4941 * +--+--+--+------------------------+------+-------------+------+-----+
4942 * |sf|op| S| 1 1 0 1 0 0 0 0 | rm | 0 0 0 0 0 0 | Rn | Rd |
4943 * +--+--+--+------------------------+------+-------------+------+-----+
4944 */
4945
disas_adc_sbc(DisasContext * s,uint32_t insn)4946 static void disas_adc_sbc(DisasContext *s, uint32_t insn)
4947 {
4948 unsigned int sf, op, setflags, rm, rn, rd;
4949 TCGv_i64 tcg_y, tcg_rn, tcg_rd;
4950
4951 sf = extract32(insn, 31, 1);
4952 op = extract32(insn, 30, 1);
4953 setflags = extract32(insn, 29, 1);
4954 rm = extract32(insn, 16, 5);
4955 rn = extract32(insn, 5, 5);
4956 rd = extract32(insn, 0, 5);
4957
4958 tcg_rd = cpu_reg(s, rd);
4959 tcg_rn = cpu_reg(s, rn);
4960
4961 if (op) {
4962 tcg_y = tcg_temp_new_i64();
4963 tcg_gen_not_i64(tcg_y, cpu_reg(s, rm));
4964 } else {
4965 tcg_y = cpu_reg(s, rm);
4966 }
4967
4968 if (setflags) {
4969 gen_adc_CC(sf, tcg_rd, tcg_rn, tcg_y);
4970 } else {
4971 gen_adc(sf, tcg_rd, tcg_rn, tcg_y);
4972 }
4973 }
4974
4975 /*
4976 * Rotate right into flags
4977 * 31 30 29 21 15 10 5 4 0
4978 * +--+--+--+-----------------+--------+-----------+------+--+------+
4979 * |sf|op| S| 1 1 0 1 0 0 0 0 | imm6 | 0 0 0 0 1 | Rn |o2| mask |
4980 * +--+--+--+-----------------+--------+-----------+------+--+------+
4981 */
disas_rotate_right_into_flags(DisasContext * s,uint32_t insn)4982 static void disas_rotate_right_into_flags(DisasContext *s, uint32_t insn)
4983 {
4984 int mask = extract32(insn, 0, 4);
4985 int o2 = extract32(insn, 4, 1);
4986 int rn = extract32(insn, 5, 5);
4987 int imm6 = extract32(insn, 15, 6);
4988 int sf_op_s = extract32(insn, 29, 3);
4989 TCGv_i64 tcg_rn;
4990 TCGv_i32 nzcv;
4991
4992 if (sf_op_s != 5 || o2 != 0 || !dc_isar_feature(aa64_condm_4, s)) {
4993 unallocated_encoding(s);
4994 return;
4995 }
4996
4997 tcg_rn = read_cpu_reg(s, rn, 1);
4998 tcg_gen_rotri_i64(tcg_rn, tcg_rn, imm6);
4999
5000 nzcv = tcg_temp_new_i32();
5001 tcg_gen_extrl_i64_i32(nzcv, tcg_rn);
5002
5003 if (mask & 8) { /* N */
5004 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3);
5005 }
5006 if (mask & 4) { /* Z */
5007 tcg_gen_not_i32(cpu_ZF, nzcv);
5008 tcg_gen_andi_i32(cpu_ZF, cpu_ZF, 4);
5009 }
5010 if (mask & 2) { /* C */
5011 tcg_gen_extract_i32(cpu_CF, nzcv, 1, 1);
5012 }
5013 if (mask & 1) { /* V */
5014 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0);
5015 }
5016 }
5017
5018 /*
5019 * Evaluate into flags
5020 * 31 30 29 21 15 14 10 5 4 0
5021 * +--+--+--+-----------------+---------+----+---------+------+--+------+
5022 * |sf|op| S| 1 1 0 1 0 0 0 0 | opcode2 | sz | 0 0 1 0 | Rn |o3| mask |
5023 * +--+--+--+-----------------+---------+----+---------+------+--+------+
5024 */
disas_evaluate_into_flags(DisasContext * s,uint32_t insn)5025 static void disas_evaluate_into_flags(DisasContext *s, uint32_t insn)
5026 {
5027 int o3_mask = extract32(insn, 0, 5);
5028 int rn = extract32(insn, 5, 5);
5029 int o2 = extract32(insn, 15, 6);
5030 int sz = extract32(insn, 14, 1);
5031 int sf_op_s = extract32(insn, 29, 3);
5032 TCGv_i32 tmp;
5033 int shift;
5034
5035 if (sf_op_s != 1 || o2 != 0 || o3_mask != 0xd ||
5036 !dc_isar_feature(aa64_condm_4, s)) {
5037 unallocated_encoding(s);
5038 return;
5039 }
5040 shift = sz ? 16 : 24; /* SETF16 or SETF8 */
5041
5042 tmp = tcg_temp_new_i32();
5043 tcg_gen_extrl_i64_i32(tmp, cpu_reg(s, rn));
5044 tcg_gen_shli_i32(cpu_NF, tmp, shift);
5045 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1);
5046 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
5047 tcg_gen_xor_i32(cpu_VF, cpu_VF, cpu_NF);
5048 }
5049
5050 /* Conditional compare (immediate / register)
5051 * 31 30 29 28 27 26 25 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
5052 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
5053 * |sf|op| S| 1 1 0 1 0 0 1 0 |imm5/rm | cond |i/r |o2| Rn |o3|nzcv |
5054 * +--+--+--+------------------------+--------+------+----+--+------+--+-----+
5055 * [1] y [0] [0]
5056 */
disas_cc(DisasContext * s,uint32_t insn)5057 static void disas_cc(DisasContext *s, uint32_t insn)
5058 {
5059 unsigned int sf, op, y, cond, rn, nzcv, is_imm;
5060 TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
5061 TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
5062 DisasCompare c;
5063
5064 if (!extract32(insn, 29, 1)) {
5065 unallocated_encoding(s);
5066 return;
5067 }
5068 if (insn & (1 << 10 | 1 << 4)) {
5069 unallocated_encoding(s);
5070 return;
5071 }
5072 sf = extract32(insn, 31, 1);
5073 op = extract32(insn, 30, 1);
5074 is_imm = extract32(insn, 11, 1);
5075 y = extract32(insn, 16, 5); /* y = rm (reg) or imm5 (imm) */
5076 cond = extract32(insn, 12, 4);
5077 rn = extract32(insn, 5, 5);
5078 nzcv = extract32(insn, 0, 4);
5079
5080 /* Set T0 = !COND. */
5081 tcg_t0 = tcg_temp_new_i32();
5082 arm_test_cc(&c, cond);
5083 tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
5084
5085 /* Load the arguments for the new comparison. */
5086 if (is_imm) {
5087 tcg_y = tcg_temp_new_i64();
5088 tcg_gen_movi_i64(tcg_y, y);
5089 } else {
5090 tcg_y = cpu_reg(s, y);
5091 }
5092 tcg_rn = cpu_reg(s, rn);
5093
5094 /* Set the flags for the new comparison. */
5095 tcg_tmp = tcg_temp_new_i64();
5096 if (op) {
5097 gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
5098 } else {
5099 gen_add_CC(sf, tcg_tmp, tcg_rn, tcg_y);
5100 }
5101
5102 /* If COND was false, force the flags to #nzcv. Compute two masks
5103 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
5104 * For tcg hosts that support ANDC, we can make do with just T1.
5105 * In either case, allow the tcg optimizer to delete any unused mask.
5106 */
5107 tcg_t1 = tcg_temp_new_i32();
5108 tcg_t2 = tcg_temp_new_i32();
5109 tcg_gen_neg_i32(tcg_t1, tcg_t0);
5110 tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
5111
5112 if (nzcv & 8) { /* N */
5113 tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
5114 } else {
5115 if (TCG_TARGET_HAS_andc_i32) {
5116 tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
5117 } else {
5118 tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
5119 }
5120 }
5121 if (nzcv & 4) { /* Z */
5122 if (TCG_TARGET_HAS_andc_i32) {
5123 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
5124 } else {
5125 tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
5126 }
5127 } else {
5128 tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
5129 }
5130 if (nzcv & 2) { /* C */
5131 tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
5132 } else {
5133 if (TCG_TARGET_HAS_andc_i32) {
5134 tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
5135 } else {
5136 tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
5137 }
5138 }
5139 if (nzcv & 1) { /* V */
5140 tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
5141 } else {
5142 if (TCG_TARGET_HAS_andc_i32) {
5143 tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
5144 } else {
5145 tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
5146 }
5147 }
5148 }
5149
5150 /* Conditional select
5151 * 31 30 29 28 21 20 16 15 12 11 10 9 5 4 0
5152 * +----+----+---+-----------------+------+------+-----+------+------+
5153 * | sf | op | S | 1 1 0 1 0 1 0 0 | Rm | cond | op2 | Rn | Rd |
5154 * +----+----+---+-----------------+------+------+-----+------+------+
5155 */
disas_cond_select(DisasContext * s,uint32_t insn)5156 static void disas_cond_select(DisasContext *s, uint32_t insn)
5157 {
5158 unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
5159 TCGv_i64 tcg_rd, zero;
5160 DisasCompare64 c;
5161
5162 if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
5163 /* S == 1 or op2<1> == 1 */
5164 unallocated_encoding(s);
5165 return;
5166 }
5167 sf = extract32(insn, 31, 1);
5168 else_inv = extract32(insn, 30, 1);
5169 rm = extract32(insn, 16, 5);
5170 cond = extract32(insn, 12, 4);
5171 else_inc = extract32(insn, 10, 1);
5172 rn = extract32(insn, 5, 5);
5173 rd = extract32(insn, 0, 5);
5174
5175 tcg_rd = cpu_reg(s, rd);
5176
5177 a64_test_cc(&c, cond);
5178 zero = tcg_constant_i64(0);
5179
5180 if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
5181 /* CSET & CSETM. */
5182 if (else_inv) {
5183 tcg_gen_negsetcond_i64(tcg_invert_cond(c.cond),
5184 tcg_rd, c.value, zero);
5185 } else {
5186 tcg_gen_setcond_i64(tcg_invert_cond(c.cond),
5187 tcg_rd, c.value, zero);
5188 }
5189 } else {
5190 TCGv_i64 t_true = cpu_reg(s, rn);
5191 TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
5192 if (else_inv && else_inc) {
5193 tcg_gen_neg_i64(t_false, t_false);
5194 } else if (else_inv) {
5195 tcg_gen_not_i64(t_false, t_false);
5196 } else if (else_inc) {
5197 tcg_gen_addi_i64(t_false, t_false, 1);
5198 }
5199 tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
5200 }
5201
5202 if (!sf) {
5203 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
5204 }
5205 }
5206
handle_clz(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5207 static void handle_clz(DisasContext *s, unsigned int sf,
5208 unsigned int rn, unsigned int rd)
5209 {
5210 TCGv_i64 tcg_rd, tcg_rn;
5211 tcg_rd = cpu_reg(s, rd);
5212 tcg_rn = cpu_reg(s, rn);
5213
5214 if (sf) {
5215 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
5216 } else {
5217 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5218 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5219 tcg_gen_clzi_i32(tcg_tmp32, tcg_tmp32, 32);
5220 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5221 }
5222 }
5223
handle_cls(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5224 static void handle_cls(DisasContext *s, unsigned int sf,
5225 unsigned int rn, unsigned int rd)
5226 {
5227 TCGv_i64 tcg_rd, tcg_rn;
5228 tcg_rd = cpu_reg(s, rd);
5229 tcg_rn = cpu_reg(s, rn);
5230
5231 if (sf) {
5232 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
5233 } else {
5234 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5235 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5236 tcg_gen_clrsb_i32(tcg_tmp32, tcg_tmp32);
5237 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5238 }
5239 }
5240
handle_rbit(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5241 static void handle_rbit(DisasContext *s, unsigned int sf,
5242 unsigned int rn, unsigned int rd)
5243 {
5244 TCGv_i64 tcg_rd, tcg_rn;
5245 tcg_rd = cpu_reg(s, rd);
5246 tcg_rn = cpu_reg(s, rn);
5247
5248 if (sf) {
5249 gen_helper_rbit64(tcg_rd, tcg_rn);
5250 } else {
5251 TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
5252 tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
5253 gen_helper_rbit(tcg_tmp32, tcg_tmp32);
5254 tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
5255 }
5256 }
5257
5258 /* REV with sf==1, opcode==3 ("REV64") */
handle_rev64(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5259 static void handle_rev64(DisasContext *s, unsigned int sf,
5260 unsigned int rn, unsigned int rd)
5261 {
5262 if (!sf) {
5263 unallocated_encoding(s);
5264 return;
5265 }
5266 tcg_gen_bswap64_i64(cpu_reg(s, rd), cpu_reg(s, rn));
5267 }
5268
5269 /* REV with sf==0, opcode==2
5270 * REV32 (sf==1, opcode==2)
5271 */
handle_rev32(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5272 static void handle_rev32(DisasContext *s, unsigned int sf,
5273 unsigned int rn, unsigned int rd)
5274 {
5275 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5276 TCGv_i64 tcg_rn = cpu_reg(s, rn);
5277
5278 if (sf) {
5279 tcg_gen_bswap64_i64(tcg_rd, tcg_rn);
5280 tcg_gen_rotri_i64(tcg_rd, tcg_rd, 32);
5281 } else {
5282 tcg_gen_bswap32_i64(tcg_rd, tcg_rn, TCG_BSWAP_OZ);
5283 }
5284 }
5285
5286 /* REV16 (opcode==1) */
handle_rev16(DisasContext * s,unsigned int sf,unsigned int rn,unsigned int rd)5287 static void handle_rev16(DisasContext *s, unsigned int sf,
5288 unsigned int rn, unsigned int rd)
5289 {
5290 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5291 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
5292 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
5293 TCGv_i64 mask = tcg_constant_i64(sf ? 0x00ff00ff00ff00ffull : 0x00ff00ff);
5294
5295 tcg_gen_shri_i64(tcg_tmp, tcg_rn, 8);
5296 tcg_gen_and_i64(tcg_rd, tcg_rn, mask);
5297 tcg_gen_and_i64(tcg_tmp, tcg_tmp, mask);
5298 tcg_gen_shli_i64(tcg_rd, tcg_rd, 8);
5299 tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_tmp);
5300 }
5301
5302 /* Data-processing (1 source)
5303 * 31 30 29 28 21 20 16 15 10 9 5 4 0
5304 * +----+---+---+-----------------+---------+--------+------+------+
5305 * | sf | 1 | S | 1 1 0 1 0 1 1 0 | opcode2 | opcode | Rn | Rd |
5306 * +----+---+---+-----------------+---------+--------+------+------+
5307 */
disas_data_proc_1src(DisasContext * s,uint32_t insn)5308 static void disas_data_proc_1src(DisasContext *s, uint32_t insn)
5309 {
5310 unsigned int sf, opcode, opcode2, rn, rd;
5311 TCGv_i64 tcg_rd;
5312
5313 if (extract32(insn, 29, 1)) {
5314 unallocated_encoding(s);
5315 return;
5316 }
5317
5318 sf = extract32(insn, 31, 1);
5319 opcode = extract32(insn, 10, 6);
5320 opcode2 = extract32(insn, 16, 5);
5321 rn = extract32(insn, 5, 5);
5322 rd = extract32(insn, 0, 5);
5323
5324 #define MAP(SF, O2, O1) ((SF) | (O1 << 1) | (O2 << 7))
5325
5326 switch (MAP(sf, opcode2, opcode)) {
5327 case MAP(0, 0x00, 0x00): /* RBIT */
5328 case MAP(1, 0x00, 0x00):
5329 handle_rbit(s, sf, rn, rd);
5330 break;
5331 case MAP(0, 0x00, 0x01): /* REV16 */
5332 case MAP(1, 0x00, 0x01):
5333 handle_rev16(s, sf, rn, rd);
5334 break;
5335 case MAP(0, 0x00, 0x02): /* REV/REV32 */
5336 case MAP(1, 0x00, 0x02):
5337 handle_rev32(s, sf, rn, rd);
5338 break;
5339 case MAP(1, 0x00, 0x03): /* REV64 */
5340 handle_rev64(s, sf, rn, rd);
5341 break;
5342 case MAP(0, 0x00, 0x04): /* CLZ */
5343 case MAP(1, 0x00, 0x04):
5344 handle_clz(s, sf, rn, rd);
5345 break;
5346 case MAP(0, 0x00, 0x05): /* CLS */
5347 case MAP(1, 0x00, 0x05):
5348 handle_cls(s, sf, rn, rd);
5349 break;
5350 case MAP(1, 0x01, 0x00): /* PACIA */
5351 if (s->pauth_active) {
5352 tcg_rd = cpu_reg(s, rd);
5353 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5354 } else if (!dc_isar_feature(aa64_pauth, s)) {
5355 goto do_unallocated;
5356 }
5357 break;
5358 case MAP(1, 0x01, 0x01): /* PACIB */
5359 if (s->pauth_active) {
5360 tcg_rd = cpu_reg(s, rd);
5361 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5362 } else if (!dc_isar_feature(aa64_pauth, s)) {
5363 goto do_unallocated;
5364 }
5365 break;
5366 case MAP(1, 0x01, 0x02): /* PACDA */
5367 if (s->pauth_active) {
5368 tcg_rd = cpu_reg(s, rd);
5369 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5370 } else if (!dc_isar_feature(aa64_pauth, s)) {
5371 goto do_unallocated;
5372 }
5373 break;
5374 case MAP(1, 0x01, 0x03): /* PACDB */
5375 if (s->pauth_active) {
5376 tcg_rd = cpu_reg(s, rd);
5377 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5378 } else if (!dc_isar_feature(aa64_pauth, s)) {
5379 goto do_unallocated;
5380 }
5381 break;
5382 case MAP(1, 0x01, 0x04): /* AUTIA */
5383 if (s->pauth_active) {
5384 tcg_rd = cpu_reg(s, rd);
5385 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5386 } else if (!dc_isar_feature(aa64_pauth, s)) {
5387 goto do_unallocated;
5388 }
5389 break;
5390 case MAP(1, 0x01, 0x05): /* AUTIB */
5391 if (s->pauth_active) {
5392 tcg_rd = cpu_reg(s, rd);
5393 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5394 } else if (!dc_isar_feature(aa64_pauth, s)) {
5395 goto do_unallocated;
5396 }
5397 break;
5398 case MAP(1, 0x01, 0x06): /* AUTDA */
5399 if (s->pauth_active) {
5400 tcg_rd = cpu_reg(s, rd);
5401 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5402 } else if (!dc_isar_feature(aa64_pauth, s)) {
5403 goto do_unallocated;
5404 }
5405 break;
5406 case MAP(1, 0x01, 0x07): /* AUTDB */
5407 if (s->pauth_active) {
5408 tcg_rd = cpu_reg(s, rd);
5409 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, cpu_reg_sp(s, rn));
5410 } else if (!dc_isar_feature(aa64_pauth, s)) {
5411 goto do_unallocated;
5412 }
5413 break;
5414 case MAP(1, 0x01, 0x08): /* PACIZA */
5415 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5416 goto do_unallocated;
5417 } else if (s->pauth_active) {
5418 tcg_rd = cpu_reg(s, rd);
5419 gen_helper_pacia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5420 }
5421 break;
5422 case MAP(1, 0x01, 0x09): /* PACIZB */
5423 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5424 goto do_unallocated;
5425 } else if (s->pauth_active) {
5426 tcg_rd = cpu_reg(s, rd);
5427 gen_helper_pacib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5428 }
5429 break;
5430 case MAP(1, 0x01, 0x0a): /* PACDZA */
5431 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5432 goto do_unallocated;
5433 } else if (s->pauth_active) {
5434 tcg_rd = cpu_reg(s, rd);
5435 gen_helper_pacda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5436 }
5437 break;
5438 case MAP(1, 0x01, 0x0b): /* PACDZB */
5439 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5440 goto do_unallocated;
5441 } else if (s->pauth_active) {
5442 tcg_rd = cpu_reg(s, rd);
5443 gen_helper_pacdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5444 }
5445 break;
5446 case MAP(1, 0x01, 0x0c): /* AUTIZA */
5447 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5448 goto do_unallocated;
5449 } else if (s->pauth_active) {
5450 tcg_rd = cpu_reg(s, rd);
5451 gen_helper_autia(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5452 }
5453 break;
5454 case MAP(1, 0x01, 0x0d): /* AUTIZB */
5455 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5456 goto do_unallocated;
5457 } else if (s->pauth_active) {
5458 tcg_rd = cpu_reg(s, rd);
5459 gen_helper_autib(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5460 }
5461 break;
5462 case MAP(1, 0x01, 0x0e): /* AUTDZA */
5463 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5464 goto do_unallocated;
5465 } else if (s->pauth_active) {
5466 tcg_rd = cpu_reg(s, rd);
5467 gen_helper_autda(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5468 }
5469 break;
5470 case MAP(1, 0x01, 0x0f): /* AUTDZB */
5471 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5472 goto do_unallocated;
5473 } else if (s->pauth_active) {
5474 tcg_rd = cpu_reg(s, rd);
5475 gen_helper_autdb(tcg_rd, tcg_env, tcg_rd, tcg_constant_i64(0));
5476 }
5477 break;
5478 case MAP(1, 0x01, 0x10): /* XPACI */
5479 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5480 goto do_unallocated;
5481 } else if (s->pauth_active) {
5482 tcg_rd = cpu_reg(s, rd);
5483 gen_helper_xpaci(tcg_rd, tcg_env, tcg_rd);
5484 }
5485 break;
5486 case MAP(1, 0x01, 0x11): /* XPACD */
5487 if (!dc_isar_feature(aa64_pauth, s) || rn != 31) {
5488 goto do_unallocated;
5489 } else if (s->pauth_active) {
5490 tcg_rd = cpu_reg(s, rd);
5491 gen_helper_xpacd(tcg_rd, tcg_env, tcg_rd);
5492 }
5493 break;
5494 default:
5495 do_unallocated:
5496 unallocated_encoding(s);
5497 break;
5498 }
5499
5500 #undef MAP
5501 }
5502
handle_div(DisasContext * s,bool is_signed,unsigned int sf,unsigned int rm,unsigned int rn,unsigned int rd)5503 static void handle_div(DisasContext *s, bool is_signed, unsigned int sf,
5504 unsigned int rm, unsigned int rn, unsigned int rd)
5505 {
5506 TCGv_i64 tcg_n, tcg_m, tcg_rd;
5507 tcg_rd = cpu_reg(s, rd);
5508
5509 if (!sf && is_signed) {
5510 tcg_n = tcg_temp_new_i64();
5511 tcg_m = tcg_temp_new_i64();
5512 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, rn));
5513 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, rm));
5514 } else {
5515 tcg_n = read_cpu_reg(s, rn, sf);
5516 tcg_m = read_cpu_reg(s, rm, sf);
5517 }
5518
5519 if (is_signed) {
5520 gen_helper_sdiv64(tcg_rd, tcg_n, tcg_m);
5521 } else {
5522 gen_helper_udiv64(tcg_rd, tcg_n, tcg_m);
5523 }
5524
5525 if (!sf) { /* zero extend final result */
5526 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
5527 }
5528 }
5529
5530 /* LSLV, LSRV, ASRV, RORV */
handle_shift_reg(DisasContext * s,enum a64_shift_type shift_type,unsigned int sf,unsigned int rm,unsigned int rn,unsigned int rd)5531 static void handle_shift_reg(DisasContext *s,
5532 enum a64_shift_type shift_type, unsigned int sf,
5533 unsigned int rm, unsigned int rn, unsigned int rd)
5534 {
5535 TCGv_i64 tcg_shift = tcg_temp_new_i64();
5536 TCGv_i64 tcg_rd = cpu_reg(s, rd);
5537 TCGv_i64 tcg_rn = read_cpu_reg(s, rn, sf);
5538
5539 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, rm), sf ? 63 : 31);
5540 shift_reg(tcg_rd, tcg_rn, sf, shift_type, tcg_shift);
5541 }
5542
5543 /* CRC32[BHWX], CRC32C[BHWX] */
handle_crc32(DisasContext * s,unsigned int sf,unsigned int sz,bool crc32c,unsigned int rm,unsigned int rn,unsigned int rd)5544 static void handle_crc32(DisasContext *s,
5545 unsigned int sf, unsigned int sz, bool crc32c,
5546 unsigned int rm, unsigned int rn, unsigned int rd)
5547 {
5548 TCGv_i64 tcg_acc, tcg_val;
5549 TCGv_i32 tcg_bytes;
5550
5551 if (!dc_isar_feature(aa64_crc32, s)
5552 || (sf == 1 && sz != 3)
5553 || (sf == 0 && sz == 3)) {
5554 unallocated_encoding(s);
5555 return;
5556 }
5557
5558 if (sz == 3) {
5559 tcg_val = cpu_reg(s, rm);
5560 } else {
5561 uint64_t mask;
5562 switch (sz) {
5563 case 0:
5564 mask = 0xFF;
5565 break;
5566 case 1:
5567 mask = 0xFFFF;
5568 break;
5569 case 2:
5570 mask = 0xFFFFFFFF;
5571 break;
5572 default:
5573 g_assert_not_reached();
5574 }
5575 tcg_val = tcg_temp_new_i64();
5576 tcg_gen_andi_i64(tcg_val, cpu_reg(s, rm), mask);
5577 }
5578
5579 tcg_acc = cpu_reg(s, rn);
5580 tcg_bytes = tcg_constant_i32(1 << sz);
5581
5582 if (crc32c) {
5583 gen_helper_crc32c_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
5584 } else {
5585 gen_helper_crc32_64(cpu_reg(s, rd), tcg_acc, tcg_val, tcg_bytes);
5586 }
5587 }
5588
5589 /* Data-processing (2 source)
5590 * 31 30 29 28 21 20 16 15 10 9 5 4 0
5591 * +----+---+---+-----------------+------+--------+------+------+
5592 * | sf | 0 | S | 1 1 0 1 0 1 1 0 | Rm | opcode | Rn | Rd |
5593 * +----+---+---+-----------------+------+--------+------+------+
5594 */
disas_data_proc_2src(DisasContext * s,uint32_t insn)5595 static void disas_data_proc_2src(DisasContext *s, uint32_t insn)
5596 {
5597 unsigned int sf, rm, opcode, rn, rd, setflag;
5598 sf = extract32(insn, 31, 1);
5599 setflag = extract32(insn, 29, 1);
5600 rm = extract32(insn, 16, 5);
5601 opcode = extract32(insn, 10, 6);
5602 rn = extract32(insn, 5, 5);
5603 rd = extract32(insn, 0, 5);
5604
5605 if (setflag && opcode != 0) {
5606 unallocated_encoding(s);
5607 return;
5608 }
5609
5610 switch (opcode) {
5611 case 0: /* SUBP(S) */
5612 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5613 goto do_unallocated;
5614 } else {
5615 TCGv_i64 tcg_n, tcg_m, tcg_d;
5616
5617 tcg_n = read_cpu_reg_sp(s, rn, true);
5618 tcg_m = read_cpu_reg_sp(s, rm, true);
5619 tcg_gen_sextract_i64(tcg_n, tcg_n, 0, 56);
5620 tcg_gen_sextract_i64(tcg_m, tcg_m, 0, 56);
5621 tcg_d = cpu_reg(s, rd);
5622
5623 if (setflag) {
5624 gen_sub_CC(true, tcg_d, tcg_n, tcg_m);
5625 } else {
5626 tcg_gen_sub_i64(tcg_d, tcg_n, tcg_m);
5627 }
5628 }
5629 break;
5630 case 2: /* UDIV */
5631 handle_div(s, false, sf, rm, rn, rd);
5632 break;
5633 case 3: /* SDIV */
5634 handle_div(s, true, sf, rm, rn, rd);
5635 break;
5636 case 4: /* IRG */
5637 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5638 goto do_unallocated;
5639 }
5640 if (s->ata[0]) {
5641 gen_helper_irg(cpu_reg_sp(s, rd), tcg_env,
5642 cpu_reg_sp(s, rn), cpu_reg(s, rm));
5643 } else {
5644 gen_address_with_allocation_tag0(cpu_reg_sp(s, rd),
5645 cpu_reg_sp(s, rn));
5646 }
5647 break;
5648 case 5: /* GMI */
5649 if (sf == 0 || !dc_isar_feature(aa64_mte_insn_reg, s)) {
5650 goto do_unallocated;
5651 } else {
5652 TCGv_i64 t = tcg_temp_new_i64();
5653
5654 tcg_gen_extract_i64(t, cpu_reg_sp(s, rn), 56, 4);
5655 tcg_gen_shl_i64(t, tcg_constant_i64(1), t);
5656 tcg_gen_or_i64(cpu_reg(s, rd), cpu_reg(s, rm), t);
5657 }
5658 break;
5659 case 8: /* LSLV */
5660 handle_shift_reg(s, A64_SHIFT_TYPE_LSL, sf, rm, rn, rd);
5661 break;
5662 case 9: /* LSRV */
5663 handle_shift_reg(s, A64_SHIFT_TYPE_LSR, sf, rm, rn, rd);
5664 break;
5665 case 10: /* ASRV */
5666 handle_shift_reg(s, A64_SHIFT_TYPE_ASR, sf, rm, rn, rd);
5667 break;
5668 case 11: /* RORV */
5669 handle_shift_reg(s, A64_SHIFT_TYPE_ROR, sf, rm, rn, rd);
5670 break;
5671 case 12: /* PACGA */
5672 if (sf == 0 || !dc_isar_feature(aa64_pauth, s)) {
5673 goto do_unallocated;
5674 }
5675 gen_helper_pacga(cpu_reg(s, rd), tcg_env,
5676 cpu_reg(s, rn), cpu_reg_sp(s, rm));
5677 break;
5678 case 16:
5679 case 17:
5680 case 18:
5681 case 19:
5682 case 20:
5683 case 21:
5684 case 22:
5685 case 23: /* CRC32 */
5686 {
5687 int sz = extract32(opcode, 0, 2);
5688 bool crc32c = extract32(opcode, 2, 1);
5689 handle_crc32(s, sf, sz, crc32c, rm, rn, rd);
5690 break;
5691 }
5692 default:
5693 do_unallocated:
5694 unallocated_encoding(s);
5695 break;
5696 }
5697 }
5698
5699 /*
5700 * Data processing - register
5701 * 31 30 29 28 25 21 20 16 10 0
5702 * +--+---+--+---+-------+-----+-------+-------+---------+
5703 * | |op0| |op1| 1 0 1 | op2 | | op3 | |
5704 * +--+---+--+---+-------+-----+-------+-------+---------+
5705 */
disas_data_proc_reg(DisasContext * s,uint32_t insn)5706 static void disas_data_proc_reg(DisasContext *s, uint32_t insn)
5707 {
5708 int op0 = extract32(insn, 30, 1);
5709 int op1 = extract32(insn, 28, 1);
5710 int op2 = extract32(insn, 21, 4);
5711 int op3 = extract32(insn, 10, 6);
5712
5713 if (!op1) {
5714 if (op2 & 8) {
5715 if (op2 & 1) {
5716 /* Add/sub (extended register) */
5717 disas_add_sub_ext_reg(s, insn);
5718 } else {
5719 /* Add/sub (shifted register) */
5720 disas_add_sub_reg(s, insn);
5721 }
5722 } else {
5723 /* Logical (shifted register) */
5724 disas_logic_reg(s, insn);
5725 }
5726 return;
5727 }
5728
5729 switch (op2) {
5730 case 0x0:
5731 switch (op3) {
5732 case 0x00: /* Add/subtract (with carry) */
5733 disas_adc_sbc(s, insn);
5734 break;
5735
5736 case 0x01: /* Rotate right into flags */
5737 case 0x21:
5738 disas_rotate_right_into_flags(s, insn);
5739 break;
5740
5741 case 0x02: /* Evaluate into flags */
5742 case 0x12:
5743 case 0x22:
5744 case 0x32:
5745 disas_evaluate_into_flags(s, insn);
5746 break;
5747
5748 default:
5749 goto do_unallocated;
5750 }
5751 break;
5752
5753 case 0x2: /* Conditional compare */
5754 disas_cc(s, insn); /* both imm and reg forms */
5755 break;
5756
5757 case 0x4: /* Conditional select */
5758 disas_cond_select(s, insn);
5759 break;
5760
5761 case 0x6: /* Data-processing */
5762 if (op0) { /* (1 source) */
5763 disas_data_proc_1src(s, insn);
5764 } else { /* (2 source) */
5765 disas_data_proc_2src(s, insn);
5766 }
5767 break;
5768 case 0x8 ... 0xf: /* (3 source) */
5769 disas_data_proc_3src(s, insn);
5770 break;
5771
5772 default:
5773 do_unallocated:
5774 unallocated_encoding(s);
5775 break;
5776 }
5777 }
5778
handle_fp_compare(DisasContext * s,int size,unsigned int rn,unsigned int rm,bool cmp_with_zero,bool signal_all_nans)5779 static void handle_fp_compare(DisasContext *s, int size,
5780 unsigned int rn, unsigned int rm,
5781 bool cmp_with_zero, bool signal_all_nans)
5782 {
5783 TCGv_i64 tcg_flags = tcg_temp_new_i64();
5784 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
5785
5786 if (size == MO_64) {
5787 TCGv_i64 tcg_vn, tcg_vm;
5788
5789 tcg_vn = read_fp_dreg(s, rn);
5790 if (cmp_with_zero) {
5791 tcg_vm = tcg_constant_i64(0);
5792 } else {
5793 tcg_vm = read_fp_dreg(s, rm);
5794 }
5795 if (signal_all_nans) {
5796 gen_helper_vfp_cmped_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5797 } else {
5798 gen_helper_vfp_cmpd_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5799 }
5800 } else {
5801 TCGv_i32 tcg_vn = tcg_temp_new_i32();
5802 TCGv_i32 tcg_vm = tcg_temp_new_i32();
5803
5804 read_vec_element_i32(s, tcg_vn, rn, 0, size);
5805 if (cmp_with_zero) {
5806 tcg_gen_movi_i32(tcg_vm, 0);
5807 } else {
5808 read_vec_element_i32(s, tcg_vm, rm, 0, size);
5809 }
5810
5811 switch (size) {
5812 case MO_32:
5813 if (signal_all_nans) {
5814 gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5815 } else {
5816 gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5817 }
5818 break;
5819 case MO_16:
5820 if (signal_all_nans) {
5821 gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5822 } else {
5823 gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
5824 }
5825 break;
5826 default:
5827 g_assert_not_reached();
5828 }
5829 }
5830
5831 gen_set_nzcv(tcg_flags);
5832 }
5833
5834 /* Floating point compare
5835 * 31 30 29 28 24 23 22 21 20 16 15 14 13 10 9 5 4 0
5836 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
5837 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | op | 1 0 0 0 | Rn | op2 |
5838 * +---+---+---+-----------+------+---+------+-----+---------+------+-------+
5839 */
disas_fp_compare(DisasContext * s,uint32_t insn)5840 static void disas_fp_compare(DisasContext *s, uint32_t insn)
5841 {
5842 unsigned int mos, type, rm, op, rn, opc, op2r;
5843 int size;
5844
5845 mos = extract32(insn, 29, 3);
5846 type = extract32(insn, 22, 2);
5847 rm = extract32(insn, 16, 5);
5848 op = extract32(insn, 14, 2);
5849 rn = extract32(insn, 5, 5);
5850 opc = extract32(insn, 3, 2);
5851 op2r = extract32(insn, 0, 3);
5852
5853 if (mos || op || op2r) {
5854 unallocated_encoding(s);
5855 return;
5856 }
5857
5858 switch (type) {
5859 case 0:
5860 size = MO_32;
5861 break;
5862 case 1:
5863 size = MO_64;
5864 break;
5865 case 3:
5866 size = MO_16;
5867 if (dc_isar_feature(aa64_fp16, s)) {
5868 break;
5869 }
5870 /* fallthru */
5871 default:
5872 unallocated_encoding(s);
5873 return;
5874 }
5875
5876 if (!fp_access_check(s)) {
5877 return;
5878 }
5879
5880 handle_fp_compare(s, size, rn, rm, opc & 1, opc & 2);
5881 }
5882
5883 /* Floating point conditional compare
5884 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 3 0
5885 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
5886 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 0 1 | Rn | op | nzcv |
5887 * +---+---+---+-----------+------+---+------+------+-----+------+----+------+
5888 */
disas_fp_ccomp(DisasContext * s,uint32_t insn)5889 static void disas_fp_ccomp(DisasContext *s, uint32_t insn)
5890 {
5891 unsigned int mos, type, rm, cond, rn, op, nzcv;
5892 TCGLabel *label_continue = NULL;
5893 int size;
5894
5895 mos = extract32(insn, 29, 3);
5896 type = extract32(insn, 22, 2);
5897 rm = extract32(insn, 16, 5);
5898 cond = extract32(insn, 12, 4);
5899 rn = extract32(insn, 5, 5);
5900 op = extract32(insn, 4, 1);
5901 nzcv = extract32(insn, 0, 4);
5902
5903 if (mos) {
5904 unallocated_encoding(s);
5905 return;
5906 }
5907
5908 switch (type) {
5909 case 0:
5910 size = MO_32;
5911 break;
5912 case 1:
5913 size = MO_64;
5914 break;
5915 case 3:
5916 size = MO_16;
5917 if (dc_isar_feature(aa64_fp16, s)) {
5918 break;
5919 }
5920 /* fallthru */
5921 default:
5922 unallocated_encoding(s);
5923 return;
5924 }
5925
5926 if (!fp_access_check(s)) {
5927 return;
5928 }
5929
5930 if (cond < 0x0e) { /* not always */
5931 TCGLabel *label_match = gen_new_label();
5932 label_continue = gen_new_label();
5933 arm_gen_test_cc(cond, label_match);
5934 /* nomatch: */
5935 gen_set_nzcv(tcg_constant_i64(nzcv << 28));
5936 tcg_gen_br(label_continue);
5937 gen_set_label(label_match);
5938 }
5939
5940 handle_fp_compare(s, size, rn, rm, false, op);
5941
5942 if (cond < 0x0e) {
5943 gen_set_label(label_continue);
5944 }
5945 }
5946
5947 /* Floating point conditional select
5948 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
5949 * +---+---+---+-----------+------+---+------+------+-----+------+------+
5950 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | cond | 1 1 | Rn | Rd |
5951 * +---+---+---+-----------+------+---+------+------+-----+------+------+
5952 */
disas_fp_csel(DisasContext * s,uint32_t insn)5953 static void disas_fp_csel(DisasContext *s, uint32_t insn)
5954 {
5955 unsigned int mos, type, rm, cond, rn, rd;
5956 TCGv_i64 t_true, t_false;
5957 DisasCompare64 c;
5958 MemOp sz;
5959
5960 mos = extract32(insn, 29, 3);
5961 type = extract32(insn, 22, 2);
5962 rm = extract32(insn, 16, 5);
5963 cond = extract32(insn, 12, 4);
5964 rn = extract32(insn, 5, 5);
5965 rd = extract32(insn, 0, 5);
5966
5967 if (mos) {
5968 unallocated_encoding(s);
5969 return;
5970 }
5971
5972 switch (type) {
5973 case 0:
5974 sz = MO_32;
5975 break;
5976 case 1:
5977 sz = MO_64;
5978 break;
5979 case 3:
5980 sz = MO_16;
5981 if (dc_isar_feature(aa64_fp16, s)) {
5982 break;
5983 }
5984 /* fallthru */
5985 default:
5986 unallocated_encoding(s);
5987 return;
5988 }
5989
5990 if (!fp_access_check(s)) {
5991 return;
5992 }
5993
5994 /* Zero extend sreg & hreg inputs to 64 bits now. */
5995 t_true = tcg_temp_new_i64();
5996 t_false = tcg_temp_new_i64();
5997 read_vec_element(s, t_true, rn, 0, sz);
5998 read_vec_element(s, t_false, rm, 0, sz);
5999
6000 a64_test_cc(&c, cond);
6001 tcg_gen_movcond_i64(c.cond, t_true, c.value, tcg_constant_i64(0),
6002 t_true, t_false);
6003
6004 /* Note that sregs & hregs write back zeros to the high bits,
6005 and we've already done the zero-extension. */
6006 write_fp_dreg(s, rd, t_true);
6007 }
6008
6009 /* Floating-point data-processing (1 source) - half precision */
handle_fp_1src_half(DisasContext * s,int opcode,int rd,int rn)6010 static void handle_fp_1src_half(DisasContext *s, int opcode, int rd, int rn)
6011 {
6012 TCGv_ptr fpst = NULL;
6013 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
6014 TCGv_i32 tcg_res = tcg_temp_new_i32();
6015
6016 switch (opcode) {
6017 case 0x0: /* FMOV */
6018 tcg_gen_mov_i32(tcg_res, tcg_op);
6019 break;
6020 case 0x1: /* FABS */
6021 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff);
6022 break;
6023 case 0x2: /* FNEG */
6024 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
6025 break;
6026 case 0x3: /* FSQRT */
6027 fpst = fpstatus_ptr(FPST_FPCR_F16);
6028 gen_helper_sqrt_f16(tcg_res, tcg_op, fpst);
6029 break;
6030 case 0x8: /* FRINTN */
6031 case 0x9: /* FRINTP */
6032 case 0xa: /* FRINTM */
6033 case 0xb: /* FRINTZ */
6034 case 0xc: /* FRINTA */
6035 {
6036 TCGv_i32 tcg_rmode;
6037
6038 fpst = fpstatus_ptr(FPST_FPCR_F16);
6039 tcg_rmode = gen_set_rmode(opcode & 7, fpst);
6040 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
6041 gen_restore_rmode(tcg_rmode, fpst);
6042 break;
6043 }
6044 case 0xe: /* FRINTX */
6045 fpst = fpstatus_ptr(FPST_FPCR_F16);
6046 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, fpst);
6047 break;
6048 case 0xf: /* FRINTI */
6049 fpst = fpstatus_ptr(FPST_FPCR_F16);
6050 gen_helper_advsimd_rinth(tcg_res, tcg_op, fpst);
6051 break;
6052 default:
6053 g_assert_not_reached();
6054 }
6055
6056 write_fp_sreg(s, rd, tcg_res);
6057 }
6058
6059 /* Floating-point data-processing (1 source) - single precision */
handle_fp_1src_single(DisasContext * s,int opcode,int rd,int rn)6060 static void handle_fp_1src_single(DisasContext *s, int opcode, int rd, int rn)
6061 {
6062 void (*gen_fpst)(TCGv_i32, TCGv_i32, TCGv_ptr);
6063 TCGv_i32 tcg_op, tcg_res;
6064 TCGv_ptr fpst;
6065 int rmode = -1;
6066
6067 tcg_op = read_fp_sreg(s, rn);
6068 tcg_res = tcg_temp_new_i32();
6069
6070 switch (opcode) {
6071 case 0x0: /* FMOV */
6072 tcg_gen_mov_i32(tcg_res, tcg_op);
6073 goto done;
6074 case 0x1: /* FABS */
6075 gen_helper_vfp_abss(tcg_res, tcg_op);
6076 goto done;
6077 case 0x2: /* FNEG */
6078 gen_helper_vfp_negs(tcg_res, tcg_op);
6079 goto done;
6080 case 0x3: /* FSQRT */
6081 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
6082 goto done;
6083 case 0x6: /* BFCVT */
6084 gen_fpst = gen_helper_bfcvt;
6085 break;
6086 case 0x8: /* FRINTN */
6087 case 0x9: /* FRINTP */
6088 case 0xa: /* FRINTM */
6089 case 0xb: /* FRINTZ */
6090 case 0xc: /* FRINTA */
6091 rmode = opcode & 7;
6092 gen_fpst = gen_helper_rints;
6093 break;
6094 case 0xe: /* FRINTX */
6095 gen_fpst = gen_helper_rints_exact;
6096 break;
6097 case 0xf: /* FRINTI */
6098 gen_fpst = gen_helper_rints;
6099 break;
6100 case 0x10: /* FRINT32Z */
6101 rmode = FPROUNDING_ZERO;
6102 gen_fpst = gen_helper_frint32_s;
6103 break;
6104 case 0x11: /* FRINT32X */
6105 gen_fpst = gen_helper_frint32_s;
6106 break;
6107 case 0x12: /* FRINT64Z */
6108 rmode = FPROUNDING_ZERO;
6109 gen_fpst = gen_helper_frint64_s;
6110 break;
6111 case 0x13: /* FRINT64X */
6112 gen_fpst = gen_helper_frint64_s;
6113 break;
6114 default:
6115 g_assert_not_reached();
6116 }
6117
6118 fpst = fpstatus_ptr(FPST_FPCR);
6119 if (rmode >= 0) {
6120 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
6121 gen_fpst(tcg_res, tcg_op, fpst);
6122 gen_restore_rmode(tcg_rmode, fpst);
6123 } else {
6124 gen_fpst(tcg_res, tcg_op, fpst);
6125 }
6126
6127 done:
6128 write_fp_sreg(s, rd, tcg_res);
6129 }
6130
6131 /* Floating-point data-processing (1 source) - double precision */
handle_fp_1src_double(DisasContext * s,int opcode,int rd,int rn)6132 static void handle_fp_1src_double(DisasContext *s, int opcode, int rd, int rn)
6133 {
6134 void (*gen_fpst)(TCGv_i64, TCGv_i64, TCGv_ptr);
6135 TCGv_i64 tcg_op, tcg_res;
6136 TCGv_ptr fpst;
6137 int rmode = -1;
6138
6139 switch (opcode) {
6140 case 0x0: /* FMOV */
6141 gen_gvec_fn2(s, false, rd, rn, tcg_gen_gvec_mov, 0);
6142 return;
6143 }
6144
6145 tcg_op = read_fp_dreg(s, rn);
6146 tcg_res = tcg_temp_new_i64();
6147
6148 switch (opcode) {
6149 case 0x1: /* FABS */
6150 gen_helper_vfp_absd(tcg_res, tcg_op);
6151 goto done;
6152 case 0x2: /* FNEG */
6153 gen_helper_vfp_negd(tcg_res, tcg_op);
6154 goto done;
6155 case 0x3: /* FSQRT */
6156 gen_helper_vfp_sqrtd(tcg_res, tcg_op, tcg_env);
6157 goto done;
6158 case 0x8: /* FRINTN */
6159 case 0x9: /* FRINTP */
6160 case 0xa: /* FRINTM */
6161 case 0xb: /* FRINTZ */
6162 case 0xc: /* FRINTA */
6163 rmode = opcode & 7;
6164 gen_fpst = gen_helper_rintd;
6165 break;
6166 case 0xe: /* FRINTX */
6167 gen_fpst = gen_helper_rintd_exact;
6168 break;
6169 case 0xf: /* FRINTI */
6170 gen_fpst = gen_helper_rintd;
6171 break;
6172 case 0x10: /* FRINT32Z */
6173 rmode = FPROUNDING_ZERO;
6174 gen_fpst = gen_helper_frint32_d;
6175 break;
6176 case 0x11: /* FRINT32X */
6177 gen_fpst = gen_helper_frint32_d;
6178 break;
6179 case 0x12: /* FRINT64Z */
6180 rmode = FPROUNDING_ZERO;
6181 gen_fpst = gen_helper_frint64_d;
6182 break;
6183 case 0x13: /* FRINT64X */
6184 gen_fpst = gen_helper_frint64_d;
6185 break;
6186 default:
6187 g_assert_not_reached();
6188 }
6189
6190 fpst = fpstatus_ptr(FPST_FPCR);
6191 if (rmode >= 0) {
6192 TCGv_i32 tcg_rmode = gen_set_rmode(rmode, fpst);
6193 gen_fpst(tcg_res, tcg_op, fpst);
6194 gen_restore_rmode(tcg_rmode, fpst);
6195 } else {
6196 gen_fpst(tcg_res, tcg_op, fpst);
6197 }
6198
6199 done:
6200 write_fp_dreg(s, rd, tcg_res);
6201 }
6202
handle_fp_fcvt(DisasContext * s,int opcode,int rd,int rn,int dtype,int ntype)6203 static void handle_fp_fcvt(DisasContext *s, int opcode,
6204 int rd, int rn, int dtype, int ntype)
6205 {
6206 switch (ntype) {
6207 case 0x0:
6208 {
6209 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
6210 if (dtype == 1) {
6211 /* Single to double */
6212 TCGv_i64 tcg_rd = tcg_temp_new_i64();
6213 gen_helper_vfp_fcvtds(tcg_rd, tcg_rn, tcg_env);
6214 write_fp_dreg(s, rd, tcg_rd);
6215 } else {
6216 /* Single to half */
6217 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6218 TCGv_i32 ahp = get_ahp_flag();
6219 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6220
6221 gen_helper_vfp_fcvt_f32_to_f16(tcg_rd, tcg_rn, fpst, ahp);
6222 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
6223 write_fp_sreg(s, rd, tcg_rd);
6224 }
6225 break;
6226 }
6227 case 0x1:
6228 {
6229 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
6230 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6231 if (dtype == 0) {
6232 /* Double to single */
6233 gen_helper_vfp_fcvtsd(tcg_rd, tcg_rn, tcg_env);
6234 } else {
6235 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6236 TCGv_i32 ahp = get_ahp_flag();
6237 /* Double to half */
6238 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd, tcg_rn, fpst, ahp);
6239 /* write_fp_sreg is OK here because top half of tcg_rd is zero */
6240 }
6241 write_fp_sreg(s, rd, tcg_rd);
6242 break;
6243 }
6244 case 0x3:
6245 {
6246 TCGv_i32 tcg_rn = read_fp_sreg(s, rn);
6247 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_FPCR);
6248 TCGv_i32 tcg_ahp = get_ahp_flag();
6249 tcg_gen_ext16u_i32(tcg_rn, tcg_rn);
6250 if (dtype == 0) {
6251 /* Half to single */
6252 TCGv_i32 tcg_rd = tcg_temp_new_i32();
6253 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
6254 write_fp_sreg(s, rd, tcg_rd);
6255 } else {
6256 /* Half to double */
6257 TCGv_i64 tcg_rd = tcg_temp_new_i64();
6258 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
6259 write_fp_dreg(s, rd, tcg_rd);
6260 }
6261 break;
6262 }
6263 default:
6264 g_assert_not_reached();
6265 }
6266 }
6267
6268 /* Floating point data-processing (1 source)
6269 * 31 30 29 28 24 23 22 21 20 15 14 10 9 5 4 0
6270 * +---+---+---+-----------+------+---+--------+-----------+------+------+
6271 * | M | 0 | S | 1 1 1 1 0 | type | 1 | opcode | 1 0 0 0 0 | Rn | Rd |
6272 * +---+---+---+-----------+------+---+--------+-----------+------+------+
6273 */
disas_fp_1src(DisasContext * s,uint32_t insn)6274 static void disas_fp_1src(DisasContext *s, uint32_t insn)
6275 {
6276 int mos = extract32(insn, 29, 3);
6277 int type = extract32(insn, 22, 2);
6278 int opcode = extract32(insn, 15, 6);
6279 int rn = extract32(insn, 5, 5);
6280 int rd = extract32(insn, 0, 5);
6281
6282 if (mos) {
6283 goto do_unallocated;
6284 }
6285
6286 switch (opcode) {
6287 case 0x4: case 0x5: case 0x7:
6288 {
6289 /* FCVT between half, single and double precision */
6290 int dtype = extract32(opcode, 0, 2);
6291 if (type == 2 || dtype == type) {
6292 goto do_unallocated;
6293 }
6294 if (!fp_access_check(s)) {
6295 return;
6296 }
6297
6298 handle_fp_fcvt(s, opcode, rd, rn, dtype, type);
6299 break;
6300 }
6301
6302 case 0x10 ... 0x13: /* FRINT{32,64}{X,Z} */
6303 if (type > 1 || !dc_isar_feature(aa64_frint, s)) {
6304 goto do_unallocated;
6305 }
6306 /* fall through */
6307 case 0x0 ... 0x3:
6308 case 0x8 ... 0xc:
6309 case 0xe ... 0xf:
6310 /* 32-to-32 and 64-to-64 ops */
6311 switch (type) {
6312 case 0:
6313 if (!fp_access_check(s)) {
6314 return;
6315 }
6316 handle_fp_1src_single(s, opcode, rd, rn);
6317 break;
6318 case 1:
6319 if (!fp_access_check(s)) {
6320 return;
6321 }
6322 handle_fp_1src_double(s, opcode, rd, rn);
6323 break;
6324 case 3:
6325 if (!dc_isar_feature(aa64_fp16, s)) {
6326 goto do_unallocated;
6327 }
6328
6329 if (!fp_access_check(s)) {
6330 return;
6331 }
6332 handle_fp_1src_half(s, opcode, rd, rn);
6333 break;
6334 default:
6335 goto do_unallocated;
6336 }
6337 break;
6338
6339 case 0x6:
6340 switch (type) {
6341 case 1: /* BFCVT */
6342 if (!dc_isar_feature(aa64_bf16, s)) {
6343 goto do_unallocated;
6344 }
6345 if (!fp_access_check(s)) {
6346 return;
6347 }
6348 handle_fp_1src_single(s, opcode, rd, rn);
6349 break;
6350 default:
6351 goto do_unallocated;
6352 }
6353 break;
6354
6355 default:
6356 do_unallocated:
6357 unallocated_encoding(s);
6358 break;
6359 }
6360 }
6361
6362 /* Floating-point data-processing (2 source) - single precision */
handle_fp_2src_single(DisasContext * s,int opcode,int rd,int rn,int rm)6363 static void handle_fp_2src_single(DisasContext *s, int opcode,
6364 int rd, int rn, int rm)
6365 {
6366 TCGv_i32 tcg_op1;
6367 TCGv_i32 tcg_op2;
6368 TCGv_i32 tcg_res;
6369 TCGv_ptr fpst;
6370
6371 tcg_res = tcg_temp_new_i32();
6372 fpst = fpstatus_ptr(FPST_FPCR);
6373 tcg_op1 = read_fp_sreg(s, rn);
6374 tcg_op2 = read_fp_sreg(s, rm);
6375
6376 switch (opcode) {
6377 case 0x0: /* FMUL */
6378 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
6379 break;
6380 case 0x1: /* FDIV */
6381 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst);
6382 break;
6383 case 0x2: /* FADD */
6384 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
6385 break;
6386 case 0x3: /* FSUB */
6387 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
6388 break;
6389 case 0x4: /* FMAX */
6390 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
6391 break;
6392 case 0x5: /* FMIN */
6393 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
6394 break;
6395 case 0x6: /* FMAXNM */
6396 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
6397 break;
6398 case 0x7: /* FMINNM */
6399 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
6400 break;
6401 case 0x8: /* FNMUL */
6402 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
6403 gen_helper_vfp_negs(tcg_res, tcg_res);
6404 break;
6405 }
6406
6407 write_fp_sreg(s, rd, tcg_res);
6408 }
6409
6410 /* Floating-point data-processing (2 source) - double precision */
handle_fp_2src_double(DisasContext * s,int opcode,int rd,int rn,int rm)6411 static void handle_fp_2src_double(DisasContext *s, int opcode,
6412 int rd, int rn, int rm)
6413 {
6414 TCGv_i64 tcg_op1;
6415 TCGv_i64 tcg_op2;
6416 TCGv_i64 tcg_res;
6417 TCGv_ptr fpst;
6418
6419 tcg_res = tcg_temp_new_i64();
6420 fpst = fpstatus_ptr(FPST_FPCR);
6421 tcg_op1 = read_fp_dreg(s, rn);
6422 tcg_op2 = read_fp_dreg(s, rm);
6423
6424 switch (opcode) {
6425 case 0x0: /* FMUL */
6426 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
6427 break;
6428 case 0x1: /* FDIV */
6429 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst);
6430 break;
6431 case 0x2: /* FADD */
6432 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
6433 break;
6434 case 0x3: /* FSUB */
6435 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
6436 break;
6437 case 0x4: /* FMAX */
6438 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
6439 break;
6440 case 0x5: /* FMIN */
6441 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
6442 break;
6443 case 0x6: /* FMAXNM */
6444 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
6445 break;
6446 case 0x7: /* FMINNM */
6447 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
6448 break;
6449 case 0x8: /* FNMUL */
6450 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
6451 gen_helper_vfp_negd(tcg_res, tcg_res);
6452 break;
6453 }
6454
6455 write_fp_dreg(s, rd, tcg_res);
6456 }
6457
6458 /* Floating-point data-processing (2 source) - half precision */
handle_fp_2src_half(DisasContext * s,int opcode,int rd,int rn,int rm)6459 static void handle_fp_2src_half(DisasContext *s, int opcode,
6460 int rd, int rn, int rm)
6461 {
6462 TCGv_i32 tcg_op1;
6463 TCGv_i32 tcg_op2;
6464 TCGv_i32 tcg_res;
6465 TCGv_ptr fpst;
6466
6467 tcg_res = tcg_temp_new_i32();
6468 fpst = fpstatus_ptr(FPST_FPCR_F16);
6469 tcg_op1 = read_fp_hreg(s, rn);
6470 tcg_op2 = read_fp_hreg(s, rm);
6471
6472 switch (opcode) {
6473 case 0x0: /* FMUL */
6474 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
6475 break;
6476 case 0x1: /* FDIV */
6477 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst);
6478 break;
6479 case 0x2: /* FADD */
6480 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
6481 break;
6482 case 0x3: /* FSUB */
6483 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
6484 break;
6485 case 0x4: /* FMAX */
6486 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
6487 break;
6488 case 0x5: /* FMIN */
6489 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
6490 break;
6491 case 0x6: /* FMAXNM */
6492 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
6493 break;
6494 case 0x7: /* FMINNM */
6495 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
6496 break;
6497 case 0x8: /* FNMUL */
6498 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
6499 tcg_gen_xori_i32(tcg_res, tcg_res, 0x8000);
6500 break;
6501 default:
6502 g_assert_not_reached();
6503 }
6504
6505 write_fp_sreg(s, rd, tcg_res);
6506 }
6507
6508 /* Floating point data-processing (2 source)
6509 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
6510 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
6511 * | M | 0 | S | 1 1 1 1 0 | type | 1 | Rm | opcode | 1 0 | Rn | Rd |
6512 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
6513 */
disas_fp_2src(DisasContext * s,uint32_t insn)6514 static void disas_fp_2src(DisasContext *s, uint32_t insn)
6515 {
6516 int mos = extract32(insn, 29, 3);
6517 int type = extract32(insn, 22, 2);
6518 int rd = extract32(insn, 0, 5);
6519 int rn = extract32(insn, 5, 5);
6520 int rm = extract32(insn, 16, 5);
6521 int opcode = extract32(insn, 12, 4);
6522
6523 if (opcode > 8 || mos) {
6524 unallocated_encoding(s);
6525 return;
6526 }
6527
6528 switch (type) {
6529 case 0:
6530 if (!fp_access_check(s)) {
6531 return;
6532 }
6533 handle_fp_2src_single(s, opcode, rd, rn, rm);
6534 break;
6535 case 1:
6536 if (!fp_access_check(s)) {
6537 return;
6538 }
6539 handle_fp_2src_double(s, opcode, rd, rn, rm);
6540 break;
6541 case 3:
6542 if (!dc_isar_feature(aa64_fp16, s)) {
6543 unallocated_encoding(s);
6544 return;
6545 }
6546 if (!fp_access_check(s)) {
6547 return;
6548 }
6549 handle_fp_2src_half(s, opcode, rd, rn, rm);
6550 break;
6551 default:
6552 unallocated_encoding(s);
6553 }
6554 }
6555
6556 /* Floating-point data-processing (3 source) - single precision */
handle_fp_3src_single(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6557 static void handle_fp_3src_single(DisasContext *s, bool o0, bool o1,
6558 int rd, int rn, int rm, int ra)
6559 {
6560 TCGv_i32 tcg_op1, tcg_op2, tcg_op3;
6561 TCGv_i32 tcg_res = tcg_temp_new_i32();
6562 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6563
6564 tcg_op1 = read_fp_sreg(s, rn);
6565 tcg_op2 = read_fp_sreg(s, rm);
6566 tcg_op3 = read_fp_sreg(s, ra);
6567
6568 /* These are fused multiply-add, and must be done as one
6569 * floating point operation with no rounding between the
6570 * multiplication and addition steps.
6571 * NB that doing the negations here as separate steps is
6572 * correct : an input NaN should come out with its sign bit
6573 * flipped if it is a negated-input.
6574 */
6575 if (o1 == true) {
6576 gen_helper_vfp_negs(tcg_op3, tcg_op3);
6577 }
6578
6579 if (o0 != o1) {
6580 gen_helper_vfp_negs(tcg_op1, tcg_op1);
6581 }
6582
6583 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6584
6585 write_fp_sreg(s, rd, tcg_res);
6586 }
6587
6588 /* Floating-point data-processing (3 source) - double precision */
handle_fp_3src_double(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6589 static void handle_fp_3src_double(DisasContext *s, bool o0, bool o1,
6590 int rd, int rn, int rm, int ra)
6591 {
6592 TCGv_i64 tcg_op1, tcg_op2, tcg_op3;
6593 TCGv_i64 tcg_res = tcg_temp_new_i64();
6594 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
6595
6596 tcg_op1 = read_fp_dreg(s, rn);
6597 tcg_op2 = read_fp_dreg(s, rm);
6598 tcg_op3 = read_fp_dreg(s, ra);
6599
6600 /* These are fused multiply-add, and must be done as one
6601 * floating point operation with no rounding between the
6602 * multiplication and addition steps.
6603 * NB that doing the negations here as separate steps is
6604 * correct : an input NaN should come out with its sign bit
6605 * flipped if it is a negated-input.
6606 */
6607 if (o1 == true) {
6608 gen_helper_vfp_negd(tcg_op3, tcg_op3);
6609 }
6610
6611 if (o0 != o1) {
6612 gen_helper_vfp_negd(tcg_op1, tcg_op1);
6613 }
6614
6615 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6616
6617 write_fp_dreg(s, rd, tcg_res);
6618 }
6619
6620 /* Floating-point data-processing (3 source) - half precision */
handle_fp_3src_half(DisasContext * s,bool o0,bool o1,int rd,int rn,int rm,int ra)6621 static void handle_fp_3src_half(DisasContext *s, bool o0, bool o1,
6622 int rd, int rn, int rm, int ra)
6623 {
6624 TCGv_i32 tcg_op1, tcg_op2, tcg_op3;
6625 TCGv_i32 tcg_res = tcg_temp_new_i32();
6626 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR_F16);
6627
6628 tcg_op1 = read_fp_hreg(s, rn);
6629 tcg_op2 = read_fp_hreg(s, rm);
6630 tcg_op3 = read_fp_hreg(s, ra);
6631
6632 /* These are fused multiply-add, and must be done as one
6633 * floating point operation with no rounding between the
6634 * multiplication and addition steps.
6635 * NB that doing the negations here as separate steps is
6636 * correct : an input NaN should come out with its sign bit
6637 * flipped if it is a negated-input.
6638 */
6639 if (o1 == true) {
6640 tcg_gen_xori_i32(tcg_op3, tcg_op3, 0x8000);
6641 }
6642
6643 if (o0 != o1) {
6644 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000);
6645 }
6646
6647 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_op3, fpst);
6648
6649 write_fp_sreg(s, rd, tcg_res);
6650 }
6651
6652 /* Floating point data-processing (3 source)
6653 * 31 30 29 28 24 23 22 21 20 16 15 14 10 9 5 4 0
6654 * +---+---+---+-----------+------+----+------+----+------+------+------+
6655 * | M | 0 | S | 1 1 1 1 1 | type | o1 | Rm | o0 | Ra | Rn | Rd |
6656 * +---+---+---+-----------+------+----+------+----+------+------+------+
6657 */
disas_fp_3src(DisasContext * s,uint32_t insn)6658 static void disas_fp_3src(DisasContext *s, uint32_t insn)
6659 {
6660 int mos = extract32(insn, 29, 3);
6661 int type = extract32(insn, 22, 2);
6662 int rd = extract32(insn, 0, 5);
6663 int rn = extract32(insn, 5, 5);
6664 int ra = extract32(insn, 10, 5);
6665 int rm = extract32(insn, 16, 5);
6666 bool o0 = extract32(insn, 15, 1);
6667 bool o1 = extract32(insn, 21, 1);
6668
6669 if (mos) {
6670 unallocated_encoding(s);
6671 return;
6672 }
6673
6674 switch (type) {
6675 case 0:
6676 if (!fp_access_check(s)) {
6677 return;
6678 }
6679 handle_fp_3src_single(s, o0, o1, rd, rn, rm, ra);
6680 break;
6681 case 1:
6682 if (!fp_access_check(s)) {
6683 return;
6684 }
6685 handle_fp_3src_double(s, o0, o1, rd, rn, rm, ra);
6686 break;
6687 case 3:
6688 if (!dc_isar_feature(aa64_fp16, s)) {
6689 unallocated_encoding(s);
6690 return;
6691 }
6692 if (!fp_access_check(s)) {
6693 return;
6694 }
6695 handle_fp_3src_half(s, o0, o1, rd, rn, rm, ra);
6696 break;
6697 default:
6698 unallocated_encoding(s);
6699 }
6700 }
6701
6702 /* Floating point immediate
6703 * 31 30 29 28 24 23 22 21 20 13 12 10 9 5 4 0
6704 * +---+---+---+-----------+------+---+------------+-------+------+------+
6705 * | M | 0 | S | 1 1 1 1 0 | type | 1 | imm8 | 1 0 0 | imm5 | Rd |
6706 * +---+---+---+-----------+------+---+------------+-------+------+------+
6707 */
disas_fp_imm(DisasContext * s,uint32_t insn)6708 static void disas_fp_imm(DisasContext *s, uint32_t insn)
6709 {
6710 int rd = extract32(insn, 0, 5);
6711 int imm5 = extract32(insn, 5, 5);
6712 int imm8 = extract32(insn, 13, 8);
6713 int type = extract32(insn, 22, 2);
6714 int mos = extract32(insn, 29, 3);
6715 uint64_t imm;
6716 MemOp sz;
6717
6718 if (mos || imm5) {
6719 unallocated_encoding(s);
6720 return;
6721 }
6722
6723 switch (type) {
6724 case 0:
6725 sz = MO_32;
6726 break;
6727 case 1:
6728 sz = MO_64;
6729 break;
6730 case 3:
6731 sz = MO_16;
6732 if (dc_isar_feature(aa64_fp16, s)) {
6733 break;
6734 }
6735 /* fallthru */
6736 default:
6737 unallocated_encoding(s);
6738 return;
6739 }
6740
6741 if (!fp_access_check(s)) {
6742 return;
6743 }
6744
6745 imm = vfp_expand_imm(sz, imm8);
6746 write_fp_dreg(s, rd, tcg_constant_i64(imm));
6747 }
6748
6749 /* Handle floating point <=> fixed point conversions. Note that we can
6750 * also deal with fp <=> integer conversions as a special case (scale == 64)
6751 * OPTME: consider handling that special case specially or at least skipping
6752 * the call to scalbn in the helpers for zero shifts.
6753 */
handle_fpfpcvt(DisasContext * s,int rd,int rn,int opcode,bool itof,int rmode,int scale,int sf,int type)6754 static void handle_fpfpcvt(DisasContext *s, int rd, int rn, int opcode,
6755 bool itof, int rmode, int scale, int sf, int type)
6756 {
6757 bool is_signed = !(opcode & 1);
6758 TCGv_ptr tcg_fpstatus;
6759 TCGv_i32 tcg_shift, tcg_single;
6760 TCGv_i64 tcg_double;
6761
6762 tcg_fpstatus = fpstatus_ptr(type == 3 ? FPST_FPCR_F16 : FPST_FPCR);
6763
6764 tcg_shift = tcg_constant_i32(64 - scale);
6765
6766 if (itof) {
6767 TCGv_i64 tcg_int = cpu_reg(s, rn);
6768 if (!sf) {
6769 TCGv_i64 tcg_extend = tcg_temp_new_i64();
6770
6771 if (is_signed) {
6772 tcg_gen_ext32s_i64(tcg_extend, tcg_int);
6773 } else {
6774 tcg_gen_ext32u_i64(tcg_extend, tcg_int);
6775 }
6776
6777 tcg_int = tcg_extend;
6778 }
6779
6780 switch (type) {
6781 case 1: /* float64 */
6782 tcg_double = tcg_temp_new_i64();
6783 if (is_signed) {
6784 gen_helper_vfp_sqtod(tcg_double, tcg_int,
6785 tcg_shift, tcg_fpstatus);
6786 } else {
6787 gen_helper_vfp_uqtod(tcg_double, tcg_int,
6788 tcg_shift, tcg_fpstatus);
6789 }
6790 write_fp_dreg(s, rd, tcg_double);
6791 break;
6792
6793 case 0: /* float32 */
6794 tcg_single = tcg_temp_new_i32();
6795 if (is_signed) {
6796 gen_helper_vfp_sqtos(tcg_single, tcg_int,
6797 tcg_shift, tcg_fpstatus);
6798 } else {
6799 gen_helper_vfp_uqtos(tcg_single, tcg_int,
6800 tcg_shift, tcg_fpstatus);
6801 }
6802 write_fp_sreg(s, rd, tcg_single);
6803 break;
6804
6805 case 3: /* float16 */
6806 tcg_single = tcg_temp_new_i32();
6807 if (is_signed) {
6808 gen_helper_vfp_sqtoh(tcg_single, tcg_int,
6809 tcg_shift, tcg_fpstatus);
6810 } else {
6811 gen_helper_vfp_uqtoh(tcg_single, tcg_int,
6812 tcg_shift, tcg_fpstatus);
6813 }
6814 write_fp_sreg(s, rd, tcg_single);
6815 break;
6816
6817 default:
6818 g_assert_not_reached();
6819 }
6820 } else {
6821 TCGv_i64 tcg_int = cpu_reg(s, rd);
6822 TCGv_i32 tcg_rmode;
6823
6824 if (extract32(opcode, 2, 1)) {
6825 /* There are too many rounding modes to all fit into rmode,
6826 * so FCVTA[US] is a special case.
6827 */
6828 rmode = FPROUNDING_TIEAWAY;
6829 }
6830
6831 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
6832
6833 switch (type) {
6834 case 1: /* float64 */
6835 tcg_double = read_fp_dreg(s, rn);
6836 if (is_signed) {
6837 if (!sf) {
6838 gen_helper_vfp_tosld(tcg_int, tcg_double,
6839 tcg_shift, tcg_fpstatus);
6840 } else {
6841 gen_helper_vfp_tosqd(tcg_int, tcg_double,
6842 tcg_shift, tcg_fpstatus);
6843 }
6844 } else {
6845 if (!sf) {
6846 gen_helper_vfp_tould(tcg_int, tcg_double,
6847 tcg_shift, tcg_fpstatus);
6848 } else {
6849 gen_helper_vfp_touqd(tcg_int, tcg_double,
6850 tcg_shift, tcg_fpstatus);
6851 }
6852 }
6853 if (!sf) {
6854 tcg_gen_ext32u_i64(tcg_int, tcg_int);
6855 }
6856 break;
6857
6858 case 0: /* float32 */
6859 tcg_single = read_fp_sreg(s, rn);
6860 if (sf) {
6861 if (is_signed) {
6862 gen_helper_vfp_tosqs(tcg_int, tcg_single,
6863 tcg_shift, tcg_fpstatus);
6864 } else {
6865 gen_helper_vfp_touqs(tcg_int, tcg_single,
6866 tcg_shift, tcg_fpstatus);
6867 }
6868 } else {
6869 TCGv_i32 tcg_dest = tcg_temp_new_i32();
6870 if (is_signed) {
6871 gen_helper_vfp_tosls(tcg_dest, tcg_single,
6872 tcg_shift, tcg_fpstatus);
6873 } else {
6874 gen_helper_vfp_touls(tcg_dest, tcg_single,
6875 tcg_shift, tcg_fpstatus);
6876 }
6877 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
6878 }
6879 break;
6880
6881 case 3: /* float16 */
6882 tcg_single = read_fp_sreg(s, rn);
6883 if (sf) {
6884 if (is_signed) {
6885 gen_helper_vfp_tosqh(tcg_int, tcg_single,
6886 tcg_shift, tcg_fpstatus);
6887 } else {
6888 gen_helper_vfp_touqh(tcg_int, tcg_single,
6889 tcg_shift, tcg_fpstatus);
6890 }
6891 } else {
6892 TCGv_i32 tcg_dest = tcg_temp_new_i32();
6893 if (is_signed) {
6894 gen_helper_vfp_toslh(tcg_dest, tcg_single,
6895 tcg_shift, tcg_fpstatus);
6896 } else {
6897 gen_helper_vfp_toulh(tcg_dest, tcg_single,
6898 tcg_shift, tcg_fpstatus);
6899 }
6900 tcg_gen_extu_i32_i64(tcg_int, tcg_dest);
6901 }
6902 break;
6903
6904 default:
6905 g_assert_not_reached();
6906 }
6907
6908 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
6909 }
6910 }
6911
6912 /* Floating point <-> fixed point conversions
6913 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
6914 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
6915 * | sf | 0 | S | 1 1 1 1 0 | type | 0 | rmode | opcode | scale | Rn | Rd |
6916 * +----+---+---+-----------+------+---+-------+--------+-------+------+------+
6917 */
disas_fp_fixed_conv(DisasContext * s,uint32_t insn)6918 static void disas_fp_fixed_conv(DisasContext *s, uint32_t insn)
6919 {
6920 int rd = extract32(insn, 0, 5);
6921 int rn = extract32(insn, 5, 5);
6922 int scale = extract32(insn, 10, 6);
6923 int opcode = extract32(insn, 16, 3);
6924 int rmode = extract32(insn, 19, 2);
6925 int type = extract32(insn, 22, 2);
6926 bool sbit = extract32(insn, 29, 1);
6927 bool sf = extract32(insn, 31, 1);
6928 bool itof;
6929
6930 if (sbit || (!sf && scale < 32)) {
6931 unallocated_encoding(s);
6932 return;
6933 }
6934
6935 switch (type) {
6936 case 0: /* float32 */
6937 case 1: /* float64 */
6938 break;
6939 case 3: /* float16 */
6940 if (dc_isar_feature(aa64_fp16, s)) {
6941 break;
6942 }
6943 /* fallthru */
6944 default:
6945 unallocated_encoding(s);
6946 return;
6947 }
6948
6949 switch ((rmode << 3) | opcode) {
6950 case 0x2: /* SCVTF */
6951 case 0x3: /* UCVTF */
6952 itof = true;
6953 break;
6954 case 0x18: /* FCVTZS */
6955 case 0x19: /* FCVTZU */
6956 itof = false;
6957 break;
6958 default:
6959 unallocated_encoding(s);
6960 return;
6961 }
6962
6963 if (!fp_access_check(s)) {
6964 return;
6965 }
6966
6967 handle_fpfpcvt(s, rd, rn, opcode, itof, FPROUNDING_ZERO, scale, sf, type);
6968 }
6969
handle_fmov(DisasContext * s,int rd,int rn,int type,bool itof)6970 static void handle_fmov(DisasContext *s, int rd, int rn, int type, bool itof)
6971 {
6972 /* FMOV: gpr to or from float, double, or top half of quad fp reg,
6973 * without conversion.
6974 */
6975
6976 if (itof) {
6977 TCGv_i64 tcg_rn = cpu_reg(s, rn);
6978 TCGv_i64 tmp;
6979
6980 switch (type) {
6981 case 0:
6982 /* 32 bit */
6983 tmp = tcg_temp_new_i64();
6984 tcg_gen_ext32u_i64(tmp, tcg_rn);
6985 write_fp_dreg(s, rd, tmp);
6986 break;
6987 case 1:
6988 /* 64 bit */
6989 write_fp_dreg(s, rd, tcg_rn);
6990 break;
6991 case 2:
6992 /* 64 bit to top half. */
6993 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, rd));
6994 clear_vec_high(s, true, rd);
6995 break;
6996 case 3:
6997 /* 16 bit */
6998 tmp = tcg_temp_new_i64();
6999 tcg_gen_ext16u_i64(tmp, tcg_rn);
7000 write_fp_dreg(s, rd, tmp);
7001 break;
7002 default:
7003 g_assert_not_reached();
7004 }
7005 } else {
7006 TCGv_i64 tcg_rd = cpu_reg(s, rd);
7007
7008 switch (type) {
7009 case 0:
7010 /* 32 bit */
7011 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_32));
7012 break;
7013 case 1:
7014 /* 64 bit */
7015 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_64));
7016 break;
7017 case 2:
7018 /* 64 bits from top half */
7019 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, rn));
7020 break;
7021 case 3:
7022 /* 16 bit */
7023 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, rn, MO_16));
7024 break;
7025 default:
7026 g_assert_not_reached();
7027 }
7028 }
7029 }
7030
handle_fjcvtzs(DisasContext * s,int rd,int rn)7031 static void handle_fjcvtzs(DisasContext *s, int rd, int rn)
7032 {
7033 TCGv_i64 t = read_fp_dreg(s, rn);
7034 TCGv_ptr fpstatus = fpstatus_ptr(FPST_FPCR);
7035
7036 gen_helper_fjcvtzs(t, t, fpstatus);
7037
7038 tcg_gen_ext32u_i64(cpu_reg(s, rd), t);
7039 tcg_gen_extrh_i64_i32(cpu_ZF, t);
7040 tcg_gen_movi_i32(cpu_CF, 0);
7041 tcg_gen_movi_i32(cpu_NF, 0);
7042 tcg_gen_movi_i32(cpu_VF, 0);
7043 }
7044
7045 /* Floating point <-> integer conversions
7046 * 31 30 29 28 24 23 22 21 20 19 18 16 15 10 9 5 4 0
7047 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
7048 * | sf | 0 | S | 1 1 1 1 0 | type | 1 | rmode | opc | 0 0 0 0 0 0 | Rn | Rd |
7049 * +----+---+---+-----------+------+---+-------+-----+-------------+----+----+
7050 */
disas_fp_int_conv(DisasContext * s,uint32_t insn)7051 static void disas_fp_int_conv(DisasContext *s, uint32_t insn)
7052 {
7053 int rd = extract32(insn, 0, 5);
7054 int rn = extract32(insn, 5, 5);
7055 int opcode = extract32(insn, 16, 3);
7056 int rmode = extract32(insn, 19, 2);
7057 int type = extract32(insn, 22, 2);
7058 bool sbit = extract32(insn, 29, 1);
7059 bool sf = extract32(insn, 31, 1);
7060 bool itof = false;
7061
7062 if (sbit) {
7063 goto do_unallocated;
7064 }
7065
7066 switch (opcode) {
7067 case 2: /* SCVTF */
7068 case 3: /* UCVTF */
7069 itof = true;
7070 /* fallthru */
7071 case 4: /* FCVTAS */
7072 case 5: /* FCVTAU */
7073 if (rmode != 0) {
7074 goto do_unallocated;
7075 }
7076 /* fallthru */
7077 case 0: /* FCVT[NPMZ]S */
7078 case 1: /* FCVT[NPMZ]U */
7079 switch (type) {
7080 case 0: /* float32 */
7081 case 1: /* float64 */
7082 break;
7083 case 3: /* float16 */
7084 if (!dc_isar_feature(aa64_fp16, s)) {
7085 goto do_unallocated;
7086 }
7087 break;
7088 default:
7089 goto do_unallocated;
7090 }
7091 if (!fp_access_check(s)) {
7092 return;
7093 }
7094 handle_fpfpcvt(s, rd, rn, opcode, itof, rmode, 64, sf, type);
7095 break;
7096
7097 default:
7098 switch (sf << 7 | type << 5 | rmode << 3 | opcode) {
7099 case 0b01100110: /* FMOV half <-> 32-bit int */
7100 case 0b01100111:
7101 case 0b11100110: /* FMOV half <-> 64-bit int */
7102 case 0b11100111:
7103 if (!dc_isar_feature(aa64_fp16, s)) {
7104 goto do_unallocated;
7105 }
7106 /* fallthru */
7107 case 0b00000110: /* FMOV 32-bit */
7108 case 0b00000111:
7109 case 0b10100110: /* FMOV 64-bit */
7110 case 0b10100111:
7111 case 0b11001110: /* FMOV top half of 128-bit */
7112 case 0b11001111:
7113 if (!fp_access_check(s)) {
7114 return;
7115 }
7116 itof = opcode & 1;
7117 handle_fmov(s, rd, rn, type, itof);
7118 break;
7119
7120 case 0b00111110: /* FJCVTZS */
7121 if (!dc_isar_feature(aa64_jscvt, s)) {
7122 goto do_unallocated;
7123 } else if (fp_access_check(s)) {
7124 handle_fjcvtzs(s, rd, rn);
7125 }
7126 break;
7127
7128 default:
7129 do_unallocated:
7130 unallocated_encoding(s);
7131 return;
7132 }
7133 break;
7134 }
7135 }
7136
7137 /* FP-specific subcases of table C3-6 (SIMD and FP data processing)
7138 * 31 30 29 28 25 24 0
7139 * +---+---+---+---------+-----------------------------+
7140 * | | 0 | | 1 1 1 1 | |
7141 * +---+---+---+---------+-----------------------------+
7142 */
disas_data_proc_fp(DisasContext * s,uint32_t insn)7143 static void disas_data_proc_fp(DisasContext *s, uint32_t insn)
7144 {
7145 if (extract32(insn, 24, 1)) {
7146 /* Floating point data-processing (3 source) */
7147 disas_fp_3src(s, insn);
7148 } else if (extract32(insn, 21, 1) == 0) {
7149 /* Floating point to fixed point conversions */
7150 disas_fp_fixed_conv(s, insn);
7151 } else {
7152 switch (extract32(insn, 10, 2)) {
7153 case 1:
7154 /* Floating point conditional compare */
7155 disas_fp_ccomp(s, insn);
7156 break;
7157 case 2:
7158 /* Floating point data-processing (2 source) */
7159 disas_fp_2src(s, insn);
7160 break;
7161 case 3:
7162 /* Floating point conditional select */
7163 disas_fp_csel(s, insn);
7164 break;
7165 case 0:
7166 switch (ctz32(extract32(insn, 12, 4))) {
7167 case 0: /* [15:12] == xxx1 */
7168 /* Floating point immediate */
7169 disas_fp_imm(s, insn);
7170 break;
7171 case 1: /* [15:12] == xx10 */
7172 /* Floating point compare */
7173 disas_fp_compare(s, insn);
7174 break;
7175 case 2: /* [15:12] == x100 */
7176 /* Floating point data-processing (1 source) */
7177 disas_fp_1src(s, insn);
7178 break;
7179 case 3: /* [15:12] == 1000 */
7180 unallocated_encoding(s);
7181 break;
7182 default: /* [15:12] == 0000 */
7183 /* Floating point <-> integer conversions */
7184 disas_fp_int_conv(s, insn);
7185 break;
7186 }
7187 break;
7188 }
7189 }
7190 }
7191
do_ext64(DisasContext * s,TCGv_i64 tcg_left,TCGv_i64 tcg_right,int pos)7192 static void do_ext64(DisasContext *s, TCGv_i64 tcg_left, TCGv_i64 tcg_right,
7193 int pos)
7194 {
7195 /* Extract 64 bits from the middle of two concatenated 64 bit
7196 * vector register slices left:right. The extracted bits start
7197 * at 'pos' bits into the right (least significant) side.
7198 * We return the result in tcg_right, and guarantee not to
7199 * trash tcg_left.
7200 */
7201 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
7202 assert(pos > 0 && pos < 64);
7203
7204 tcg_gen_shri_i64(tcg_right, tcg_right, pos);
7205 tcg_gen_shli_i64(tcg_tmp, tcg_left, 64 - pos);
7206 tcg_gen_or_i64(tcg_right, tcg_right, tcg_tmp);
7207 }
7208
7209 /* EXT
7210 * 31 30 29 24 23 22 21 20 16 15 14 11 10 9 5 4 0
7211 * +---+---+-------------+-----+---+------+---+------+---+------+------+
7212 * | 0 | Q | 1 0 1 1 1 0 | op2 | 0 | Rm | 0 | imm4 | 0 | Rn | Rd |
7213 * +---+---+-------------+-----+---+------+---+------+---+------+------+
7214 */
disas_simd_ext(DisasContext * s,uint32_t insn)7215 static void disas_simd_ext(DisasContext *s, uint32_t insn)
7216 {
7217 int is_q = extract32(insn, 30, 1);
7218 int op2 = extract32(insn, 22, 2);
7219 int imm4 = extract32(insn, 11, 4);
7220 int rm = extract32(insn, 16, 5);
7221 int rn = extract32(insn, 5, 5);
7222 int rd = extract32(insn, 0, 5);
7223 int pos = imm4 << 3;
7224 TCGv_i64 tcg_resl, tcg_resh;
7225
7226 if (op2 != 0 || (!is_q && extract32(imm4, 3, 1))) {
7227 unallocated_encoding(s);
7228 return;
7229 }
7230
7231 if (!fp_access_check(s)) {
7232 return;
7233 }
7234
7235 tcg_resh = tcg_temp_new_i64();
7236 tcg_resl = tcg_temp_new_i64();
7237
7238 /* Vd gets bits starting at pos bits into Vm:Vn. This is
7239 * either extracting 128 bits from a 128:128 concatenation, or
7240 * extracting 64 bits from a 64:64 concatenation.
7241 */
7242 if (!is_q) {
7243 read_vec_element(s, tcg_resl, rn, 0, MO_64);
7244 if (pos != 0) {
7245 read_vec_element(s, tcg_resh, rm, 0, MO_64);
7246 do_ext64(s, tcg_resh, tcg_resl, pos);
7247 }
7248 } else {
7249 TCGv_i64 tcg_hh;
7250 typedef struct {
7251 int reg;
7252 int elt;
7253 } EltPosns;
7254 EltPosns eltposns[] = { {rn, 0}, {rn, 1}, {rm, 0}, {rm, 1} };
7255 EltPosns *elt = eltposns;
7256
7257 if (pos >= 64) {
7258 elt++;
7259 pos -= 64;
7260 }
7261
7262 read_vec_element(s, tcg_resl, elt->reg, elt->elt, MO_64);
7263 elt++;
7264 read_vec_element(s, tcg_resh, elt->reg, elt->elt, MO_64);
7265 elt++;
7266 if (pos != 0) {
7267 do_ext64(s, tcg_resh, tcg_resl, pos);
7268 tcg_hh = tcg_temp_new_i64();
7269 read_vec_element(s, tcg_hh, elt->reg, elt->elt, MO_64);
7270 do_ext64(s, tcg_hh, tcg_resh, pos);
7271 }
7272 }
7273
7274 write_vec_element(s, tcg_resl, rd, 0, MO_64);
7275 if (is_q) {
7276 write_vec_element(s, tcg_resh, rd, 1, MO_64);
7277 }
7278 clear_vec_high(s, is_q, rd);
7279 }
7280
7281 /* TBL/TBX
7282 * 31 30 29 24 23 22 21 20 16 15 14 13 12 11 10 9 5 4 0
7283 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
7284 * | 0 | Q | 0 0 1 1 1 0 | op2 | 0 | Rm | 0 | len | op | 0 0 | Rn | Rd |
7285 * +---+---+-------------+-----+---+------+---+-----+----+-----+------+------+
7286 */
disas_simd_tb(DisasContext * s,uint32_t insn)7287 static void disas_simd_tb(DisasContext *s, uint32_t insn)
7288 {
7289 int op2 = extract32(insn, 22, 2);
7290 int is_q = extract32(insn, 30, 1);
7291 int rm = extract32(insn, 16, 5);
7292 int rn = extract32(insn, 5, 5);
7293 int rd = extract32(insn, 0, 5);
7294 int is_tbx = extract32(insn, 12, 1);
7295 int len = (extract32(insn, 13, 2) + 1) * 16;
7296
7297 if (op2 != 0) {
7298 unallocated_encoding(s);
7299 return;
7300 }
7301
7302 if (!fp_access_check(s)) {
7303 return;
7304 }
7305
7306 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
7307 vec_full_reg_offset(s, rm), tcg_env,
7308 is_q ? 16 : 8, vec_full_reg_size(s),
7309 (len << 6) | (is_tbx << 5) | rn,
7310 gen_helper_simd_tblx);
7311 }
7312
7313 /* ZIP/UZP/TRN
7314 * 31 30 29 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0
7315 * +---+---+-------------+------+---+------+---+------------------+------+
7316 * | 0 | Q | 0 0 1 1 1 0 | size | 0 | Rm | 0 | opc | 1 0 | Rn | Rd |
7317 * +---+---+-------------+------+---+------+---+------------------+------+
7318 */
disas_simd_zip_trn(DisasContext * s,uint32_t insn)7319 static void disas_simd_zip_trn(DisasContext *s, uint32_t insn)
7320 {
7321 int rd = extract32(insn, 0, 5);
7322 int rn = extract32(insn, 5, 5);
7323 int rm = extract32(insn, 16, 5);
7324 int size = extract32(insn, 22, 2);
7325 /* opc field bits [1:0] indicate ZIP/UZP/TRN;
7326 * bit 2 indicates 1 vs 2 variant of the insn.
7327 */
7328 int opcode = extract32(insn, 12, 2);
7329 bool part = extract32(insn, 14, 1);
7330 bool is_q = extract32(insn, 30, 1);
7331 int esize = 8 << size;
7332 int i;
7333 int datasize = is_q ? 128 : 64;
7334 int elements = datasize / esize;
7335 TCGv_i64 tcg_res[2], tcg_ele;
7336
7337 if (opcode == 0 || (size == 3 && !is_q)) {
7338 unallocated_encoding(s);
7339 return;
7340 }
7341
7342 if (!fp_access_check(s)) {
7343 return;
7344 }
7345
7346 tcg_res[0] = tcg_temp_new_i64();
7347 tcg_res[1] = is_q ? tcg_temp_new_i64() : NULL;
7348 tcg_ele = tcg_temp_new_i64();
7349
7350 for (i = 0; i < elements; i++) {
7351 int o, w;
7352
7353 switch (opcode) {
7354 case 1: /* UZP1/2 */
7355 {
7356 int midpoint = elements / 2;
7357 if (i < midpoint) {
7358 read_vec_element(s, tcg_ele, rn, 2 * i + part, size);
7359 } else {
7360 read_vec_element(s, tcg_ele, rm,
7361 2 * (i - midpoint) + part, size);
7362 }
7363 break;
7364 }
7365 case 2: /* TRN1/2 */
7366 if (i & 1) {
7367 read_vec_element(s, tcg_ele, rm, (i & ~1) + part, size);
7368 } else {
7369 read_vec_element(s, tcg_ele, rn, (i & ~1) + part, size);
7370 }
7371 break;
7372 case 3: /* ZIP1/2 */
7373 {
7374 int base = part * elements / 2;
7375 if (i & 1) {
7376 read_vec_element(s, tcg_ele, rm, base + (i >> 1), size);
7377 } else {
7378 read_vec_element(s, tcg_ele, rn, base + (i >> 1), size);
7379 }
7380 break;
7381 }
7382 default:
7383 g_assert_not_reached();
7384 }
7385
7386 w = (i * esize) / 64;
7387 o = (i * esize) % 64;
7388 if (o == 0) {
7389 tcg_gen_mov_i64(tcg_res[w], tcg_ele);
7390 } else {
7391 tcg_gen_shli_i64(tcg_ele, tcg_ele, o);
7392 tcg_gen_or_i64(tcg_res[w], tcg_res[w], tcg_ele);
7393 }
7394 }
7395
7396 for (i = 0; i <= is_q; ++i) {
7397 write_vec_element(s, tcg_res[i], rd, i, MO_64);
7398 }
7399 clear_vec_high(s, is_q, rd);
7400 }
7401
7402 /*
7403 * do_reduction_op helper
7404 *
7405 * This mirrors the Reduce() pseudocode in the ARM ARM. It is
7406 * important for correct NaN propagation that we do these
7407 * operations in exactly the order specified by the pseudocode.
7408 *
7409 * This is a recursive function, TCG temps should be freed by the
7410 * calling function once it is done with the values.
7411 */
do_reduction_op(DisasContext * s,int fpopcode,int rn,int esize,int size,int vmap,TCGv_ptr fpst)7412 static TCGv_i32 do_reduction_op(DisasContext *s, int fpopcode, int rn,
7413 int esize, int size, int vmap, TCGv_ptr fpst)
7414 {
7415 if (esize == size) {
7416 int element;
7417 MemOp msize = esize == 16 ? MO_16 : MO_32;
7418 TCGv_i32 tcg_elem;
7419
7420 /* We should have one register left here */
7421 assert(ctpop8(vmap) == 1);
7422 element = ctz32(vmap);
7423 assert(element < 8);
7424
7425 tcg_elem = tcg_temp_new_i32();
7426 read_vec_element_i32(s, tcg_elem, rn, element, msize);
7427 return tcg_elem;
7428 } else {
7429 int bits = size / 2;
7430 int shift = ctpop8(vmap) / 2;
7431 int vmap_lo = (vmap >> shift) & vmap;
7432 int vmap_hi = (vmap & ~vmap_lo);
7433 TCGv_i32 tcg_hi, tcg_lo, tcg_res;
7434
7435 tcg_hi = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_hi, fpst);
7436 tcg_lo = do_reduction_op(s, fpopcode, rn, esize, bits, vmap_lo, fpst);
7437 tcg_res = tcg_temp_new_i32();
7438
7439 switch (fpopcode) {
7440 case 0x0c: /* fmaxnmv half-precision */
7441 gen_helper_advsimd_maxnumh(tcg_res, tcg_lo, tcg_hi, fpst);
7442 break;
7443 case 0x0f: /* fmaxv half-precision */
7444 gen_helper_advsimd_maxh(tcg_res, tcg_lo, tcg_hi, fpst);
7445 break;
7446 case 0x1c: /* fminnmv half-precision */
7447 gen_helper_advsimd_minnumh(tcg_res, tcg_lo, tcg_hi, fpst);
7448 break;
7449 case 0x1f: /* fminv half-precision */
7450 gen_helper_advsimd_minh(tcg_res, tcg_lo, tcg_hi, fpst);
7451 break;
7452 case 0x2c: /* fmaxnmv */
7453 gen_helper_vfp_maxnums(tcg_res, tcg_lo, tcg_hi, fpst);
7454 break;
7455 case 0x2f: /* fmaxv */
7456 gen_helper_vfp_maxs(tcg_res, tcg_lo, tcg_hi, fpst);
7457 break;
7458 case 0x3c: /* fminnmv */
7459 gen_helper_vfp_minnums(tcg_res, tcg_lo, tcg_hi, fpst);
7460 break;
7461 case 0x3f: /* fminv */
7462 gen_helper_vfp_mins(tcg_res, tcg_lo, tcg_hi, fpst);
7463 break;
7464 default:
7465 g_assert_not_reached();
7466 }
7467 return tcg_res;
7468 }
7469 }
7470
7471 /* AdvSIMD across lanes
7472 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
7473 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
7474 * | 0 | Q | U | 0 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd |
7475 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
7476 */
disas_simd_across_lanes(DisasContext * s,uint32_t insn)7477 static void disas_simd_across_lanes(DisasContext *s, uint32_t insn)
7478 {
7479 int rd = extract32(insn, 0, 5);
7480 int rn = extract32(insn, 5, 5);
7481 int size = extract32(insn, 22, 2);
7482 int opcode = extract32(insn, 12, 5);
7483 bool is_q = extract32(insn, 30, 1);
7484 bool is_u = extract32(insn, 29, 1);
7485 bool is_fp = false;
7486 bool is_min = false;
7487 int esize;
7488 int elements;
7489 int i;
7490 TCGv_i64 tcg_res, tcg_elt;
7491
7492 switch (opcode) {
7493 case 0x1b: /* ADDV */
7494 if (is_u) {
7495 unallocated_encoding(s);
7496 return;
7497 }
7498 /* fall through */
7499 case 0x3: /* SADDLV, UADDLV */
7500 case 0xa: /* SMAXV, UMAXV */
7501 case 0x1a: /* SMINV, UMINV */
7502 if (size == 3 || (size == 2 && !is_q)) {
7503 unallocated_encoding(s);
7504 return;
7505 }
7506 break;
7507 case 0xc: /* FMAXNMV, FMINNMV */
7508 case 0xf: /* FMAXV, FMINV */
7509 /* Bit 1 of size field encodes min vs max and the actual size
7510 * depends on the encoding of the U bit. If not set (and FP16
7511 * enabled) then we do half-precision float instead of single
7512 * precision.
7513 */
7514 is_min = extract32(size, 1, 1);
7515 is_fp = true;
7516 if (!is_u && dc_isar_feature(aa64_fp16, s)) {
7517 size = 1;
7518 } else if (!is_u || !is_q || extract32(size, 0, 1)) {
7519 unallocated_encoding(s);
7520 return;
7521 } else {
7522 size = 2;
7523 }
7524 break;
7525 default:
7526 unallocated_encoding(s);
7527 return;
7528 }
7529
7530 if (!fp_access_check(s)) {
7531 return;
7532 }
7533
7534 esize = 8 << size;
7535 elements = (is_q ? 128 : 64) / esize;
7536
7537 tcg_res = tcg_temp_new_i64();
7538 tcg_elt = tcg_temp_new_i64();
7539
7540 /* These instructions operate across all lanes of a vector
7541 * to produce a single result. We can guarantee that a 64
7542 * bit intermediate is sufficient:
7543 * + for [US]ADDLV the maximum element size is 32 bits, and
7544 * the result type is 64 bits
7545 * + for FMAX*V, FMIN*V, ADDV the intermediate type is the
7546 * same as the element size, which is 32 bits at most
7547 * For the integer operations we can choose to work at 64
7548 * or 32 bits and truncate at the end; for simplicity
7549 * we use 64 bits always. The floating point
7550 * ops do require 32 bit intermediates, though.
7551 */
7552 if (!is_fp) {
7553 read_vec_element(s, tcg_res, rn, 0, size | (is_u ? 0 : MO_SIGN));
7554
7555 for (i = 1; i < elements; i++) {
7556 read_vec_element(s, tcg_elt, rn, i, size | (is_u ? 0 : MO_SIGN));
7557
7558 switch (opcode) {
7559 case 0x03: /* SADDLV / UADDLV */
7560 case 0x1b: /* ADDV */
7561 tcg_gen_add_i64(tcg_res, tcg_res, tcg_elt);
7562 break;
7563 case 0x0a: /* SMAXV / UMAXV */
7564 if (is_u) {
7565 tcg_gen_umax_i64(tcg_res, tcg_res, tcg_elt);
7566 } else {
7567 tcg_gen_smax_i64(tcg_res, tcg_res, tcg_elt);
7568 }
7569 break;
7570 case 0x1a: /* SMINV / UMINV */
7571 if (is_u) {
7572 tcg_gen_umin_i64(tcg_res, tcg_res, tcg_elt);
7573 } else {
7574 tcg_gen_smin_i64(tcg_res, tcg_res, tcg_elt);
7575 }
7576 break;
7577 default:
7578 g_assert_not_reached();
7579 }
7580
7581 }
7582 } else {
7583 /* Floating point vector reduction ops which work across 32
7584 * bit (single) or 16 bit (half-precision) intermediates.
7585 * Note that correct NaN propagation requires that we do these
7586 * operations in exactly the order specified by the pseudocode.
7587 */
7588 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
7589 int fpopcode = opcode | is_min << 4 | is_u << 5;
7590 int vmap = (1 << elements) - 1;
7591 TCGv_i32 tcg_res32 = do_reduction_op(s, fpopcode, rn, esize,
7592 (is_q ? 128 : 64), vmap, fpst);
7593 tcg_gen_extu_i32_i64(tcg_res, tcg_res32);
7594 }
7595
7596 /* Now truncate the result to the width required for the final output */
7597 if (opcode == 0x03) {
7598 /* SADDLV, UADDLV: result is 2*esize */
7599 size++;
7600 }
7601
7602 switch (size) {
7603 case 0:
7604 tcg_gen_ext8u_i64(tcg_res, tcg_res);
7605 break;
7606 case 1:
7607 tcg_gen_ext16u_i64(tcg_res, tcg_res);
7608 break;
7609 case 2:
7610 tcg_gen_ext32u_i64(tcg_res, tcg_res);
7611 break;
7612 case 3:
7613 break;
7614 default:
7615 g_assert_not_reached();
7616 }
7617
7618 write_fp_dreg(s, rd, tcg_res);
7619 }
7620
7621 /* DUP (Element, Vector)
7622 *
7623 * 31 30 29 21 20 16 15 10 9 5 4 0
7624 * +---+---+-------------------+--------+-------------+------+------+
7625 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd |
7626 * +---+---+-------------------+--------+-------------+------+------+
7627 *
7628 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7629 */
handle_simd_dupe(DisasContext * s,int is_q,int rd,int rn,int imm5)7630 static void handle_simd_dupe(DisasContext *s, int is_q, int rd, int rn,
7631 int imm5)
7632 {
7633 int size = ctz32(imm5);
7634 int index;
7635
7636 if (size > 3 || (size == 3 && !is_q)) {
7637 unallocated_encoding(s);
7638 return;
7639 }
7640
7641 if (!fp_access_check(s)) {
7642 return;
7643 }
7644
7645 index = imm5 >> (size + 1);
7646 tcg_gen_gvec_dup_mem(size, vec_full_reg_offset(s, rd),
7647 vec_reg_offset(s, rn, index, size),
7648 is_q ? 16 : 8, vec_full_reg_size(s));
7649 }
7650
7651 /* DUP (element, scalar)
7652 * 31 21 20 16 15 10 9 5 4 0
7653 * +-----------------------+--------+-------------+------+------+
7654 * | 0 1 0 1 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 0 1 | Rn | Rd |
7655 * +-----------------------+--------+-------------+------+------+
7656 */
handle_simd_dupes(DisasContext * s,int rd,int rn,int imm5)7657 static void handle_simd_dupes(DisasContext *s, int rd, int rn,
7658 int imm5)
7659 {
7660 int size = ctz32(imm5);
7661 int index;
7662 TCGv_i64 tmp;
7663
7664 if (size > 3) {
7665 unallocated_encoding(s);
7666 return;
7667 }
7668
7669 if (!fp_access_check(s)) {
7670 return;
7671 }
7672
7673 index = imm5 >> (size + 1);
7674
7675 /* This instruction just extracts the specified element and
7676 * zero-extends it into the bottom of the destination register.
7677 */
7678 tmp = tcg_temp_new_i64();
7679 read_vec_element(s, tmp, rn, index, size);
7680 write_fp_dreg(s, rd, tmp);
7681 }
7682
7683 /* DUP (General)
7684 *
7685 * 31 30 29 21 20 16 15 10 9 5 4 0
7686 * +---+---+-------------------+--------+-------------+------+------+
7687 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 0 1 1 | Rn | Rd |
7688 * +---+---+-------------------+--------+-------------+------+------+
7689 *
7690 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7691 */
handle_simd_dupg(DisasContext * s,int is_q,int rd,int rn,int imm5)7692 static void handle_simd_dupg(DisasContext *s, int is_q, int rd, int rn,
7693 int imm5)
7694 {
7695 int size = ctz32(imm5);
7696 uint32_t dofs, oprsz, maxsz;
7697
7698 if (size > 3 || ((size == 3) && !is_q)) {
7699 unallocated_encoding(s);
7700 return;
7701 }
7702
7703 if (!fp_access_check(s)) {
7704 return;
7705 }
7706
7707 dofs = vec_full_reg_offset(s, rd);
7708 oprsz = is_q ? 16 : 8;
7709 maxsz = vec_full_reg_size(s);
7710
7711 tcg_gen_gvec_dup_i64(size, dofs, oprsz, maxsz, cpu_reg(s, rn));
7712 }
7713
7714 /* INS (Element)
7715 *
7716 * 31 21 20 16 15 14 11 10 9 5 4 0
7717 * +-----------------------+--------+------------+---+------+------+
7718 * | 0 1 1 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7719 * +-----------------------+--------+------------+---+------+------+
7720 *
7721 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7722 * index: encoded in imm5<4:size+1>
7723 */
handle_simd_inse(DisasContext * s,int rd,int rn,int imm4,int imm5)7724 static void handle_simd_inse(DisasContext *s, int rd, int rn,
7725 int imm4, int imm5)
7726 {
7727 int size = ctz32(imm5);
7728 int src_index, dst_index;
7729 TCGv_i64 tmp;
7730
7731 if (size > 3) {
7732 unallocated_encoding(s);
7733 return;
7734 }
7735
7736 if (!fp_access_check(s)) {
7737 return;
7738 }
7739
7740 dst_index = extract32(imm5, 1+size, 5);
7741 src_index = extract32(imm4, size, 4);
7742
7743 tmp = tcg_temp_new_i64();
7744
7745 read_vec_element(s, tmp, rn, src_index, size);
7746 write_vec_element(s, tmp, rd, dst_index, size);
7747
7748 /* INS is considered a 128-bit write for SVE. */
7749 clear_vec_high(s, true, rd);
7750 }
7751
7752
7753 /* INS (General)
7754 *
7755 * 31 21 20 16 15 10 9 5 4 0
7756 * +-----------------------+--------+-------------+------+------+
7757 * | 0 1 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 0 1 1 1 | Rn | Rd |
7758 * +-----------------------+--------+-------------+------+------+
7759 *
7760 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7761 * index: encoded in imm5<4:size+1>
7762 */
handle_simd_insg(DisasContext * s,int rd,int rn,int imm5)7763 static void handle_simd_insg(DisasContext *s, int rd, int rn, int imm5)
7764 {
7765 int size = ctz32(imm5);
7766 int idx;
7767
7768 if (size > 3) {
7769 unallocated_encoding(s);
7770 return;
7771 }
7772
7773 if (!fp_access_check(s)) {
7774 return;
7775 }
7776
7777 idx = extract32(imm5, 1 + size, 4 - size);
7778 write_vec_element(s, cpu_reg(s, rn), rd, idx, size);
7779
7780 /* INS is considered a 128-bit write for SVE. */
7781 clear_vec_high(s, true, rd);
7782 }
7783
7784 /*
7785 * UMOV (General)
7786 * SMOV (General)
7787 *
7788 * 31 30 29 21 20 16 15 12 10 9 5 4 0
7789 * +---+---+-------------------+--------+-------------+------+------+
7790 * | 0 | Q | 0 0 1 1 1 0 0 0 0 | imm5 | 0 0 1 U 1 1 | Rn | Rd |
7791 * +---+---+-------------------+--------+-------------+------+------+
7792 *
7793 * U: unsigned when set
7794 * size: encoded in imm5 (see ARM ARM LowestSetBit())
7795 */
handle_simd_umov_smov(DisasContext * s,int is_q,int is_signed,int rn,int rd,int imm5)7796 static void handle_simd_umov_smov(DisasContext *s, int is_q, int is_signed,
7797 int rn, int rd, int imm5)
7798 {
7799 int size = ctz32(imm5);
7800 int element;
7801 TCGv_i64 tcg_rd;
7802
7803 /* Check for UnallocatedEncodings */
7804 if (is_signed) {
7805 if (size > 2 || (size == 2 && !is_q)) {
7806 unallocated_encoding(s);
7807 return;
7808 }
7809 } else {
7810 if (size > 3
7811 || (size < 3 && is_q)
7812 || (size == 3 && !is_q)) {
7813 unallocated_encoding(s);
7814 return;
7815 }
7816 }
7817
7818 if (!fp_access_check(s)) {
7819 return;
7820 }
7821
7822 element = extract32(imm5, 1+size, 4);
7823
7824 tcg_rd = cpu_reg(s, rd);
7825 read_vec_element(s, tcg_rd, rn, element, size | (is_signed ? MO_SIGN : 0));
7826 if (is_signed && !is_q) {
7827 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
7828 }
7829 }
7830
7831 /* AdvSIMD copy
7832 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0
7833 * +---+---+----+-----------------+------+---+------+---+------+------+
7834 * | 0 | Q | op | 0 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7835 * +---+---+----+-----------------+------+---+------+---+------+------+
7836 */
disas_simd_copy(DisasContext * s,uint32_t insn)7837 static void disas_simd_copy(DisasContext *s, uint32_t insn)
7838 {
7839 int rd = extract32(insn, 0, 5);
7840 int rn = extract32(insn, 5, 5);
7841 int imm4 = extract32(insn, 11, 4);
7842 int op = extract32(insn, 29, 1);
7843 int is_q = extract32(insn, 30, 1);
7844 int imm5 = extract32(insn, 16, 5);
7845
7846 if (op) {
7847 if (is_q) {
7848 /* INS (element) */
7849 handle_simd_inse(s, rd, rn, imm4, imm5);
7850 } else {
7851 unallocated_encoding(s);
7852 }
7853 } else {
7854 switch (imm4) {
7855 case 0:
7856 /* DUP (element - vector) */
7857 handle_simd_dupe(s, is_q, rd, rn, imm5);
7858 break;
7859 case 1:
7860 /* DUP (general) */
7861 handle_simd_dupg(s, is_q, rd, rn, imm5);
7862 break;
7863 case 3:
7864 if (is_q) {
7865 /* INS (general) */
7866 handle_simd_insg(s, rd, rn, imm5);
7867 } else {
7868 unallocated_encoding(s);
7869 }
7870 break;
7871 case 5:
7872 case 7:
7873 /* UMOV/SMOV (is_q indicates 32/64; imm4 indicates signedness) */
7874 handle_simd_umov_smov(s, is_q, (imm4 == 5), rn, rd, imm5);
7875 break;
7876 default:
7877 unallocated_encoding(s);
7878 break;
7879 }
7880 }
7881 }
7882
7883 /* AdvSIMD modified immediate
7884 * 31 30 29 28 19 18 16 15 12 11 10 9 5 4 0
7885 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
7886 * | 0 | Q | op | 0 1 1 1 1 0 0 0 0 0 | abc | cmode | o2 | 1 | defgh | Rd |
7887 * +---+---+----+---------------------+-----+-------+----+---+-------+------+
7888 *
7889 * There are a number of operations that can be carried out here:
7890 * MOVI - move (shifted) imm into register
7891 * MVNI - move inverted (shifted) imm into register
7892 * ORR - bitwise OR of (shifted) imm with register
7893 * BIC - bitwise clear of (shifted) imm with register
7894 * With ARMv8.2 we also have:
7895 * FMOV half-precision
7896 */
disas_simd_mod_imm(DisasContext * s,uint32_t insn)7897 static void disas_simd_mod_imm(DisasContext *s, uint32_t insn)
7898 {
7899 int rd = extract32(insn, 0, 5);
7900 int cmode = extract32(insn, 12, 4);
7901 int o2 = extract32(insn, 11, 1);
7902 uint64_t abcdefgh = extract32(insn, 5, 5) | (extract32(insn, 16, 3) << 5);
7903 bool is_neg = extract32(insn, 29, 1);
7904 bool is_q = extract32(insn, 30, 1);
7905 uint64_t imm = 0;
7906
7907 if (o2 != 0 || ((cmode == 0xf) && is_neg && !is_q)) {
7908 /* Check for FMOV (vector, immediate) - half-precision */
7909 if (!(dc_isar_feature(aa64_fp16, s) && o2 && cmode == 0xf)) {
7910 unallocated_encoding(s);
7911 return;
7912 }
7913 }
7914
7915 if (!fp_access_check(s)) {
7916 return;
7917 }
7918
7919 if (cmode == 15 && o2 && !is_neg) {
7920 /* FMOV (vector, immediate) - half-precision */
7921 imm = vfp_expand_imm(MO_16, abcdefgh);
7922 /* now duplicate across the lanes */
7923 imm = dup_const(MO_16, imm);
7924 } else {
7925 imm = asimd_imm_const(abcdefgh, cmode, is_neg);
7926 }
7927
7928 if (!((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9)) {
7929 /* MOVI or MVNI, with MVNI negation handled above. */
7930 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), is_q ? 16 : 8,
7931 vec_full_reg_size(s), imm);
7932 } else {
7933 /* ORR or BIC, with BIC negation to AND handled above. */
7934 if (is_neg) {
7935 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_andi, MO_64);
7936 } else {
7937 gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_ori, MO_64);
7938 }
7939 }
7940 }
7941
7942 /* AdvSIMD scalar copy
7943 * 31 30 29 28 21 20 16 15 14 11 10 9 5 4 0
7944 * +-----+----+-----------------+------+---+------+---+------+------+
7945 * | 0 1 | op | 1 1 1 1 0 0 0 0 | imm5 | 0 | imm4 | 1 | Rn | Rd |
7946 * +-----+----+-----------------+------+---+------+---+------+------+
7947 */
disas_simd_scalar_copy(DisasContext * s,uint32_t insn)7948 static void disas_simd_scalar_copy(DisasContext *s, uint32_t insn)
7949 {
7950 int rd = extract32(insn, 0, 5);
7951 int rn = extract32(insn, 5, 5);
7952 int imm4 = extract32(insn, 11, 4);
7953 int imm5 = extract32(insn, 16, 5);
7954 int op = extract32(insn, 29, 1);
7955
7956 if (op != 0 || imm4 != 0) {
7957 unallocated_encoding(s);
7958 return;
7959 }
7960
7961 /* DUP (element, scalar) */
7962 handle_simd_dupes(s, rd, rn, imm5);
7963 }
7964
7965 /* AdvSIMD scalar pairwise
7966 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
7967 * +-----+---+-----------+------+-----------+--------+-----+------+------+
7968 * | 0 1 | U | 1 1 1 1 0 | size | 1 1 0 0 0 | opcode | 1 0 | Rn | Rd |
7969 * +-----+---+-----------+------+-----------+--------+-----+------+------+
7970 */
disas_simd_scalar_pairwise(DisasContext * s,uint32_t insn)7971 static void disas_simd_scalar_pairwise(DisasContext *s, uint32_t insn)
7972 {
7973 int u = extract32(insn, 29, 1);
7974 int size = extract32(insn, 22, 2);
7975 int opcode = extract32(insn, 12, 5);
7976 int rn = extract32(insn, 5, 5);
7977 int rd = extract32(insn, 0, 5);
7978 TCGv_ptr fpst;
7979
7980 /* For some ops (the FP ones), size[1] is part of the encoding.
7981 * For ADDP strictly it is not but size[1] is always 1 for valid
7982 * encodings.
7983 */
7984 opcode |= (extract32(size, 1, 1) << 5);
7985
7986 switch (opcode) {
7987 case 0x3b: /* ADDP */
7988 if (u || size != 3) {
7989 unallocated_encoding(s);
7990 return;
7991 }
7992 if (!fp_access_check(s)) {
7993 return;
7994 }
7995
7996 fpst = NULL;
7997 break;
7998 case 0xc: /* FMAXNMP */
7999 case 0xd: /* FADDP */
8000 case 0xf: /* FMAXP */
8001 case 0x2c: /* FMINNMP */
8002 case 0x2f: /* FMINP */
8003 /* FP op, size[0] is 32 or 64 bit*/
8004 if (!u) {
8005 if (!dc_isar_feature(aa64_fp16, s)) {
8006 unallocated_encoding(s);
8007 return;
8008 } else {
8009 size = MO_16;
8010 }
8011 } else {
8012 size = extract32(size, 0, 1) ? MO_64 : MO_32;
8013 }
8014
8015 if (!fp_access_check(s)) {
8016 return;
8017 }
8018
8019 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8020 break;
8021 default:
8022 unallocated_encoding(s);
8023 return;
8024 }
8025
8026 if (size == MO_64) {
8027 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
8028 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
8029 TCGv_i64 tcg_res = tcg_temp_new_i64();
8030
8031 read_vec_element(s, tcg_op1, rn, 0, MO_64);
8032 read_vec_element(s, tcg_op2, rn, 1, MO_64);
8033
8034 switch (opcode) {
8035 case 0x3b: /* ADDP */
8036 tcg_gen_add_i64(tcg_res, tcg_op1, tcg_op2);
8037 break;
8038 case 0xc: /* FMAXNMP */
8039 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
8040 break;
8041 case 0xd: /* FADDP */
8042 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
8043 break;
8044 case 0xf: /* FMAXP */
8045 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
8046 break;
8047 case 0x2c: /* FMINNMP */
8048 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
8049 break;
8050 case 0x2f: /* FMINP */
8051 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
8052 break;
8053 default:
8054 g_assert_not_reached();
8055 }
8056
8057 write_fp_dreg(s, rd, tcg_res);
8058 } else {
8059 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
8060 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
8061 TCGv_i32 tcg_res = tcg_temp_new_i32();
8062
8063 read_vec_element_i32(s, tcg_op1, rn, 0, size);
8064 read_vec_element_i32(s, tcg_op2, rn, 1, size);
8065
8066 if (size == MO_16) {
8067 switch (opcode) {
8068 case 0xc: /* FMAXNMP */
8069 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
8070 break;
8071 case 0xd: /* FADDP */
8072 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
8073 break;
8074 case 0xf: /* FMAXP */
8075 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
8076 break;
8077 case 0x2c: /* FMINNMP */
8078 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
8079 break;
8080 case 0x2f: /* FMINP */
8081 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
8082 break;
8083 default:
8084 g_assert_not_reached();
8085 }
8086 } else {
8087 switch (opcode) {
8088 case 0xc: /* FMAXNMP */
8089 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
8090 break;
8091 case 0xd: /* FADDP */
8092 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
8093 break;
8094 case 0xf: /* FMAXP */
8095 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
8096 break;
8097 case 0x2c: /* FMINNMP */
8098 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
8099 break;
8100 case 0x2f: /* FMINP */
8101 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
8102 break;
8103 default:
8104 g_assert_not_reached();
8105 }
8106 }
8107
8108 write_fp_sreg(s, rd, tcg_res);
8109 }
8110 }
8111
8112 /*
8113 * Common SSHR[RA]/USHR[RA] - Shift right (optional rounding/accumulate)
8114 *
8115 * This code is handles the common shifting code and is used by both
8116 * the vector and scalar code.
8117 */
handle_shri_with_rndacc(TCGv_i64 tcg_res,TCGv_i64 tcg_src,TCGv_i64 tcg_rnd,bool accumulate,bool is_u,int size,int shift)8118 static void handle_shri_with_rndacc(TCGv_i64 tcg_res, TCGv_i64 tcg_src,
8119 TCGv_i64 tcg_rnd, bool accumulate,
8120 bool is_u, int size, int shift)
8121 {
8122 bool extended_result = false;
8123 bool round = tcg_rnd != NULL;
8124 int ext_lshift = 0;
8125 TCGv_i64 tcg_src_hi;
8126
8127 if (round && size == 3) {
8128 extended_result = true;
8129 ext_lshift = 64 - shift;
8130 tcg_src_hi = tcg_temp_new_i64();
8131 } else if (shift == 64) {
8132 if (!accumulate && is_u) {
8133 /* result is zero */
8134 tcg_gen_movi_i64(tcg_res, 0);
8135 return;
8136 }
8137 }
8138
8139 /* Deal with the rounding step */
8140 if (round) {
8141 if (extended_result) {
8142 TCGv_i64 tcg_zero = tcg_constant_i64(0);
8143 if (!is_u) {
8144 /* take care of sign extending tcg_res */
8145 tcg_gen_sari_i64(tcg_src_hi, tcg_src, 63);
8146 tcg_gen_add2_i64(tcg_src, tcg_src_hi,
8147 tcg_src, tcg_src_hi,
8148 tcg_rnd, tcg_zero);
8149 } else {
8150 tcg_gen_add2_i64(tcg_src, tcg_src_hi,
8151 tcg_src, tcg_zero,
8152 tcg_rnd, tcg_zero);
8153 }
8154 } else {
8155 tcg_gen_add_i64(tcg_src, tcg_src, tcg_rnd);
8156 }
8157 }
8158
8159 /* Now do the shift right */
8160 if (round && extended_result) {
8161 /* extended case, >64 bit precision required */
8162 if (ext_lshift == 0) {
8163 /* special case, only high bits matter */
8164 tcg_gen_mov_i64(tcg_src, tcg_src_hi);
8165 } else {
8166 tcg_gen_shri_i64(tcg_src, tcg_src, shift);
8167 tcg_gen_shli_i64(tcg_src_hi, tcg_src_hi, ext_lshift);
8168 tcg_gen_or_i64(tcg_src, tcg_src, tcg_src_hi);
8169 }
8170 } else {
8171 if (is_u) {
8172 if (shift == 64) {
8173 /* essentially shifting in 64 zeros */
8174 tcg_gen_movi_i64(tcg_src, 0);
8175 } else {
8176 tcg_gen_shri_i64(tcg_src, tcg_src, shift);
8177 }
8178 } else {
8179 if (shift == 64) {
8180 /* effectively extending the sign-bit */
8181 tcg_gen_sari_i64(tcg_src, tcg_src, 63);
8182 } else {
8183 tcg_gen_sari_i64(tcg_src, tcg_src, shift);
8184 }
8185 }
8186 }
8187
8188 if (accumulate) {
8189 tcg_gen_add_i64(tcg_res, tcg_res, tcg_src);
8190 } else {
8191 tcg_gen_mov_i64(tcg_res, tcg_src);
8192 }
8193 }
8194
8195 /* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */
handle_scalar_simd_shri(DisasContext * s,bool is_u,int immh,int immb,int opcode,int rn,int rd)8196 static void handle_scalar_simd_shri(DisasContext *s,
8197 bool is_u, int immh, int immb,
8198 int opcode, int rn, int rd)
8199 {
8200 const int size = 3;
8201 int immhb = immh << 3 | immb;
8202 int shift = 2 * (8 << size) - immhb;
8203 bool accumulate = false;
8204 bool round = false;
8205 bool insert = false;
8206 TCGv_i64 tcg_rn;
8207 TCGv_i64 tcg_rd;
8208 TCGv_i64 tcg_round;
8209
8210 if (!extract32(immh, 3, 1)) {
8211 unallocated_encoding(s);
8212 return;
8213 }
8214
8215 if (!fp_access_check(s)) {
8216 return;
8217 }
8218
8219 switch (opcode) {
8220 case 0x02: /* SSRA / USRA (accumulate) */
8221 accumulate = true;
8222 break;
8223 case 0x04: /* SRSHR / URSHR (rounding) */
8224 round = true;
8225 break;
8226 case 0x06: /* SRSRA / URSRA (accum + rounding) */
8227 accumulate = round = true;
8228 break;
8229 case 0x08: /* SRI */
8230 insert = true;
8231 break;
8232 }
8233
8234 if (round) {
8235 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
8236 } else {
8237 tcg_round = NULL;
8238 }
8239
8240 tcg_rn = read_fp_dreg(s, rn);
8241 tcg_rd = (accumulate || insert) ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
8242
8243 if (insert) {
8244 /* shift count same as element size is valid but does nothing;
8245 * special case to avoid potential shift by 64.
8246 */
8247 int esize = 8 << size;
8248 if (shift != esize) {
8249 tcg_gen_shri_i64(tcg_rn, tcg_rn, shift);
8250 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, 0, esize - shift);
8251 }
8252 } else {
8253 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
8254 accumulate, is_u, size, shift);
8255 }
8256
8257 write_fp_dreg(s, rd, tcg_rd);
8258 }
8259
8260 /* SHL/SLI - Scalar shift left */
handle_scalar_simd_shli(DisasContext * s,bool insert,int immh,int immb,int opcode,int rn,int rd)8261 static void handle_scalar_simd_shli(DisasContext *s, bool insert,
8262 int immh, int immb, int opcode,
8263 int rn, int rd)
8264 {
8265 int size = 32 - clz32(immh) - 1;
8266 int immhb = immh << 3 | immb;
8267 int shift = immhb - (8 << size);
8268 TCGv_i64 tcg_rn;
8269 TCGv_i64 tcg_rd;
8270
8271 if (!extract32(immh, 3, 1)) {
8272 unallocated_encoding(s);
8273 return;
8274 }
8275
8276 if (!fp_access_check(s)) {
8277 return;
8278 }
8279
8280 tcg_rn = read_fp_dreg(s, rn);
8281 tcg_rd = insert ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
8282
8283 if (insert) {
8284 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, shift, 64 - shift);
8285 } else {
8286 tcg_gen_shli_i64(tcg_rd, tcg_rn, shift);
8287 }
8288
8289 write_fp_dreg(s, rd, tcg_rd);
8290 }
8291
8292 /* SQSHRN/SQSHRUN - Saturating (signed/unsigned) shift right with
8293 * (signed/unsigned) narrowing */
handle_vec_simd_sqshrn(DisasContext * s,bool is_scalar,bool is_q,bool is_u_shift,bool is_u_narrow,int immh,int immb,int opcode,int rn,int rd)8294 static void handle_vec_simd_sqshrn(DisasContext *s, bool is_scalar, bool is_q,
8295 bool is_u_shift, bool is_u_narrow,
8296 int immh, int immb, int opcode,
8297 int rn, int rd)
8298 {
8299 int immhb = immh << 3 | immb;
8300 int size = 32 - clz32(immh) - 1;
8301 int esize = 8 << size;
8302 int shift = (2 * esize) - immhb;
8303 int elements = is_scalar ? 1 : (64 / esize);
8304 bool round = extract32(opcode, 0, 1);
8305 MemOp ldop = (size + 1) | (is_u_shift ? 0 : MO_SIGN);
8306 TCGv_i64 tcg_rn, tcg_rd, tcg_round;
8307 TCGv_i32 tcg_rd_narrowed;
8308 TCGv_i64 tcg_final;
8309
8310 static NeonGenNarrowEnvFn * const signed_narrow_fns[4][2] = {
8311 { gen_helper_neon_narrow_sat_s8,
8312 gen_helper_neon_unarrow_sat8 },
8313 { gen_helper_neon_narrow_sat_s16,
8314 gen_helper_neon_unarrow_sat16 },
8315 { gen_helper_neon_narrow_sat_s32,
8316 gen_helper_neon_unarrow_sat32 },
8317 { NULL, NULL },
8318 };
8319 static NeonGenNarrowEnvFn * const unsigned_narrow_fns[4] = {
8320 gen_helper_neon_narrow_sat_u8,
8321 gen_helper_neon_narrow_sat_u16,
8322 gen_helper_neon_narrow_sat_u32,
8323 NULL
8324 };
8325 NeonGenNarrowEnvFn *narrowfn;
8326
8327 int i;
8328
8329 assert(size < 4);
8330
8331 if (extract32(immh, 3, 1)) {
8332 unallocated_encoding(s);
8333 return;
8334 }
8335
8336 if (!fp_access_check(s)) {
8337 return;
8338 }
8339
8340 if (is_u_shift) {
8341 narrowfn = unsigned_narrow_fns[size];
8342 } else {
8343 narrowfn = signed_narrow_fns[size][is_u_narrow ? 1 : 0];
8344 }
8345
8346 tcg_rn = tcg_temp_new_i64();
8347 tcg_rd = tcg_temp_new_i64();
8348 tcg_rd_narrowed = tcg_temp_new_i32();
8349 tcg_final = tcg_temp_new_i64();
8350
8351 if (round) {
8352 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
8353 } else {
8354 tcg_round = NULL;
8355 }
8356
8357 for (i = 0; i < elements; i++) {
8358 read_vec_element(s, tcg_rn, rn, i, ldop);
8359 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
8360 false, is_u_shift, size+1, shift);
8361 narrowfn(tcg_rd_narrowed, tcg_env, tcg_rd);
8362 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd_narrowed);
8363 if (i == 0) {
8364 tcg_gen_extract_i64(tcg_final, tcg_rd, 0, esize);
8365 } else {
8366 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize);
8367 }
8368 }
8369
8370 if (!is_q) {
8371 write_vec_element(s, tcg_final, rd, 0, MO_64);
8372 } else {
8373 write_vec_element(s, tcg_final, rd, 1, MO_64);
8374 }
8375 clear_vec_high(s, is_q, rd);
8376 }
8377
8378 /* SQSHLU, UQSHL, SQSHL: saturating left shifts */
handle_simd_qshl(DisasContext * s,bool scalar,bool is_q,bool src_unsigned,bool dst_unsigned,int immh,int immb,int rn,int rd)8379 static void handle_simd_qshl(DisasContext *s, bool scalar, bool is_q,
8380 bool src_unsigned, bool dst_unsigned,
8381 int immh, int immb, int rn, int rd)
8382 {
8383 int immhb = immh << 3 | immb;
8384 int size = 32 - clz32(immh) - 1;
8385 int shift = immhb - (8 << size);
8386 int pass;
8387
8388 assert(immh != 0);
8389 assert(!(scalar && is_q));
8390
8391 if (!scalar) {
8392 if (!is_q && extract32(immh, 3, 1)) {
8393 unallocated_encoding(s);
8394 return;
8395 }
8396
8397 /* Since we use the variable-shift helpers we must
8398 * replicate the shift count into each element of
8399 * the tcg_shift value.
8400 */
8401 switch (size) {
8402 case 0:
8403 shift |= shift << 8;
8404 /* fall through */
8405 case 1:
8406 shift |= shift << 16;
8407 break;
8408 case 2:
8409 case 3:
8410 break;
8411 default:
8412 g_assert_not_reached();
8413 }
8414 }
8415
8416 if (!fp_access_check(s)) {
8417 return;
8418 }
8419
8420 if (size == 3) {
8421 TCGv_i64 tcg_shift = tcg_constant_i64(shift);
8422 static NeonGenTwo64OpEnvFn * const fns[2][2] = {
8423 { gen_helper_neon_qshl_s64, gen_helper_neon_qshlu_s64 },
8424 { NULL, gen_helper_neon_qshl_u64 },
8425 };
8426 NeonGenTwo64OpEnvFn *genfn = fns[src_unsigned][dst_unsigned];
8427 int maxpass = is_q ? 2 : 1;
8428
8429 for (pass = 0; pass < maxpass; pass++) {
8430 TCGv_i64 tcg_op = tcg_temp_new_i64();
8431
8432 read_vec_element(s, tcg_op, rn, pass, MO_64);
8433 genfn(tcg_op, tcg_env, tcg_op, tcg_shift);
8434 write_vec_element(s, tcg_op, rd, pass, MO_64);
8435 }
8436 clear_vec_high(s, is_q, rd);
8437 } else {
8438 TCGv_i32 tcg_shift = tcg_constant_i32(shift);
8439 static NeonGenTwoOpEnvFn * const fns[2][2][3] = {
8440 {
8441 { gen_helper_neon_qshl_s8,
8442 gen_helper_neon_qshl_s16,
8443 gen_helper_neon_qshl_s32 },
8444 { gen_helper_neon_qshlu_s8,
8445 gen_helper_neon_qshlu_s16,
8446 gen_helper_neon_qshlu_s32 }
8447 }, {
8448 { NULL, NULL, NULL },
8449 { gen_helper_neon_qshl_u8,
8450 gen_helper_neon_qshl_u16,
8451 gen_helper_neon_qshl_u32 }
8452 }
8453 };
8454 NeonGenTwoOpEnvFn *genfn = fns[src_unsigned][dst_unsigned][size];
8455 MemOp memop = scalar ? size : MO_32;
8456 int maxpass = scalar ? 1 : is_q ? 4 : 2;
8457
8458 for (pass = 0; pass < maxpass; pass++) {
8459 TCGv_i32 tcg_op = tcg_temp_new_i32();
8460
8461 read_vec_element_i32(s, tcg_op, rn, pass, memop);
8462 genfn(tcg_op, tcg_env, tcg_op, tcg_shift);
8463 if (scalar) {
8464 switch (size) {
8465 case 0:
8466 tcg_gen_ext8u_i32(tcg_op, tcg_op);
8467 break;
8468 case 1:
8469 tcg_gen_ext16u_i32(tcg_op, tcg_op);
8470 break;
8471 case 2:
8472 break;
8473 default:
8474 g_assert_not_reached();
8475 }
8476 write_fp_sreg(s, rd, tcg_op);
8477 } else {
8478 write_vec_element_i32(s, tcg_op, rd, pass, MO_32);
8479 }
8480 }
8481
8482 if (!scalar) {
8483 clear_vec_high(s, is_q, rd);
8484 }
8485 }
8486 }
8487
8488 /* Common vector code for handling integer to FP conversion */
handle_simd_intfp_conv(DisasContext * s,int rd,int rn,int elements,int is_signed,int fracbits,int size)8489 static void handle_simd_intfp_conv(DisasContext *s, int rd, int rn,
8490 int elements, int is_signed,
8491 int fracbits, int size)
8492 {
8493 TCGv_ptr tcg_fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8494 TCGv_i32 tcg_shift = NULL;
8495
8496 MemOp mop = size | (is_signed ? MO_SIGN : 0);
8497 int pass;
8498
8499 if (fracbits || size == MO_64) {
8500 tcg_shift = tcg_constant_i32(fracbits);
8501 }
8502
8503 if (size == MO_64) {
8504 TCGv_i64 tcg_int64 = tcg_temp_new_i64();
8505 TCGv_i64 tcg_double = tcg_temp_new_i64();
8506
8507 for (pass = 0; pass < elements; pass++) {
8508 read_vec_element(s, tcg_int64, rn, pass, mop);
8509
8510 if (is_signed) {
8511 gen_helper_vfp_sqtod(tcg_double, tcg_int64,
8512 tcg_shift, tcg_fpst);
8513 } else {
8514 gen_helper_vfp_uqtod(tcg_double, tcg_int64,
8515 tcg_shift, tcg_fpst);
8516 }
8517 if (elements == 1) {
8518 write_fp_dreg(s, rd, tcg_double);
8519 } else {
8520 write_vec_element(s, tcg_double, rd, pass, MO_64);
8521 }
8522 }
8523 } else {
8524 TCGv_i32 tcg_int32 = tcg_temp_new_i32();
8525 TCGv_i32 tcg_float = tcg_temp_new_i32();
8526
8527 for (pass = 0; pass < elements; pass++) {
8528 read_vec_element_i32(s, tcg_int32, rn, pass, mop);
8529
8530 switch (size) {
8531 case MO_32:
8532 if (fracbits) {
8533 if (is_signed) {
8534 gen_helper_vfp_sltos(tcg_float, tcg_int32,
8535 tcg_shift, tcg_fpst);
8536 } else {
8537 gen_helper_vfp_ultos(tcg_float, tcg_int32,
8538 tcg_shift, tcg_fpst);
8539 }
8540 } else {
8541 if (is_signed) {
8542 gen_helper_vfp_sitos(tcg_float, tcg_int32, tcg_fpst);
8543 } else {
8544 gen_helper_vfp_uitos(tcg_float, tcg_int32, tcg_fpst);
8545 }
8546 }
8547 break;
8548 case MO_16:
8549 if (fracbits) {
8550 if (is_signed) {
8551 gen_helper_vfp_sltoh(tcg_float, tcg_int32,
8552 tcg_shift, tcg_fpst);
8553 } else {
8554 gen_helper_vfp_ultoh(tcg_float, tcg_int32,
8555 tcg_shift, tcg_fpst);
8556 }
8557 } else {
8558 if (is_signed) {
8559 gen_helper_vfp_sitoh(tcg_float, tcg_int32, tcg_fpst);
8560 } else {
8561 gen_helper_vfp_uitoh(tcg_float, tcg_int32, tcg_fpst);
8562 }
8563 }
8564 break;
8565 default:
8566 g_assert_not_reached();
8567 }
8568
8569 if (elements == 1) {
8570 write_fp_sreg(s, rd, tcg_float);
8571 } else {
8572 write_vec_element_i32(s, tcg_float, rd, pass, size);
8573 }
8574 }
8575 }
8576
8577 clear_vec_high(s, elements << size == 16, rd);
8578 }
8579
8580 /* UCVTF/SCVTF - Integer to FP conversion */
handle_simd_shift_intfp_conv(DisasContext * s,bool is_scalar,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)8581 static void handle_simd_shift_intfp_conv(DisasContext *s, bool is_scalar,
8582 bool is_q, bool is_u,
8583 int immh, int immb, int opcode,
8584 int rn, int rd)
8585 {
8586 int size, elements, fracbits;
8587 int immhb = immh << 3 | immb;
8588
8589 if (immh & 8) {
8590 size = MO_64;
8591 if (!is_scalar && !is_q) {
8592 unallocated_encoding(s);
8593 return;
8594 }
8595 } else if (immh & 4) {
8596 size = MO_32;
8597 } else if (immh & 2) {
8598 size = MO_16;
8599 if (!dc_isar_feature(aa64_fp16, s)) {
8600 unallocated_encoding(s);
8601 return;
8602 }
8603 } else {
8604 /* immh == 0 would be a failure of the decode logic */
8605 g_assert(immh == 1);
8606 unallocated_encoding(s);
8607 return;
8608 }
8609
8610 if (is_scalar) {
8611 elements = 1;
8612 } else {
8613 elements = (8 << is_q) >> size;
8614 }
8615 fracbits = (16 << size) - immhb;
8616
8617 if (!fp_access_check(s)) {
8618 return;
8619 }
8620
8621 handle_simd_intfp_conv(s, rd, rn, elements, !is_u, fracbits, size);
8622 }
8623
8624 /* FCVTZS, FVCVTZU - FP to fixedpoint conversion */
handle_simd_shift_fpint_conv(DisasContext * s,bool is_scalar,bool is_q,bool is_u,int immh,int immb,int rn,int rd)8625 static void handle_simd_shift_fpint_conv(DisasContext *s, bool is_scalar,
8626 bool is_q, bool is_u,
8627 int immh, int immb, int rn, int rd)
8628 {
8629 int immhb = immh << 3 | immb;
8630 int pass, size, fracbits;
8631 TCGv_ptr tcg_fpstatus;
8632 TCGv_i32 tcg_rmode, tcg_shift;
8633
8634 if (immh & 0x8) {
8635 size = MO_64;
8636 if (!is_scalar && !is_q) {
8637 unallocated_encoding(s);
8638 return;
8639 }
8640 } else if (immh & 0x4) {
8641 size = MO_32;
8642 } else if (immh & 0x2) {
8643 size = MO_16;
8644 if (!dc_isar_feature(aa64_fp16, s)) {
8645 unallocated_encoding(s);
8646 return;
8647 }
8648 } else {
8649 /* Should have split out AdvSIMD modified immediate earlier. */
8650 assert(immh == 1);
8651 unallocated_encoding(s);
8652 return;
8653 }
8654
8655 if (!fp_access_check(s)) {
8656 return;
8657 }
8658
8659 assert(!(is_scalar && is_q));
8660
8661 tcg_fpstatus = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
8662 tcg_rmode = gen_set_rmode(FPROUNDING_ZERO, tcg_fpstatus);
8663 fracbits = (16 << size) - immhb;
8664 tcg_shift = tcg_constant_i32(fracbits);
8665
8666 if (size == MO_64) {
8667 int maxpass = is_scalar ? 1 : 2;
8668
8669 for (pass = 0; pass < maxpass; pass++) {
8670 TCGv_i64 tcg_op = tcg_temp_new_i64();
8671
8672 read_vec_element(s, tcg_op, rn, pass, MO_64);
8673 if (is_u) {
8674 gen_helper_vfp_touqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8675 } else {
8676 gen_helper_vfp_tosqd(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8677 }
8678 write_vec_element(s, tcg_op, rd, pass, MO_64);
8679 }
8680 clear_vec_high(s, is_q, rd);
8681 } else {
8682 void (*fn)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
8683 int maxpass = is_scalar ? 1 : ((8 << is_q) >> size);
8684
8685 switch (size) {
8686 case MO_16:
8687 if (is_u) {
8688 fn = gen_helper_vfp_touhh;
8689 } else {
8690 fn = gen_helper_vfp_toshh;
8691 }
8692 break;
8693 case MO_32:
8694 if (is_u) {
8695 fn = gen_helper_vfp_touls;
8696 } else {
8697 fn = gen_helper_vfp_tosls;
8698 }
8699 break;
8700 default:
8701 g_assert_not_reached();
8702 }
8703
8704 for (pass = 0; pass < maxpass; pass++) {
8705 TCGv_i32 tcg_op = tcg_temp_new_i32();
8706
8707 read_vec_element_i32(s, tcg_op, rn, pass, size);
8708 fn(tcg_op, tcg_op, tcg_shift, tcg_fpstatus);
8709 if (is_scalar) {
8710 write_fp_sreg(s, rd, tcg_op);
8711 } else {
8712 write_vec_element_i32(s, tcg_op, rd, pass, size);
8713 }
8714 }
8715 if (!is_scalar) {
8716 clear_vec_high(s, is_q, rd);
8717 }
8718 }
8719
8720 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
8721 }
8722
8723 /* AdvSIMD scalar shift by immediate
8724 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
8725 * +-----+---+-------------+------+------+--------+---+------+------+
8726 * | 0 1 | U | 1 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
8727 * +-----+---+-------------+------+------+--------+---+------+------+
8728 *
8729 * This is the scalar version so it works on a fixed sized registers
8730 */
disas_simd_scalar_shift_imm(DisasContext * s,uint32_t insn)8731 static void disas_simd_scalar_shift_imm(DisasContext *s, uint32_t insn)
8732 {
8733 int rd = extract32(insn, 0, 5);
8734 int rn = extract32(insn, 5, 5);
8735 int opcode = extract32(insn, 11, 5);
8736 int immb = extract32(insn, 16, 3);
8737 int immh = extract32(insn, 19, 4);
8738 bool is_u = extract32(insn, 29, 1);
8739
8740 if (immh == 0) {
8741 unallocated_encoding(s);
8742 return;
8743 }
8744
8745 switch (opcode) {
8746 case 0x08: /* SRI */
8747 if (!is_u) {
8748 unallocated_encoding(s);
8749 return;
8750 }
8751 /* fall through */
8752 case 0x00: /* SSHR / USHR */
8753 case 0x02: /* SSRA / USRA */
8754 case 0x04: /* SRSHR / URSHR */
8755 case 0x06: /* SRSRA / URSRA */
8756 handle_scalar_simd_shri(s, is_u, immh, immb, opcode, rn, rd);
8757 break;
8758 case 0x0a: /* SHL / SLI */
8759 handle_scalar_simd_shli(s, is_u, immh, immb, opcode, rn, rd);
8760 break;
8761 case 0x1c: /* SCVTF, UCVTF */
8762 handle_simd_shift_intfp_conv(s, true, false, is_u, immh, immb,
8763 opcode, rn, rd);
8764 break;
8765 case 0x10: /* SQSHRUN, SQSHRUN2 */
8766 case 0x11: /* SQRSHRUN, SQRSHRUN2 */
8767 if (!is_u) {
8768 unallocated_encoding(s);
8769 return;
8770 }
8771 handle_vec_simd_sqshrn(s, true, false, false, true,
8772 immh, immb, opcode, rn, rd);
8773 break;
8774 case 0x12: /* SQSHRN, SQSHRN2, UQSHRN */
8775 case 0x13: /* SQRSHRN, SQRSHRN2, UQRSHRN, UQRSHRN2 */
8776 handle_vec_simd_sqshrn(s, true, false, is_u, is_u,
8777 immh, immb, opcode, rn, rd);
8778 break;
8779 case 0xc: /* SQSHLU */
8780 if (!is_u) {
8781 unallocated_encoding(s);
8782 return;
8783 }
8784 handle_simd_qshl(s, true, false, false, true, immh, immb, rn, rd);
8785 break;
8786 case 0xe: /* SQSHL, UQSHL */
8787 handle_simd_qshl(s, true, false, is_u, is_u, immh, immb, rn, rd);
8788 break;
8789 case 0x1f: /* FCVTZS, FCVTZU */
8790 handle_simd_shift_fpint_conv(s, true, false, is_u, immh, immb, rn, rd);
8791 break;
8792 default:
8793 unallocated_encoding(s);
8794 break;
8795 }
8796 }
8797
8798 /* AdvSIMD scalar three different
8799 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
8800 * +-----+---+-----------+------+---+------+--------+-----+------+------+
8801 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
8802 * +-----+---+-----------+------+---+------+--------+-----+------+------+
8803 */
disas_simd_scalar_three_reg_diff(DisasContext * s,uint32_t insn)8804 static void disas_simd_scalar_three_reg_diff(DisasContext *s, uint32_t insn)
8805 {
8806 bool is_u = extract32(insn, 29, 1);
8807 int size = extract32(insn, 22, 2);
8808 int opcode = extract32(insn, 12, 4);
8809 int rm = extract32(insn, 16, 5);
8810 int rn = extract32(insn, 5, 5);
8811 int rd = extract32(insn, 0, 5);
8812
8813 if (is_u) {
8814 unallocated_encoding(s);
8815 return;
8816 }
8817
8818 switch (opcode) {
8819 case 0x9: /* SQDMLAL, SQDMLAL2 */
8820 case 0xb: /* SQDMLSL, SQDMLSL2 */
8821 case 0xd: /* SQDMULL, SQDMULL2 */
8822 if (size == 0 || size == 3) {
8823 unallocated_encoding(s);
8824 return;
8825 }
8826 break;
8827 default:
8828 unallocated_encoding(s);
8829 return;
8830 }
8831
8832 if (!fp_access_check(s)) {
8833 return;
8834 }
8835
8836 if (size == 2) {
8837 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
8838 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
8839 TCGv_i64 tcg_res = tcg_temp_new_i64();
8840
8841 read_vec_element(s, tcg_op1, rn, 0, MO_32 | MO_SIGN);
8842 read_vec_element(s, tcg_op2, rm, 0, MO_32 | MO_SIGN);
8843
8844 tcg_gen_mul_i64(tcg_res, tcg_op1, tcg_op2);
8845 gen_helper_neon_addl_saturate_s64(tcg_res, tcg_env, tcg_res, tcg_res);
8846
8847 switch (opcode) {
8848 case 0xd: /* SQDMULL, SQDMULL2 */
8849 break;
8850 case 0xb: /* SQDMLSL, SQDMLSL2 */
8851 tcg_gen_neg_i64(tcg_res, tcg_res);
8852 /* fall through */
8853 case 0x9: /* SQDMLAL, SQDMLAL2 */
8854 read_vec_element(s, tcg_op1, rd, 0, MO_64);
8855 gen_helper_neon_addl_saturate_s64(tcg_res, tcg_env,
8856 tcg_res, tcg_op1);
8857 break;
8858 default:
8859 g_assert_not_reached();
8860 }
8861
8862 write_fp_dreg(s, rd, tcg_res);
8863 } else {
8864 TCGv_i32 tcg_op1 = read_fp_hreg(s, rn);
8865 TCGv_i32 tcg_op2 = read_fp_hreg(s, rm);
8866 TCGv_i64 tcg_res = tcg_temp_new_i64();
8867
8868 gen_helper_neon_mull_s16(tcg_res, tcg_op1, tcg_op2);
8869 gen_helper_neon_addl_saturate_s32(tcg_res, tcg_env, tcg_res, tcg_res);
8870
8871 switch (opcode) {
8872 case 0xd: /* SQDMULL, SQDMULL2 */
8873 break;
8874 case 0xb: /* SQDMLSL, SQDMLSL2 */
8875 gen_helper_neon_negl_u32(tcg_res, tcg_res);
8876 /* fall through */
8877 case 0x9: /* SQDMLAL, SQDMLAL2 */
8878 {
8879 TCGv_i64 tcg_op3 = tcg_temp_new_i64();
8880 read_vec_element(s, tcg_op3, rd, 0, MO_32);
8881 gen_helper_neon_addl_saturate_s32(tcg_res, tcg_env,
8882 tcg_res, tcg_op3);
8883 break;
8884 }
8885 default:
8886 g_assert_not_reached();
8887 }
8888
8889 tcg_gen_ext32u_i64(tcg_res, tcg_res);
8890 write_fp_dreg(s, rd, tcg_res);
8891 }
8892 }
8893
handle_3same_64(DisasContext * s,int opcode,bool u,TCGv_i64 tcg_rd,TCGv_i64 tcg_rn,TCGv_i64 tcg_rm)8894 static void handle_3same_64(DisasContext *s, int opcode, bool u,
8895 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 tcg_rm)
8896 {
8897 /* Handle 64x64->64 opcodes which are shared between the scalar
8898 * and vector 3-same groups. We cover every opcode where size == 3
8899 * is valid in either the three-reg-same (integer, not pairwise)
8900 * or scalar-three-reg-same groups.
8901 */
8902 TCGCond cond;
8903
8904 switch (opcode) {
8905 case 0x1: /* SQADD */
8906 if (u) {
8907 gen_helper_neon_qadd_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8908 } else {
8909 gen_helper_neon_qadd_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8910 }
8911 break;
8912 case 0x5: /* SQSUB */
8913 if (u) {
8914 gen_helper_neon_qsub_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8915 } else {
8916 gen_helper_neon_qsub_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8917 }
8918 break;
8919 case 0x6: /* CMGT, CMHI */
8920 cond = u ? TCG_COND_GTU : TCG_COND_GT;
8921 do_cmop:
8922 /* 64 bit integer comparison, result = test ? -1 : 0. */
8923 tcg_gen_negsetcond_i64(cond, tcg_rd, tcg_rn, tcg_rm);
8924 break;
8925 case 0x7: /* CMGE, CMHS */
8926 cond = u ? TCG_COND_GEU : TCG_COND_GE;
8927 goto do_cmop;
8928 case 0x11: /* CMTST, CMEQ */
8929 if (u) {
8930 cond = TCG_COND_EQ;
8931 goto do_cmop;
8932 }
8933 gen_cmtst_i64(tcg_rd, tcg_rn, tcg_rm);
8934 break;
8935 case 0x8: /* SSHL, USHL */
8936 if (u) {
8937 gen_ushl_i64(tcg_rd, tcg_rn, tcg_rm);
8938 } else {
8939 gen_sshl_i64(tcg_rd, tcg_rn, tcg_rm);
8940 }
8941 break;
8942 case 0x9: /* SQSHL, UQSHL */
8943 if (u) {
8944 gen_helper_neon_qshl_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8945 } else {
8946 gen_helper_neon_qshl_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8947 }
8948 break;
8949 case 0xa: /* SRSHL, URSHL */
8950 if (u) {
8951 gen_helper_neon_rshl_u64(tcg_rd, tcg_rn, tcg_rm);
8952 } else {
8953 gen_helper_neon_rshl_s64(tcg_rd, tcg_rn, tcg_rm);
8954 }
8955 break;
8956 case 0xb: /* SQRSHL, UQRSHL */
8957 if (u) {
8958 gen_helper_neon_qrshl_u64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8959 } else {
8960 gen_helper_neon_qrshl_s64(tcg_rd, tcg_env, tcg_rn, tcg_rm);
8961 }
8962 break;
8963 case 0x10: /* ADD, SUB */
8964 if (u) {
8965 tcg_gen_sub_i64(tcg_rd, tcg_rn, tcg_rm);
8966 } else {
8967 tcg_gen_add_i64(tcg_rd, tcg_rn, tcg_rm);
8968 }
8969 break;
8970 default:
8971 g_assert_not_reached();
8972 }
8973 }
8974
8975 /* Handle the 3-same-operands float operations; shared by the scalar
8976 * and vector encodings. The caller must filter out any encodings
8977 * not allocated for the encoding it is dealing with.
8978 */
handle_3same_float(DisasContext * s,int size,int elements,int fpopcode,int rd,int rn,int rm)8979 static void handle_3same_float(DisasContext *s, int size, int elements,
8980 int fpopcode, int rd, int rn, int rm)
8981 {
8982 int pass;
8983 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
8984
8985 for (pass = 0; pass < elements; pass++) {
8986 if (size) {
8987 /* Double */
8988 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
8989 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
8990 TCGv_i64 tcg_res = tcg_temp_new_i64();
8991
8992 read_vec_element(s, tcg_op1, rn, pass, MO_64);
8993 read_vec_element(s, tcg_op2, rm, pass, MO_64);
8994
8995 switch (fpopcode) {
8996 case 0x39: /* FMLS */
8997 /* As usual for ARM, separate negation for fused multiply-add */
8998 gen_helper_vfp_negd(tcg_op1, tcg_op1);
8999 /* fall through */
9000 case 0x19: /* FMLA */
9001 read_vec_element(s, tcg_res, rd, pass, MO_64);
9002 gen_helper_vfp_muladdd(tcg_res, tcg_op1, tcg_op2,
9003 tcg_res, fpst);
9004 break;
9005 case 0x18: /* FMAXNM */
9006 gen_helper_vfp_maxnumd(tcg_res, tcg_op1, tcg_op2, fpst);
9007 break;
9008 case 0x1a: /* FADD */
9009 gen_helper_vfp_addd(tcg_res, tcg_op1, tcg_op2, fpst);
9010 break;
9011 case 0x1b: /* FMULX */
9012 gen_helper_vfp_mulxd(tcg_res, tcg_op1, tcg_op2, fpst);
9013 break;
9014 case 0x1c: /* FCMEQ */
9015 gen_helper_neon_ceq_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9016 break;
9017 case 0x1e: /* FMAX */
9018 gen_helper_vfp_maxd(tcg_res, tcg_op1, tcg_op2, fpst);
9019 break;
9020 case 0x1f: /* FRECPS */
9021 gen_helper_recpsf_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9022 break;
9023 case 0x38: /* FMINNM */
9024 gen_helper_vfp_minnumd(tcg_res, tcg_op1, tcg_op2, fpst);
9025 break;
9026 case 0x3a: /* FSUB */
9027 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
9028 break;
9029 case 0x3e: /* FMIN */
9030 gen_helper_vfp_mind(tcg_res, tcg_op1, tcg_op2, fpst);
9031 break;
9032 case 0x3f: /* FRSQRTS */
9033 gen_helper_rsqrtsf_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9034 break;
9035 case 0x5b: /* FMUL */
9036 gen_helper_vfp_muld(tcg_res, tcg_op1, tcg_op2, fpst);
9037 break;
9038 case 0x5c: /* FCMGE */
9039 gen_helper_neon_cge_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9040 break;
9041 case 0x5d: /* FACGE */
9042 gen_helper_neon_acge_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9043 break;
9044 case 0x5f: /* FDIV */
9045 gen_helper_vfp_divd(tcg_res, tcg_op1, tcg_op2, fpst);
9046 break;
9047 case 0x7a: /* FABD */
9048 gen_helper_vfp_subd(tcg_res, tcg_op1, tcg_op2, fpst);
9049 gen_helper_vfp_absd(tcg_res, tcg_res);
9050 break;
9051 case 0x7c: /* FCMGT */
9052 gen_helper_neon_cgt_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9053 break;
9054 case 0x7d: /* FACGT */
9055 gen_helper_neon_acgt_f64(tcg_res, tcg_op1, tcg_op2, fpst);
9056 break;
9057 default:
9058 g_assert_not_reached();
9059 }
9060
9061 write_vec_element(s, tcg_res, rd, pass, MO_64);
9062 } else {
9063 /* Single */
9064 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
9065 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
9066 TCGv_i32 tcg_res = tcg_temp_new_i32();
9067
9068 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32);
9069 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32);
9070
9071 switch (fpopcode) {
9072 case 0x39: /* FMLS */
9073 /* As usual for ARM, separate negation for fused multiply-add */
9074 gen_helper_vfp_negs(tcg_op1, tcg_op1);
9075 /* fall through */
9076 case 0x19: /* FMLA */
9077 read_vec_element_i32(s, tcg_res, rd, pass, MO_32);
9078 gen_helper_vfp_muladds(tcg_res, tcg_op1, tcg_op2,
9079 tcg_res, fpst);
9080 break;
9081 case 0x1a: /* FADD */
9082 gen_helper_vfp_adds(tcg_res, tcg_op1, tcg_op2, fpst);
9083 break;
9084 case 0x1b: /* FMULX */
9085 gen_helper_vfp_mulxs(tcg_res, tcg_op1, tcg_op2, fpst);
9086 break;
9087 case 0x1c: /* FCMEQ */
9088 gen_helper_neon_ceq_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9089 break;
9090 case 0x1e: /* FMAX */
9091 gen_helper_vfp_maxs(tcg_res, tcg_op1, tcg_op2, fpst);
9092 break;
9093 case 0x1f: /* FRECPS */
9094 gen_helper_recpsf_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9095 break;
9096 case 0x18: /* FMAXNM */
9097 gen_helper_vfp_maxnums(tcg_res, tcg_op1, tcg_op2, fpst);
9098 break;
9099 case 0x38: /* FMINNM */
9100 gen_helper_vfp_minnums(tcg_res, tcg_op1, tcg_op2, fpst);
9101 break;
9102 case 0x3a: /* FSUB */
9103 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
9104 break;
9105 case 0x3e: /* FMIN */
9106 gen_helper_vfp_mins(tcg_res, tcg_op1, tcg_op2, fpst);
9107 break;
9108 case 0x3f: /* FRSQRTS */
9109 gen_helper_rsqrtsf_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9110 break;
9111 case 0x5b: /* FMUL */
9112 gen_helper_vfp_muls(tcg_res, tcg_op1, tcg_op2, fpst);
9113 break;
9114 case 0x5c: /* FCMGE */
9115 gen_helper_neon_cge_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9116 break;
9117 case 0x5d: /* FACGE */
9118 gen_helper_neon_acge_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9119 break;
9120 case 0x5f: /* FDIV */
9121 gen_helper_vfp_divs(tcg_res, tcg_op1, tcg_op2, fpst);
9122 break;
9123 case 0x7a: /* FABD */
9124 gen_helper_vfp_subs(tcg_res, tcg_op1, tcg_op2, fpst);
9125 gen_helper_vfp_abss(tcg_res, tcg_res);
9126 break;
9127 case 0x7c: /* FCMGT */
9128 gen_helper_neon_cgt_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9129 break;
9130 case 0x7d: /* FACGT */
9131 gen_helper_neon_acgt_f32(tcg_res, tcg_op1, tcg_op2, fpst);
9132 break;
9133 default:
9134 g_assert_not_reached();
9135 }
9136
9137 if (elements == 1) {
9138 /* scalar single so clear high part */
9139 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
9140
9141 tcg_gen_extu_i32_i64(tcg_tmp, tcg_res);
9142 write_vec_element(s, tcg_tmp, rd, pass, MO_64);
9143 } else {
9144 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
9145 }
9146 }
9147 }
9148
9149 clear_vec_high(s, elements * (size ? 8 : 4) > 8, rd);
9150 }
9151
9152 /* AdvSIMD scalar three same
9153 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
9154 * +-----+---+-----------+------+---+------+--------+---+------+------+
9155 * | 0 1 | U | 1 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
9156 * +-----+---+-----------+------+---+------+--------+---+------+------+
9157 */
disas_simd_scalar_three_reg_same(DisasContext * s,uint32_t insn)9158 static void disas_simd_scalar_three_reg_same(DisasContext *s, uint32_t insn)
9159 {
9160 int rd = extract32(insn, 0, 5);
9161 int rn = extract32(insn, 5, 5);
9162 int opcode = extract32(insn, 11, 5);
9163 int rm = extract32(insn, 16, 5);
9164 int size = extract32(insn, 22, 2);
9165 bool u = extract32(insn, 29, 1);
9166 TCGv_i64 tcg_rd;
9167
9168 if (opcode >= 0x18) {
9169 /* Floating point: U, size[1] and opcode indicate operation */
9170 int fpopcode = opcode | (extract32(size, 1, 1) << 5) | (u << 6);
9171 switch (fpopcode) {
9172 case 0x1b: /* FMULX */
9173 case 0x1f: /* FRECPS */
9174 case 0x3f: /* FRSQRTS */
9175 case 0x5d: /* FACGE */
9176 case 0x7d: /* FACGT */
9177 case 0x1c: /* FCMEQ */
9178 case 0x5c: /* FCMGE */
9179 case 0x7c: /* FCMGT */
9180 case 0x7a: /* FABD */
9181 break;
9182 default:
9183 unallocated_encoding(s);
9184 return;
9185 }
9186
9187 if (!fp_access_check(s)) {
9188 return;
9189 }
9190
9191 handle_3same_float(s, extract32(size, 0, 1), 1, fpopcode, rd, rn, rm);
9192 return;
9193 }
9194
9195 switch (opcode) {
9196 case 0x1: /* SQADD, UQADD */
9197 case 0x5: /* SQSUB, UQSUB */
9198 case 0x9: /* SQSHL, UQSHL */
9199 case 0xb: /* SQRSHL, UQRSHL */
9200 break;
9201 case 0x8: /* SSHL, USHL */
9202 case 0xa: /* SRSHL, URSHL */
9203 case 0x6: /* CMGT, CMHI */
9204 case 0x7: /* CMGE, CMHS */
9205 case 0x11: /* CMTST, CMEQ */
9206 case 0x10: /* ADD, SUB (vector) */
9207 if (size != 3) {
9208 unallocated_encoding(s);
9209 return;
9210 }
9211 break;
9212 case 0x16: /* SQDMULH, SQRDMULH (vector) */
9213 if (size != 1 && size != 2) {
9214 unallocated_encoding(s);
9215 return;
9216 }
9217 break;
9218 default:
9219 unallocated_encoding(s);
9220 return;
9221 }
9222
9223 if (!fp_access_check(s)) {
9224 return;
9225 }
9226
9227 tcg_rd = tcg_temp_new_i64();
9228
9229 if (size == 3) {
9230 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
9231 TCGv_i64 tcg_rm = read_fp_dreg(s, rm);
9232
9233 handle_3same_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rm);
9234 } else {
9235 /* Do a single operation on the lowest element in the vector.
9236 * We use the standard Neon helpers and rely on 0 OP 0 == 0 with
9237 * no side effects for all these operations.
9238 * OPTME: special-purpose helpers would avoid doing some
9239 * unnecessary work in the helper for the 8 and 16 bit cases.
9240 */
9241 NeonGenTwoOpEnvFn *genenvfn;
9242 TCGv_i32 tcg_rn = tcg_temp_new_i32();
9243 TCGv_i32 tcg_rm = tcg_temp_new_i32();
9244 TCGv_i32 tcg_rd32 = tcg_temp_new_i32();
9245
9246 read_vec_element_i32(s, tcg_rn, rn, 0, size);
9247 read_vec_element_i32(s, tcg_rm, rm, 0, size);
9248
9249 switch (opcode) {
9250 case 0x1: /* SQADD, UQADD */
9251 {
9252 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9253 { gen_helper_neon_qadd_s8, gen_helper_neon_qadd_u8 },
9254 { gen_helper_neon_qadd_s16, gen_helper_neon_qadd_u16 },
9255 { gen_helper_neon_qadd_s32, gen_helper_neon_qadd_u32 },
9256 };
9257 genenvfn = fns[size][u];
9258 break;
9259 }
9260 case 0x5: /* SQSUB, UQSUB */
9261 {
9262 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9263 { gen_helper_neon_qsub_s8, gen_helper_neon_qsub_u8 },
9264 { gen_helper_neon_qsub_s16, gen_helper_neon_qsub_u16 },
9265 { gen_helper_neon_qsub_s32, gen_helper_neon_qsub_u32 },
9266 };
9267 genenvfn = fns[size][u];
9268 break;
9269 }
9270 case 0x9: /* SQSHL, UQSHL */
9271 {
9272 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9273 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 },
9274 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 },
9275 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 },
9276 };
9277 genenvfn = fns[size][u];
9278 break;
9279 }
9280 case 0xb: /* SQRSHL, UQRSHL */
9281 {
9282 static NeonGenTwoOpEnvFn * const fns[3][2] = {
9283 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 },
9284 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 },
9285 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 },
9286 };
9287 genenvfn = fns[size][u];
9288 break;
9289 }
9290 case 0x16: /* SQDMULH, SQRDMULH */
9291 {
9292 static NeonGenTwoOpEnvFn * const fns[2][2] = {
9293 { gen_helper_neon_qdmulh_s16, gen_helper_neon_qrdmulh_s16 },
9294 { gen_helper_neon_qdmulh_s32, gen_helper_neon_qrdmulh_s32 },
9295 };
9296 assert(size == 1 || size == 2);
9297 genenvfn = fns[size - 1][u];
9298 break;
9299 }
9300 default:
9301 g_assert_not_reached();
9302 }
9303
9304 genenvfn(tcg_rd32, tcg_env, tcg_rn, tcg_rm);
9305 tcg_gen_extu_i32_i64(tcg_rd, tcg_rd32);
9306 }
9307
9308 write_fp_dreg(s, rd, tcg_rd);
9309 }
9310
9311 /* AdvSIMD scalar three same FP16
9312 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0
9313 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+
9314 * | 0 1 | U | 1 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd |
9315 * +-----+---+-----------+---+-----+------+-----+--------+---+----+----+
9316 * v: 0101 1110 0100 0000 0000 0100 0000 0000 => 5e400400
9317 * m: 1101 1111 0110 0000 1100 0100 0000 0000 => df60c400
9318 */
disas_simd_scalar_three_reg_same_fp16(DisasContext * s,uint32_t insn)9319 static void disas_simd_scalar_three_reg_same_fp16(DisasContext *s,
9320 uint32_t insn)
9321 {
9322 int rd = extract32(insn, 0, 5);
9323 int rn = extract32(insn, 5, 5);
9324 int opcode = extract32(insn, 11, 3);
9325 int rm = extract32(insn, 16, 5);
9326 bool u = extract32(insn, 29, 1);
9327 bool a = extract32(insn, 23, 1);
9328 int fpopcode = opcode | (a << 3) | (u << 4);
9329 TCGv_ptr fpst;
9330 TCGv_i32 tcg_op1;
9331 TCGv_i32 tcg_op2;
9332 TCGv_i32 tcg_res;
9333
9334 switch (fpopcode) {
9335 case 0x03: /* FMULX */
9336 case 0x04: /* FCMEQ (reg) */
9337 case 0x07: /* FRECPS */
9338 case 0x0f: /* FRSQRTS */
9339 case 0x14: /* FCMGE (reg) */
9340 case 0x15: /* FACGE */
9341 case 0x1a: /* FABD */
9342 case 0x1c: /* FCMGT (reg) */
9343 case 0x1d: /* FACGT */
9344 break;
9345 default:
9346 unallocated_encoding(s);
9347 return;
9348 }
9349
9350 if (!dc_isar_feature(aa64_fp16, s)) {
9351 unallocated_encoding(s);
9352 }
9353
9354 if (!fp_access_check(s)) {
9355 return;
9356 }
9357
9358 fpst = fpstatus_ptr(FPST_FPCR_F16);
9359
9360 tcg_op1 = read_fp_hreg(s, rn);
9361 tcg_op2 = read_fp_hreg(s, rm);
9362 tcg_res = tcg_temp_new_i32();
9363
9364 switch (fpopcode) {
9365 case 0x03: /* FMULX */
9366 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst);
9367 break;
9368 case 0x04: /* FCMEQ (reg) */
9369 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9370 break;
9371 case 0x07: /* FRECPS */
9372 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9373 break;
9374 case 0x0f: /* FRSQRTS */
9375 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9376 break;
9377 case 0x14: /* FCMGE (reg) */
9378 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9379 break;
9380 case 0x15: /* FACGE */
9381 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9382 break;
9383 case 0x1a: /* FABD */
9384 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
9385 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff);
9386 break;
9387 case 0x1c: /* FCMGT (reg) */
9388 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9389 break;
9390 case 0x1d: /* FACGT */
9391 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
9392 break;
9393 default:
9394 g_assert_not_reached();
9395 }
9396
9397 write_fp_sreg(s, rd, tcg_res);
9398 }
9399
9400 /* AdvSIMD scalar three same extra
9401 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
9402 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9403 * | 0 1 | U | 1 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
9404 * +-----+---+-----------+------+---+------+---+--------+---+----+----+
9405 */
disas_simd_scalar_three_reg_same_extra(DisasContext * s,uint32_t insn)9406 static void disas_simd_scalar_three_reg_same_extra(DisasContext *s,
9407 uint32_t insn)
9408 {
9409 int rd = extract32(insn, 0, 5);
9410 int rn = extract32(insn, 5, 5);
9411 int opcode = extract32(insn, 11, 4);
9412 int rm = extract32(insn, 16, 5);
9413 int size = extract32(insn, 22, 2);
9414 bool u = extract32(insn, 29, 1);
9415 TCGv_i32 ele1, ele2, ele3;
9416 TCGv_i64 res;
9417 bool feature;
9418
9419 switch (u * 16 + opcode) {
9420 case 0x10: /* SQRDMLAH (vector) */
9421 case 0x11: /* SQRDMLSH (vector) */
9422 if (size != 1 && size != 2) {
9423 unallocated_encoding(s);
9424 return;
9425 }
9426 feature = dc_isar_feature(aa64_rdm, s);
9427 break;
9428 default:
9429 unallocated_encoding(s);
9430 return;
9431 }
9432 if (!feature) {
9433 unallocated_encoding(s);
9434 return;
9435 }
9436 if (!fp_access_check(s)) {
9437 return;
9438 }
9439
9440 /* Do a single operation on the lowest element in the vector.
9441 * We use the standard Neon helpers and rely on 0 OP 0 == 0
9442 * with no side effects for all these operations.
9443 * OPTME: special-purpose helpers would avoid doing some
9444 * unnecessary work in the helper for the 16 bit cases.
9445 */
9446 ele1 = tcg_temp_new_i32();
9447 ele2 = tcg_temp_new_i32();
9448 ele3 = tcg_temp_new_i32();
9449
9450 read_vec_element_i32(s, ele1, rn, 0, size);
9451 read_vec_element_i32(s, ele2, rm, 0, size);
9452 read_vec_element_i32(s, ele3, rd, 0, size);
9453
9454 switch (opcode) {
9455 case 0x0: /* SQRDMLAH */
9456 if (size == 1) {
9457 gen_helper_neon_qrdmlah_s16(ele3, tcg_env, ele1, ele2, ele3);
9458 } else {
9459 gen_helper_neon_qrdmlah_s32(ele3, tcg_env, ele1, ele2, ele3);
9460 }
9461 break;
9462 case 0x1: /* SQRDMLSH */
9463 if (size == 1) {
9464 gen_helper_neon_qrdmlsh_s16(ele3, tcg_env, ele1, ele2, ele3);
9465 } else {
9466 gen_helper_neon_qrdmlsh_s32(ele3, tcg_env, ele1, ele2, ele3);
9467 }
9468 break;
9469 default:
9470 g_assert_not_reached();
9471 }
9472
9473 res = tcg_temp_new_i64();
9474 tcg_gen_extu_i32_i64(res, ele3);
9475 write_fp_dreg(s, rd, res);
9476 }
9477
handle_2misc_64(DisasContext * s,int opcode,bool u,TCGv_i64 tcg_rd,TCGv_i64 tcg_rn,TCGv_i32 tcg_rmode,TCGv_ptr tcg_fpstatus)9478 static void handle_2misc_64(DisasContext *s, int opcode, bool u,
9479 TCGv_i64 tcg_rd, TCGv_i64 tcg_rn,
9480 TCGv_i32 tcg_rmode, TCGv_ptr tcg_fpstatus)
9481 {
9482 /* Handle 64->64 opcodes which are shared between the scalar and
9483 * vector 2-reg-misc groups. We cover every integer opcode where size == 3
9484 * is valid in either group and also the double-precision fp ops.
9485 * The caller only need provide tcg_rmode and tcg_fpstatus if the op
9486 * requires them.
9487 */
9488 TCGCond cond;
9489
9490 switch (opcode) {
9491 case 0x4: /* CLS, CLZ */
9492 if (u) {
9493 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
9494 } else {
9495 tcg_gen_clrsb_i64(tcg_rd, tcg_rn);
9496 }
9497 break;
9498 case 0x5: /* NOT */
9499 /* This opcode is shared with CNT and RBIT but we have earlier
9500 * enforced that size == 3 if and only if this is the NOT insn.
9501 */
9502 tcg_gen_not_i64(tcg_rd, tcg_rn);
9503 break;
9504 case 0x7: /* SQABS, SQNEG */
9505 if (u) {
9506 gen_helper_neon_qneg_s64(tcg_rd, tcg_env, tcg_rn);
9507 } else {
9508 gen_helper_neon_qabs_s64(tcg_rd, tcg_env, tcg_rn);
9509 }
9510 break;
9511 case 0xa: /* CMLT */
9512 cond = TCG_COND_LT;
9513 do_cmop:
9514 /* 64 bit integer comparison against zero, result is test ? -1 : 0. */
9515 tcg_gen_negsetcond_i64(cond, tcg_rd, tcg_rn, tcg_constant_i64(0));
9516 break;
9517 case 0x8: /* CMGT, CMGE */
9518 cond = u ? TCG_COND_GE : TCG_COND_GT;
9519 goto do_cmop;
9520 case 0x9: /* CMEQ, CMLE */
9521 cond = u ? TCG_COND_LE : TCG_COND_EQ;
9522 goto do_cmop;
9523 case 0xb: /* ABS, NEG */
9524 if (u) {
9525 tcg_gen_neg_i64(tcg_rd, tcg_rn);
9526 } else {
9527 tcg_gen_abs_i64(tcg_rd, tcg_rn);
9528 }
9529 break;
9530 case 0x2f: /* FABS */
9531 gen_helper_vfp_absd(tcg_rd, tcg_rn);
9532 break;
9533 case 0x6f: /* FNEG */
9534 gen_helper_vfp_negd(tcg_rd, tcg_rn);
9535 break;
9536 case 0x7f: /* FSQRT */
9537 gen_helper_vfp_sqrtd(tcg_rd, tcg_rn, tcg_env);
9538 break;
9539 case 0x1a: /* FCVTNS */
9540 case 0x1b: /* FCVTMS */
9541 case 0x1c: /* FCVTAS */
9542 case 0x3a: /* FCVTPS */
9543 case 0x3b: /* FCVTZS */
9544 gen_helper_vfp_tosqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
9545 break;
9546 case 0x5a: /* FCVTNU */
9547 case 0x5b: /* FCVTMU */
9548 case 0x5c: /* FCVTAU */
9549 case 0x7a: /* FCVTPU */
9550 case 0x7b: /* FCVTZU */
9551 gen_helper_vfp_touqd(tcg_rd, tcg_rn, tcg_constant_i32(0), tcg_fpstatus);
9552 break;
9553 case 0x18: /* FRINTN */
9554 case 0x19: /* FRINTM */
9555 case 0x38: /* FRINTP */
9556 case 0x39: /* FRINTZ */
9557 case 0x58: /* FRINTA */
9558 case 0x79: /* FRINTI */
9559 gen_helper_rintd(tcg_rd, tcg_rn, tcg_fpstatus);
9560 break;
9561 case 0x59: /* FRINTX */
9562 gen_helper_rintd_exact(tcg_rd, tcg_rn, tcg_fpstatus);
9563 break;
9564 case 0x1e: /* FRINT32Z */
9565 case 0x5e: /* FRINT32X */
9566 gen_helper_frint32_d(tcg_rd, tcg_rn, tcg_fpstatus);
9567 break;
9568 case 0x1f: /* FRINT64Z */
9569 case 0x5f: /* FRINT64X */
9570 gen_helper_frint64_d(tcg_rd, tcg_rn, tcg_fpstatus);
9571 break;
9572 default:
9573 g_assert_not_reached();
9574 }
9575 }
9576
handle_2misc_fcmp_zero(DisasContext * s,int opcode,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9577 static void handle_2misc_fcmp_zero(DisasContext *s, int opcode,
9578 bool is_scalar, bool is_u, bool is_q,
9579 int size, int rn, int rd)
9580 {
9581 bool is_double = (size == MO_64);
9582 TCGv_ptr fpst;
9583
9584 if (!fp_access_check(s)) {
9585 return;
9586 }
9587
9588 fpst = fpstatus_ptr(size == MO_16 ? FPST_FPCR_F16 : FPST_FPCR);
9589
9590 if (is_double) {
9591 TCGv_i64 tcg_op = tcg_temp_new_i64();
9592 TCGv_i64 tcg_zero = tcg_constant_i64(0);
9593 TCGv_i64 tcg_res = tcg_temp_new_i64();
9594 NeonGenTwoDoubleOpFn *genfn;
9595 bool swap = false;
9596 int pass;
9597
9598 switch (opcode) {
9599 case 0x2e: /* FCMLT (zero) */
9600 swap = true;
9601 /* fallthrough */
9602 case 0x2c: /* FCMGT (zero) */
9603 genfn = gen_helper_neon_cgt_f64;
9604 break;
9605 case 0x2d: /* FCMEQ (zero) */
9606 genfn = gen_helper_neon_ceq_f64;
9607 break;
9608 case 0x6d: /* FCMLE (zero) */
9609 swap = true;
9610 /* fall through */
9611 case 0x6c: /* FCMGE (zero) */
9612 genfn = gen_helper_neon_cge_f64;
9613 break;
9614 default:
9615 g_assert_not_reached();
9616 }
9617
9618 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9619 read_vec_element(s, tcg_op, rn, pass, MO_64);
9620 if (swap) {
9621 genfn(tcg_res, tcg_zero, tcg_op, fpst);
9622 } else {
9623 genfn(tcg_res, tcg_op, tcg_zero, fpst);
9624 }
9625 write_vec_element(s, tcg_res, rd, pass, MO_64);
9626 }
9627
9628 clear_vec_high(s, !is_scalar, rd);
9629 } else {
9630 TCGv_i32 tcg_op = tcg_temp_new_i32();
9631 TCGv_i32 tcg_zero = tcg_constant_i32(0);
9632 TCGv_i32 tcg_res = tcg_temp_new_i32();
9633 NeonGenTwoSingleOpFn *genfn;
9634 bool swap = false;
9635 int pass, maxpasses;
9636
9637 if (size == MO_16) {
9638 switch (opcode) {
9639 case 0x2e: /* FCMLT (zero) */
9640 swap = true;
9641 /* fall through */
9642 case 0x2c: /* FCMGT (zero) */
9643 genfn = gen_helper_advsimd_cgt_f16;
9644 break;
9645 case 0x2d: /* FCMEQ (zero) */
9646 genfn = gen_helper_advsimd_ceq_f16;
9647 break;
9648 case 0x6d: /* FCMLE (zero) */
9649 swap = true;
9650 /* fall through */
9651 case 0x6c: /* FCMGE (zero) */
9652 genfn = gen_helper_advsimd_cge_f16;
9653 break;
9654 default:
9655 g_assert_not_reached();
9656 }
9657 } else {
9658 switch (opcode) {
9659 case 0x2e: /* FCMLT (zero) */
9660 swap = true;
9661 /* fall through */
9662 case 0x2c: /* FCMGT (zero) */
9663 genfn = gen_helper_neon_cgt_f32;
9664 break;
9665 case 0x2d: /* FCMEQ (zero) */
9666 genfn = gen_helper_neon_ceq_f32;
9667 break;
9668 case 0x6d: /* FCMLE (zero) */
9669 swap = true;
9670 /* fall through */
9671 case 0x6c: /* FCMGE (zero) */
9672 genfn = gen_helper_neon_cge_f32;
9673 break;
9674 default:
9675 g_assert_not_reached();
9676 }
9677 }
9678
9679 if (is_scalar) {
9680 maxpasses = 1;
9681 } else {
9682 int vector_size = 8 << is_q;
9683 maxpasses = vector_size >> size;
9684 }
9685
9686 for (pass = 0; pass < maxpasses; pass++) {
9687 read_vec_element_i32(s, tcg_op, rn, pass, size);
9688 if (swap) {
9689 genfn(tcg_res, tcg_zero, tcg_op, fpst);
9690 } else {
9691 genfn(tcg_res, tcg_op, tcg_zero, fpst);
9692 }
9693 if (is_scalar) {
9694 write_fp_sreg(s, rd, tcg_res);
9695 } else {
9696 write_vec_element_i32(s, tcg_res, rd, pass, size);
9697 }
9698 }
9699
9700 if (!is_scalar) {
9701 clear_vec_high(s, is_q, rd);
9702 }
9703 }
9704 }
9705
handle_2misc_reciprocal(DisasContext * s,int opcode,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9706 static void handle_2misc_reciprocal(DisasContext *s, int opcode,
9707 bool is_scalar, bool is_u, bool is_q,
9708 int size, int rn, int rd)
9709 {
9710 bool is_double = (size == 3);
9711 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9712
9713 if (is_double) {
9714 TCGv_i64 tcg_op = tcg_temp_new_i64();
9715 TCGv_i64 tcg_res = tcg_temp_new_i64();
9716 int pass;
9717
9718 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9719 read_vec_element(s, tcg_op, rn, pass, MO_64);
9720 switch (opcode) {
9721 case 0x3d: /* FRECPE */
9722 gen_helper_recpe_f64(tcg_res, tcg_op, fpst);
9723 break;
9724 case 0x3f: /* FRECPX */
9725 gen_helper_frecpx_f64(tcg_res, tcg_op, fpst);
9726 break;
9727 case 0x7d: /* FRSQRTE */
9728 gen_helper_rsqrte_f64(tcg_res, tcg_op, fpst);
9729 break;
9730 default:
9731 g_assert_not_reached();
9732 }
9733 write_vec_element(s, tcg_res, rd, pass, MO_64);
9734 }
9735 clear_vec_high(s, !is_scalar, rd);
9736 } else {
9737 TCGv_i32 tcg_op = tcg_temp_new_i32();
9738 TCGv_i32 tcg_res = tcg_temp_new_i32();
9739 int pass, maxpasses;
9740
9741 if (is_scalar) {
9742 maxpasses = 1;
9743 } else {
9744 maxpasses = is_q ? 4 : 2;
9745 }
9746
9747 for (pass = 0; pass < maxpasses; pass++) {
9748 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
9749
9750 switch (opcode) {
9751 case 0x3c: /* URECPE */
9752 gen_helper_recpe_u32(tcg_res, tcg_op);
9753 break;
9754 case 0x3d: /* FRECPE */
9755 gen_helper_recpe_f32(tcg_res, tcg_op, fpst);
9756 break;
9757 case 0x3f: /* FRECPX */
9758 gen_helper_frecpx_f32(tcg_res, tcg_op, fpst);
9759 break;
9760 case 0x7d: /* FRSQRTE */
9761 gen_helper_rsqrte_f32(tcg_res, tcg_op, fpst);
9762 break;
9763 default:
9764 g_assert_not_reached();
9765 }
9766
9767 if (is_scalar) {
9768 write_fp_sreg(s, rd, tcg_res);
9769 } else {
9770 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
9771 }
9772 }
9773 if (!is_scalar) {
9774 clear_vec_high(s, is_q, rd);
9775 }
9776 }
9777 }
9778
handle_2misc_narrow(DisasContext * s,bool scalar,int opcode,bool u,bool is_q,int size,int rn,int rd)9779 static void handle_2misc_narrow(DisasContext *s, bool scalar,
9780 int opcode, bool u, bool is_q,
9781 int size, int rn, int rd)
9782 {
9783 /* Handle 2-reg-misc ops which are narrowing (so each 2*size element
9784 * in the source becomes a size element in the destination).
9785 */
9786 int pass;
9787 TCGv_i32 tcg_res[2];
9788 int destelt = is_q ? 2 : 0;
9789 int passes = scalar ? 1 : 2;
9790
9791 if (scalar) {
9792 tcg_res[1] = tcg_constant_i32(0);
9793 }
9794
9795 for (pass = 0; pass < passes; pass++) {
9796 TCGv_i64 tcg_op = tcg_temp_new_i64();
9797 NeonGenNarrowFn *genfn = NULL;
9798 NeonGenNarrowEnvFn *genenvfn = NULL;
9799
9800 if (scalar) {
9801 read_vec_element(s, tcg_op, rn, pass, size + 1);
9802 } else {
9803 read_vec_element(s, tcg_op, rn, pass, MO_64);
9804 }
9805 tcg_res[pass] = tcg_temp_new_i32();
9806
9807 switch (opcode) {
9808 case 0x12: /* XTN, SQXTUN */
9809 {
9810 static NeonGenNarrowFn * const xtnfns[3] = {
9811 gen_helper_neon_narrow_u8,
9812 gen_helper_neon_narrow_u16,
9813 tcg_gen_extrl_i64_i32,
9814 };
9815 static NeonGenNarrowEnvFn * const sqxtunfns[3] = {
9816 gen_helper_neon_unarrow_sat8,
9817 gen_helper_neon_unarrow_sat16,
9818 gen_helper_neon_unarrow_sat32,
9819 };
9820 if (u) {
9821 genenvfn = sqxtunfns[size];
9822 } else {
9823 genfn = xtnfns[size];
9824 }
9825 break;
9826 }
9827 case 0x14: /* SQXTN, UQXTN */
9828 {
9829 static NeonGenNarrowEnvFn * const fns[3][2] = {
9830 { gen_helper_neon_narrow_sat_s8,
9831 gen_helper_neon_narrow_sat_u8 },
9832 { gen_helper_neon_narrow_sat_s16,
9833 gen_helper_neon_narrow_sat_u16 },
9834 { gen_helper_neon_narrow_sat_s32,
9835 gen_helper_neon_narrow_sat_u32 },
9836 };
9837 genenvfn = fns[size][u];
9838 break;
9839 }
9840 case 0x16: /* FCVTN, FCVTN2 */
9841 /* 32 bit to 16 bit or 64 bit to 32 bit float conversion */
9842 if (size == 2) {
9843 gen_helper_vfp_fcvtsd(tcg_res[pass], tcg_op, tcg_env);
9844 } else {
9845 TCGv_i32 tcg_lo = tcg_temp_new_i32();
9846 TCGv_i32 tcg_hi = tcg_temp_new_i32();
9847 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9848 TCGv_i32 ahp = get_ahp_flag();
9849
9850 tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
9851 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp);
9852 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp);
9853 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16);
9854 }
9855 break;
9856 case 0x36: /* BFCVTN, BFCVTN2 */
9857 {
9858 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
9859 gen_helper_bfcvt_pair(tcg_res[pass], tcg_op, fpst);
9860 }
9861 break;
9862 case 0x56: /* FCVTXN, FCVTXN2 */
9863 /* 64 bit to 32 bit float conversion
9864 * with von Neumann rounding (round to odd)
9865 */
9866 assert(size == 2);
9867 gen_helper_fcvtx_f64_to_f32(tcg_res[pass], tcg_op, tcg_env);
9868 break;
9869 default:
9870 g_assert_not_reached();
9871 }
9872
9873 if (genfn) {
9874 genfn(tcg_res[pass], tcg_op);
9875 } else if (genenvfn) {
9876 genenvfn(tcg_res[pass], tcg_env, tcg_op);
9877 }
9878 }
9879
9880 for (pass = 0; pass < 2; pass++) {
9881 write_vec_element_i32(s, tcg_res[pass], rd, destelt + pass, MO_32);
9882 }
9883 clear_vec_high(s, is_q, rd);
9884 }
9885
9886 /* Remaining saturating accumulating ops */
handle_2misc_satacc(DisasContext * s,bool is_scalar,bool is_u,bool is_q,int size,int rn,int rd)9887 static void handle_2misc_satacc(DisasContext *s, bool is_scalar, bool is_u,
9888 bool is_q, int size, int rn, int rd)
9889 {
9890 bool is_double = (size == 3);
9891
9892 if (is_double) {
9893 TCGv_i64 tcg_rn = tcg_temp_new_i64();
9894 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9895 int pass;
9896
9897 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
9898 read_vec_element(s, tcg_rn, rn, pass, MO_64);
9899 read_vec_element(s, tcg_rd, rd, pass, MO_64);
9900
9901 if (is_u) { /* USQADD */
9902 gen_helper_neon_uqadd_s64(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9903 } else { /* SUQADD */
9904 gen_helper_neon_sqadd_u64(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9905 }
9906 write_vec_element(s, tcg_rd, rd, pass, MO_64);
9907 }
9908 clear_vec_high(s, !is_scalar, rd);
9909 } else {
9910 TCGv_i32 tcg_rn = tcg_temp_new_i32();
9911 TCGv_i32 tcg_rd = tcg_temp_new_i32();
9912 int pass, maxpasses;
9913
9914 if (is_scalar) {
9915 maxpasses = 1;
9916 } else {
9917 maxpasses = is_q ? 4 : 2;
9918 }
9919
9920 for (pass = 0; pass < maxpasses; pass++) {
9921 if (is_scalar) {
9922 read_vec_element_i32(s, tcg_rn, rn, pass, size);
9923 read_vec_element_i32(s, tcg_rd, rd, pass, size);
9924 } else {
9925 read_vec_element_i32(s, tcg_rn, rn, pass, MO_32);
9926 read_vec_element_i32(s, tcg_rd, rd, pass, MO_32);
9927 }
9928
9929 if (is_u) { /* USQADD */
9930 switch (size) {
9931 case 0:
9932 gen_helper_neon_uqadd_s8(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9933 break;
9934 case 1:
9935 gen_helper_neon_uqadd_s16(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9936 break;
9937 case 2:
9938 gen_helper_neon_uqadd_s32(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9939 break;
9940 default:
9941 g_assert_not_reached();
9942 }
9943 } else { /* SUQADD */
9944 switch (size) {
9945 case 0:
9946 gen_helper_neon_sqadd_u8(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9947 break;
9948 case 1:
9949 gen_helper_neon_sqadd_u16(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9950 break;
9951 case 2:
9952 gen_helper_neon_sqadd_u32(tcg_rd, tcg_env, tcg_rn, tcg_rd);
9953 break;
9954 default:
9955 g_assert_not_reached();
9956 }
9957 }
9958
9959 if (is_scalar) {
9960 write_vec_element(s, tcg_constant_i64(0), rd, 0, MO_64);
9961 }
9962 write_vec_element_i32(s, tcg_rd, rd, pass, MO_32);
9963 }
9964 clear_vec_high(s, is_q, rd);
9965 }
9966 }
9967
9968 /* AdvSIMD scalar two reg misc
9969 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
9970 * +-----+---+-----------+------+-----------+--------+-----+------+------+
9971 * | 0 1 | U | 1 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
9972 * +-----+---+-----------+------+-----------+--------+-----+------+------+
9973 */
disas_simd_scalar_two_reg_misc(DisasContext * s,uint32_t insn)9974 static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn)
9975 {
9976 int rd = extract32(insn, 0, 5);
9977 int rn = extract32(insn, 5, 5);
9978 int opcode = extract32(insn, 12, 5);
9979 int size = extract32(insn, 22, 2);
9980 bool u = extract32(insn, 29, 1);
9981 bool is_fcvt = false;
9982 int rmode;
9983 TCGv_i32 tcg_rmode;
9984 TCGv_ptr tcg_fpstatus;
9985
9986 switch (opcode) {
9987 case 0x3: /* USQADD / SUQADD*/
9988 if (!fp_access_check(s)) {
9989 return;
9990 }
9991 handle_2misc_satacc(s, true, u, false, size, rn, rd);
9992 return;
9993 case 0x7: /* SQABS / SQNEG */
9994 break;
9995 case 0xa: /* CMLT */
9996 if (u) {
9997 unallocated_encoding(s);
9998 return;
9999 }
10000 /* fall through */
10001 case 0x8: /* CMGT, CMGE */
10002 case 0x9: /* CMEQ, CMLE */
10003 case 0xb: /* ABS, NEG */
10004 if (size != 3) {
10005 unallocated_encoding(s);
10006 return;
10007 }
10008 break;
10009 case 0x12: /* SQXTUN */
10010 if (!u) {
10011 unallocated_encoding(s);
10012 return;
10013 }
10014 /* fall through */
10015 case 0x14: /* SQXTN, UQXTN */
10016 if (size == 3) {
10017 unallocated_encoding(s);
10018 return;
10019 }
10020 if (!fp_access_check(s)) {
10021 return;
10022 }
10023 handle_2misc_narrow(s, true, opcode, u, false, size, rn, rd);
10024 return;
10025 case 0xc ... 0xf:
10026 case 0x16 ... 0x1d:
10027 case 0x1f:
10028 /* Floating point: U, size[1] and opcode indicate operation;
10029 * size[0] indicates single or double precision.
10030 */
10031 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
10032 size = extract32(size, 0, 1) ? 3 : 2;
10033 switch (opcode) {
10034 case 0x2c: /* FCMGT (zero) */
10035 case 0x2d: /* FCMEQ (zero) */
10036 case 0x2e: /* FCMLT (zero) */
10037 case 0x6c: /* FCMGE (zero) */
10038 case 0x6d: /* FCMLE (zero) */
10039 handle_2misc_fcmp_zero(s, opcode, true, u, true, size, rn, rd);
10040 return;
10041 case 0x1d: /* SCVTF */
10042 case 0x5d: /* UCVTF */
10043 {
10044 bool is_signed = (opcode == 0x1d);
10045 if (!fp_access_check(s)) {
10046 return;
10047 }
10048 handle_simd_intfp_conv(s, rd, rn, 1, is_signed, 0, size);
10049 return;
10050 }
10051 case 0x3d: /* FRECPE */
10052 case 0x3f: /* FRECPX */
10053 case 0x7d: /* FRSQRTE */
10054 if (!fp_access_check(s)) {
10055 return;
10056 }
10057 handle_2misc_reciprocal(s, opcode, true, u, true, size, rn, rd);
10058 return;
10059 case 0x1a: /* FCVTNS */
10060 case 0x1b: /* FCVTMS */
10061 case 0x3a: /* FCVTPS */
10062 case 0x3b: /* FCVTZS */
10063 case 0x5a: /* FCVTNU */
10064 case 0x5b: /* FCVTMU */
10065 case 0x7a: /* FCVTPU */
10066 case 0x7b: /* FCVTZU */
10067 is_fcvt = true;
10068 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
10069 break;
10070 case 0x1c: /* FCVTAS */
10071 case 0x5c: /* FCVTAU */
10072 /* TIEAWAY doesn't fit in the usual rounding mode encoding */
10073 is_fcvt = true;
10074 rmode = FPROUNDING_TIEAWAY;
10075 break;
10076 case 0x56: /* FCVTXN, FCVTXN2 */
10077 if (size == 2) {
10078 unallocated_encoding(s);
10079 return;
10080 }
10081 if (!fp_access_check(s)) {
10082 return;
10083 }
10084 handle_2misc_narrow(s, true, opcode, u, false, size - 1, rn, rd);
10085 return;
10086 default:
10087 unallocated_encoding(s);
10088 return;
10089 }
10090 break;
10091 default:
10092 unallocated_encoding(s);
10093 return;
10094 }
10095
10096 if (!fp_access_check(s)) {
10097 return;
10098 }
10099
10100 if (is_fcvt) {
10101 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
10102 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
10103 } else {
10104 tcg_fpstatus = NULL;
10105 tcg_rmode = NULL;
10106 }
10107
10108 if (size == 3) {
10109 TCGv_i64 tcg_rn = read_fp_dreg(s, rn);
10110 TCGv_i64 tcg_rd = tcg_temp_new_i64();
10111
10112 handle_2misc_64(s, opcode, u, tcg_rd, tcg_rn, tcg_rmode, tcg_fpstatus);
10113 write_fp_dreg(s, rd, tcg_rd);
10114 } else {
10115 TCGv_i32 tcg_rn = tcg_temp_new_i32();
10116 TCGv_i32 tcg_rd = tcg_temp_new_i32();
10117
10118 read_vec_element_i32(s, tcg_rn, rn, 0, size);
10119
10120 switch (opcode) {
10121 case 0x7: /* SQABS, SQNEG */
10122 {
10123 NeonGenOneOpEnvFn *genfn;
10124 static NeonGenOneOpEnvFn * const fns[3][2] = {
10125 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
10126 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
10127 { gen_helper_neon_qabs_s32, gen_helper_neon_qneg_s32 },
10128 };
10129 genfn = fns[size][u];
10130 genfn(tcg_rd, tcg_env, tcg_rn);
10131 break;
10132 }
10133 case 0x1a: /* FCVTNS */
10134 case 0x1b: /* FCVTMS */
10135 case 0x1c: /* FCVTAS */
10136 case 0x3a: /* FCVTPS */
10137 case 0x3b: /* FCVTZS */
10138 gen_helper_vfp_tosls(tcg_rd, tcg_rn, tcg_constant_i32(0),
10139 tcg_fpstatus);
10140 break;
10141 case 0x5a: /* FCVTNU */
10142 case 0x5b: /* FCVTMU */
10143 case 0x5c: /* FCVTAU */
10144 case 0x7a: /* FCVTPU */
10145 case 0x7b: /* FCVTZU */
10146 gen_helper_vfp_touls(tcg_rd, tcg_rn, tcg_constant_i32(0),
10147 tcg_fpstatus);
10148 break;
10149 default:
10150 g_assert_not_reached();
10151 }
10152
10153 write_fp_sreg(s, rd, tcg_rd);
10154 }
10155
10156 if (is_fcvt) {
10157 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
10158 }
10159 }
10160
10161 /* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */
handle_vec_simd_shri(DisasContext * s,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)10162 static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u,
10163 int immh, int immb, int opcode, int rn, int rd)
10164 {
10165 int size = 32 - clz32(immh) - 1;
10166 int immhb = immh << 3 | immb;
10167 int shift = 2 * (8 << size) - immhb;
10168 GVecGen2iFn *gvec_fn;
10169
10170 if (extract32(immh, 3, 1) && !is_q) {
10171 unallocated_encoding(s);
10172 return;
10173 }
10174 tcg_debug_assert(size <= 3);
10175
10176 if (!fp_access_check(s)) {
10177 return;
10178 }
10179
10180 switch (opcode) {
10181 case 0x02: /* SSRA / USRA (accumulate) */
10182 gvec_fn = is_u ? gen_gvec_usra : gen_gvec_ssra;
10183 break;
10184
10185 case 0x08: /* SRI */
10186 gvec_fn = gen_gvec_sri;
10187 break;
10188
10189 case 0x00: /* SSHR / USHR */
10190 if (is_u) {
10191 if (shift == 8 << size) {
10192 /* Shift count the same size as element size produces zero. */
10193 tcg_gen_gvec_dup_imm(size, vec_full_reg_offset(s, rd),
10194 is_q ? 16 : 8, vec_full_reg_size(s), 0);
10195 return;
10196 }
10197 gvec_fn = tcg_gen_gvec_shri;
10198 } else {
10199 /* Shift count the same size as element size produces all sign. */
10200 if (shift == 8 << size) {
10201 shift -= 1;
10202 }
10203 gvec_fn = tcg_gen_gvec_sari;
10204 }
10205 break;
10206
10207 case 0x04: /* SRSHR / URSHR (rounding) */
10208 gvec_fn = is_u ? gen_gvec_urshr : gen_gvec_srshr;
10209 break;
10210
10211 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10212 gvec_fn = is_u ? gen_gvec_ursra : gen_gvec_srsra;
10213 break;
10214
10215 default:
10216 g_assert_not_reached();
10217 }
10218
10219 gen_gvec_fn2i(s, is_q, rd, rn, shift, gvec_fn, size);
10220 }
10221
10222 /* SHL/SLI - Vector shift left */
handle_vec_simd_shli(DisasContext * s,bool is_q,bool insert,int immh,int immb,int opcode,int rn,int rd)10223 static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert,
10224 int immh, int immb, int opcode, int rn, int rd)
10225 {
10226 int size = 32 - clz32(immh) - 1;
10227 int immhb = immh << 3 | immb;
10228 int shift = immhb - (8 << size);
10229
10230 /* Range of size is limited by decode: immh is a non-zero 4 bit field */
10231 assert(size >= 0 && size <= 3);
10232
10233 if (extract32(immh, 3, 1) && !is_q) {
10234 unallocated_encoding(s);
10235 return;
10236 }
10237
10238 if (!fp_access_check(s)) {
10239 return;
10240 }
10241
10242 if (insert) {
10243 gen_gvec_fn2i(s, is_q, rd, rn, shift, gen_gvec_sli, size);
10244 } else {
10245 gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shli, size);
10246 }
10247 }
10248
10249 /* USHLL/SHLL - Vector shift left with widening */
handle_vec_simd_wshli(DisasContext * s,bool is_q,bool is_u,int immh,int immb,int opcode,int rn,int rd)10250 static void handle_vec_simd_wshli(DisasContext *s, bool is_q, bool is_u,
10251 int immh, int immb, int opcode, int rn, int rd)
10252 {
10253 int size = 32 - clz32(immh) - 1;
10254 int immhb = immh << 3 | immb;
10255 int shift = immhb - (8 << size);
10256 int dsize = 64;
10257 int esize = 8 << size;
10258 int elements = dsize/esize;
10259 TCGv_i64 tcg_rn = tcg_temp_new_i64();
10260 TCGv_i64 tcg_rd = tcg_temp_new_i64();
10261 int i;
10262
10263 if (size >= 3) {
10264 unallocated_encoding(s);
10265 return;
10266 }
10267
10268 if (!fp_access_check(s)) {
10269 return;
10270 }
10271
10272 /* For the LL variants the store is larger than the load,
10273 * so if rd == rn we would overwrite parts of our input.
10274 * So load everything right now and use shifts in the main loop.
10275 */
10276 read_vec_element(s, tcg_rn, rn, is_q ? 1 : 0, MO_64);
10277
10278 for (i = 0; i < elements; i++) {
10279 tcg_gen_shri_i64(tcg_rd, tcg_rn, i * esize);
10280 ext_and_shift_reg(tcg_rd, tcg_rd, size | (!is_u << 2), 0);
10281 tcg_gen_shli_i64(tcg_rd, tcg_rd, shift);
10282 write_vec_element(s, tcg_rd, rd, i, size + 1);
10283 }
10284 }
10285
10286 /* SHRN/RSHRN - Shift right with narrowing (and potential rounding) */
handle_vec_simd_shrn(DisasContext * s,bool is_q,int immh,int immb,int opcode,int rn,int rd)10287 static void handle_vec_simd_shrn(DisasContext *s, bool is_q,
10288 int immh, int immb, int opcode, int rn, int rd)
10289 {
10290 int immhb = immh << 3 | immb;
10291 int size = 32 - clz32(immh) - 1;
10292 int dsize = 64;
10293 int esize = 8 << size;
10294 int elements = dsize/esize;
10295 int shift = (2 * esize) - immhb;
10296 bool round = extract32(opcode, 0, 1);
10297 TCGv_i64 tcg_rn, tcg_rd, tcg_final;
10298 TCGv_i64 tcg_round;
10299 int i;
10300
10301 if (extract32(immh, 3, 1)) {
10302 unallocated_encoding(s);
10303 return;
10304 }
10305
10306 if (!fp_access_check(s)) {
10307 return;
10308 }
10309
10310 tcg_rn = tcg_temp_new_i64();
10311 tcg_rd = tcg_temp_new_i64();
10312 tcg_final = tcg_temp_new_i64();
10313 read_vec_element(s, tcg_final, rd, is_q ? 1 : 0, MO_64);
10314
10315 if (round) {
10316 tcg_round = tcg_constant_i64(1ULL << (shift - 1));
10317 } else {
10318 tcg_round = NULL;
10319 }
10320
10321 for (i = 0; i < elements; i++) {
10322 read_vec_element(s, tcg_rn, rn, i, size+1);
10323 handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
10324 false, true, size+1, shift);
10325
10326 tcg_gen_deposit_i64(tcg_final, tcg_final, tcg_rd, esize * i, esize);
10327 }
10328
10329 if (!is_q) {
10330 write_vec_element(s, tcg_final, rd, 0, MO_64);
10331 } else {
10332 write_vec_element(s, tcg_final, rd, 1, MO_64);
10333 }
10334
10335 clear_vec_high(s, is_q, rd);
10336 }
10337
10338
10339 /* AdvSIMD shift by immediate
10340 * 31 30 29 28 23 22 19 18 16 15 11 10 9 5 4 0
10341 * +---+---+---+-------------+------+------+--------+---+------+------+
10342 * | 0 | Q | U | 0 1 1 1 1 0 | immh | immb | opcode | 1 | Rn | Rd |
10343 * +---+---+---+-------------+------+------+--------+---+------+------+
10344 */
disas_simd_shift_imm(DisasContext * s,uint32_t insn)10345 static void disas_simd_shift_imm(DisasContext *s, uint32_t insn)
10346 {
10347 int rd = extract32(insn, 0, 5);
10348 int rn = extract32(insn, 5, 5);
10349 int opcode = extract32(insn, 11, 5);
10350 int immb = extract32(insn, 16, 3);
10351 int immh = extract32(insn, 19, 4);
10352 bool is_u = extract32(insn, 29, 1);
10353 bool is_q = extract32(insn, 30, 1);
10354
10355 /* data_proc_simd[] has sent immh == 0 to disas_simd_mod_imm. */
10356 assert(immh != 0);
10357
10358 switch (opcode) {
10359 case 0x08: /* SRI */
10360 if (!is_u) {
10361 unallocated_encoding(s);
10362 return;
10363 }
10364 /* fall through */
10365 case 0x00: /* SSHR / USHR */
10366 case 0x02: /* SSRA / USRA (accumulate) */
10367 case 0x04: /* SRSHR / URSHR (rounding) */
10368 case 0x06: /* SRSRA / URSRA (accum + rounding) */
10369 handle_vec_simd_shri(s, is_q, is_u, immh, immb, opcode, rn, rd);
10370 break;
10371 case 0x0a: /* SHL / SLI */
10372 handle_vec_simd_shli(s, is_q, is_u, immh, immb, opcode, rn, rd);
10373 break;
10374 case 0x10: /* SHRN */
10375 case 0x11: /* RSHRN / SQRSHRUN */
10376 if (is_u) {
10377 handle_vec_simd_sqshrn(s, false, is_q, false, true, immh, immb,
10378 opcode, rn, rd);
10379 } else {
10380 handle_vec_simd_shrn(s, is_q, immh, immb, opcode, rn, rd);
10381 }
10382 break;
10383 case 0x12: /* SQSHRN / UQSHRN */
10384 case 0x13: /* SQRSHRN / UQRSHRN */
10385 handle_vec_simd_sqshrn(s, false, is_q, is_u, is_u, immh, immb,
10386 opcode, rn, rd);
10387 break;
10388 case 0x14: /* SSHLL / USHLL */
10389 handle_vec_simd_wshli(s, is_q, is_u, immh, immb, opcode, rn, rd);
10390 break;
10391 case 0x1c: /* SCVTF / UCVTF */
10392 handle_simd_shift_intfp_conv(s, false, is_q, is_u, immh, immb,
10393 opcode, rn, rd);
10394 break;
10395 case 0xc: /* SQSHLU */
10396 if (!is_u) {
10397 unallocated_encoding(s);
10398 return;
10399 }
10400 handle_simd_qshl(s, false, is_q, false, true, immh, immb, rn, rd);
10401 break;
10402 case 0xe: /* SQSHL, UQSHL */
10403 handle_simd_qshl(s, false, is_q, is_u, is_u, immh, immb, rn, rd);
10404 break;
10405 case 0x1f: /* FCVTZS/ FCVTZU */
10406 handle_simd_shift_fpint_conv(s, false, is_q, is_u, immh, immb, rn, rd);
10407 return;
10408 default:
10409 unallocated_encoding(s);
10410 return;
10411 }
10412 }
10413
10414 /* Generate code to do a "long" addition or subtraction, ie one done in
10415 * TCGv_i64 on vector lanes twice the width specified by size.
10416 */
gen_neon_addl(int size,bool is_sub,TCGv_i64 tcg_res,TCGv_i64 tcg_op1,TCGv_i64 tcg_op2)10417 static void gen_neon_addl(int size, bool is_sub, TCGv_i64 tcg_res,
10418 TCGv_i64 tcg_op1, TCGv_i64 tcg_op2)
10419 {
10420 static NeonGenTwo64OpFn * const fns[3][2] = {
10421 { gen_helper_neon_addl_u16, gen_helper_neon_subl_u16 },
10422 { gen_helper_neon_addl_u32, gen_helper_neon_subl_u32 },
10423 { tcg_gen_add_i64, tcg_gen_sub_i64 },
10424 };
10425 NeonGenTwo64OpFn *genfn;
10426 assert(size < 3);
10427
10428 genfn = fns[size][is_sub];
10429 genfn(tcg_res, tcg_op1, tcg_op2);
10430 }
10431
handle_3rd_widening(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10432 static void handle_3rd_widening(DisasContext *s, int is_q, int is_u, int size,
10433 int opcode, int rd, int rn, int rm)
10434 {
10435 /* 3-reg-different widening insns: 64 x 64 -> 128 */
10436 TCGv_i64 tcg_res[2];
10437 int pass, accop;
10438
10439 tcg_res[0] = tcg_temp_new_i64();
10440 tcg_res[1] = tcg_temp_new_i64();
10441
10442 /* Does this op do an adding accumulate, a subtracting accumulate,
10443 * or no accumulate at all?
10444 */
10445 switch (opcode) {
10446 case 5:
10447 case 8:
10448 case 9:
10449 accop = 1;
10450 break;
10451 case 10:
10452 case 11:
10453 accop = -1;
10454 break;
10455 default:
10456 accop = 0;
10457 break;
10458 }
10459
10460 if (accop != 0) {
10461 read_vec_element(s, tcg_res[0], rd, 0, MO_64);
10462 read_vec_element(s, tcg_res[1], rd, 1, MO_64);
10463 }
10464
10465 /* size == 2 means two 32x32->64 operations; this is worth special
10466 * casing because we can generally handle it inline.
10467 */
10468 if (size == 2) {
10469 for (pass = 0; pass < 2; pass++) {
10470 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10471 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10472 TCGv_i64 tcg_passres;
10473 MemOp memop = MO_32 | (is_u ? 0 : MO_SIGN);
10474
10475 int elt = pass + is_q * 2;
10476
10477 read_vec_element(s, tcg_op1, rn, elt, memop);
10478 read_vec_element(s, tcg_op2, rm, elt, memop);
10479
10480 if (accop == 0) {
10481 tcg_passres = tcg_res[pass];
10482 } else {
10483 tcg_passres = tcg_temp_new_i64();
10484 }
10485
10486 switch (opcode) {
10487 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10488 tcg_gen_add_i64(tcg_passres, tcg_op1, tcg_op2);
10489 break;
10490 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10491 tcg_gen_sub_i64(tcg_passres, tcg_op1, tcg_op2);
10492 break;
10493 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10494 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10495 {
10496 TCGv_i64 tcg_tmp1 = tcg_temp_new_i64();
10497 TCGv_i64 tcg_tmp2 = tcg_temp_new_i64();
10498
10499 tcg_gen_sub_i64(tcg_tmp1, tcg_op1, tcg_op2);
10500 tcg_gen_sub_i64(tcg_tmp2, tcg_op2, tcg_op1);
10501 tcg_gen_movcond_i64(is_u ? TCG_COND_GEU : TCG_COND_GE,
10502 tcg_passres,
10503 tcg_op1, tcg_op2, tcg_tmp1, tcg_tmp2);
10504 break;
10505 }
10506 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10507 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10508 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10509 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2);
10510 break;
10511 case 9: /* SQDMLAL, SQDMLAL2 */
10512 case 11: /* SQDMLSL, SQDMLSL2 */
10513 case 13: /* SQDMULL, SQDMULL2 */
10514 tcg_gen_mul_i64(tcg_passres, tcg_op1, tcg_op2);
10515 gen_helper_neon_addl_saturate_s64(tcg_passres, tcg_env,
10516 tcg_passres, tcg_passres);
10517 break;
10518 default:
10519 g_assert_not_reached();
10520 }
10521
10522 if (opcode == 9 || opcode == 11) {
10523 /* saturating accumulate ops */
10524 if (accop < 0) {
10525 tcg_gen_neg_i64(tcg_passres, tcg_passres);
10526 }
10527 gen_helper_neon_addl_saturate_s64(tcg_res[pass], tcg_env,
10528 tcg_res[pass], tcg_passres);
10529 } else if (accop > 0) {
10530 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
10531 } else if (accop < 0) {
10532 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
10533 }
10534 }
10535 } else {
10536 /* size 0 or 1, generally helper functions */
10537 for (pass = 0; pass < 2; pass++) {
10538 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
10539 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10540 TCGv_i64 tcg_passres;
10541 int elt = pass + is_q * 2;
10542
10543 read_vec_element_i32(s, tcg_op1, rn, elt, MO_32);
10544 read_vec_element_i32(s, tcg_op2, rm, elt, MO_32);
10545
10546 if (accop == 0) {
10547 tcg_passres = tcg_res[pass];
10548 } else {
10549 tcg_passres = tcg_temp_new_i64();
10550 }
10551
10552 switch (opcode) {
10553 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10554 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10555 {
10556 TCGv_i64 tcg_op2_64 = tcg_temp_new_i64();
10557 static NeonGenWidenFn * const widenfns[2][2] = {
10558 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 },
10559 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 },
10560 };
10561 NeonGenWidenFn *widenfn = widenfns[size][is_u];
10562
10563 widenfn(tcg_op2_64, tcg_op2);
10564 widenfn(tcg_passres, tcg_op1);
10565 gen_neon_addl(size, (opcode == 2), tcg_passres,
10566 tcg_passres, tcg_op2_64);
10567 break;
10568 }
10569 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10570 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10571 if (size == 0) {
10572 if (is_u) {
10573 gen_helper_neon_abdl_u16(tcg_passres, tcg_op1, tcg_op2);
10574 } else {
10575 gen_helper_neon_abdl_s16(tcg_passres, tcg_op1, tcg_op2);
10576 }
10577 } else {
10578 if (is_u) {
10579 gen_helper_neon_abdl_u32(tcg_passres, tcg_op1, tcg_op2);
10580 } else {
10581 gen_helper_neon_abdl_s32(tcg_passres, tcg_op1, tcg_op2);
10582 }
10583 }
10584 break;
10585 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10586 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10587 case 12: /* UMULL, UMULL2, SMULL, SMULL2 */
10588 if (size == 0) {
10589 if (is_u) {
10590 gen_helper_neon_mull_u8(tcg_passres, tcg_op1, tcg_op2);
10591 } else {
10592 gen_helper_neon_mull_s8(tcg_passres, tcg_op1, tcg_op2);
10593 }
10594 } else {
10595 if (is_u) {
10596 gen_helper_neon_mull_u16(tcg_passres, tcg_op1, tcg_op2);
10597 } else {
10598 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2);
10599 }
10600 }
10601 break;
10602 case 9: /* SQDMLAL, SQDMLAL2 */
10603 case 11: /* SQDMLSL, SQDMLSL2 */
10604 case 13: /* SQDMULL, SQDMULL2 */
10605 assert(size == 1);
10606 gen_helper_neon_mull_s16(tcg_passres, tcg_op1, tcg_op2);
10607 gen_helper_neon_addl_saturate_s32(tcg_passres, tcg_env,
10608 tcg_passres, tcg_passres);
10609 break;
10610 default:
10611 g_assert_not_reached();
10612 }
10613
10614 if (accop != 0) {
10615 if (opcode == 9 || opcode == 11) {
10616 /* saturating accumulate ops */
10617 if (accop < 0) {
10618 gen_helper_neon_negl_u32(tcg_passres, tcg_passres);
10619 }
10620 gen_helper_neon_addl_saturate_s32(tcg_res[pass], tcg_env,
10621 tcg_res[pass],
10622 tcg_passres);
10623 } else {
10624 gen_neon_addl(size, (accop < 0), tcg_res[pass],
10625 tcg_res[pass], tcg_passres);
10626 }
10627 }
10628 }
10629 }
10630
10631 write_vec_element(s, tcg_res[0], rd, 0, MO_64);
10632 write_vec_element(s, tcg_res[1], rd, 1, MO_64);
10633 }
10634
handle_3rd_wide(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10635 static void handle_3rd_wide(DisasContext *s, int is_q, int is_u, int size,
10636 int opcode, int rd, int rn, int rm)
10637 {
10638 TCGv_i64 tcg_res[2];
10639 int part = is_q ? 2 : 0;
10640 int pass;
10641
10642 for (pass = 0; pass < 2; pass++) {
10643 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10644 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10645 TCGv_i64 tcg_op2_wide = tcg_temp_new_i64();
10646 static NeonGenWidenFn * const widenfns[3][2] = {
10647 { gen_helper_neon_widen_s8, gen_helper_neon_widen_u8 },
10648 { gen_helper_neon_widen_s16, gen_helper_neon_widen_u16 },
10649 { tcg_gen_ext_i32_i64, tcg_gen_extu_i32_i64 },
10650 };
10651 NeonGenWidenFn *widenfn = widenfns[size][is_u];
10652
10653 read_vec_element(s, tcg_op1, rn, pass, MO_64);
10654 read_vec_element_i32(s, tcg_op2, rm, part + pass, MO_32);
10655 widenfn(tcg_op2_wide, tcg_op2);
10656 tcg_res[pass] = tcg_temp_new_i64();
10657 gen_neon_addl(size, (opcode == 3),
10658 tcg_res[pass], tcg_op1, tcg_op2_wide);
10659 }
10660
10661 for (pass = 0; pass < 2; pass++) {
10662 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10663 }
10664 }
10665
do_narrow_round_high_u32(TCGv_i32 res,TCGv_i64 in)10666 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in)
10667 {
10668 tcg_gen_addi_i64(in, in, 1U << 31);
10669 tcg_gen_extrh_i64_i32(res, in);
10670 }
10671
handle_3rd_narrowing(DisasContext * s,int is_q,int is_u,int size,int opcode,int rd,int rn,int rm)10672 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
10673 int opcode, int rd, int rn, int rm)
10674 {
10675 TCGv_i32 tcg_res[2];
10676 int part = is_q ? 2 : 0;
10677 int pass;
10678
10679 for (pass = 0; pass < 2; pass++) {
10680 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10681 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10682 TCGv_i64 tcg_wideres = tcg_temp_new_i64();
10683 static NeonGenNarrowFn * const narrowfns[3][2] = {
10684 { gen_helper_neon_narrow_high_u8,
10685 gen_helper_neon_narrow_round_high_u8 },
10686 { gen_helper_neon_narrow_high_u16,
10687 gen_helper_neon_narrow_round_high_u16 },
10688 { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 },
10689 };
10690 NeonGenNarrowFn *gennarrow = narrowfns[size][is_u];
10691
10692 read_vec_element(s, tcg_op1, rn, pass, MO_64);
10693 read_vec_element(s, tcg_op2, rm, pass, MO_64);
10694
10695 gen_neon_addl(size, (opcode == 6), tcg_wideres, tcg_op1, tcg_op2);
10696
10697 tcg_res[pass] = tcg_temp_new_i32();
10698 gennarrow(tcg_res[pass], tcg_wideres);
10699 }
10700
10701 for (pass = 0; pass < 2; pass++) {
10702 write_vec_element_i32(s, tcg_res[pass], rd, pass + part, MO_32);
10703 }
10704 clear_vec_high(s, is_q, rd);
10705 }
10706
10707 /* AdvSIMD three different
10708 * 31 30 29 28 24 23 22 21 20 16 15 12 11 10 9 5 4 0
10709 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10710 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 0 0 | Rn | Rd |
10711 * +---+---+---+-----------+------+---+------+--------+-----+------+------+
10712 */
disas_simd_three_reg_diff(DisasContext * s,uint32_t insn)10713 static void disas_simd_three_reg_diff(DisasContext *s, uint32_t insn)
10714 {
10715 /* Instructions in this group fall into three basic classes
10716 * (in each case with the operation working on each element in
10717 * the input vectors):
10718 * (1) widening 64 x 64 -> 128 (with possibly Vd as an extra
10719 * 128 bit input)
10720 * (2) wide 64 x 128 -> 128
10721 * (3) narrowing 128 x 128 -> 64
10722 * Here we do initial decode, catch unallocated cases and
10723 * dispatch to separate functions for each class.
10724 */
10725 int is_q = extract32(insn, 30, 1);
10726 int is_u = extract32(insn, 29, 1);
10727 int size = extract32(insn, 22, 2);
10728 int opcode = extract32(insn, 12, 4);
10729 int rm = extract32(insn, 16, 5);
10730 int rn = extract32(insn, 5, 5);
10731 int rd = extract32(insn, 0, 5);
10732
10733 switch (opcode) {
10734 case 1: /* SADDW, SADDW2, UADDW, UADDW2 */
10735 case 3: /* SSUBW, SSUBW2, USUBW, USUBW2 */
10736 /* 64 x 128 -> 128 */
10737 if (size == 3) {
10738 unallocated_encoding(s);
10739 return;
10740 }
10741 if (!fp_access_check(s)) {
10742 return;
10743 }
10744 handle_3rd_wide(s, is_q, is_u, size, opcode, rd, rn, rm);
10745 break;
10746 case 4: /* ADDHN, ADDHN2, RADDHN, RADDHN2 */
10747 case 6: /* SUBHN, SUBHN2, RSUBHN, RSUBHN2 */
10748 /* 128 x 128 -> 64 */
10749 if (size == 3) {
10750 unallocated_encoding(s);
10751 return;
10752 }
10753 if (!fp_access_check(s)) {
10754 return;
10755 }
10756 handle_3rd_narrowing(s, is_q, is_u, size, opcode, rd, rn, rm);
10757 break;
10758 case 14: /* PMULL, PMULL2 */
10759 if (is_u) {
10760 unallocated_encoding(s);
10761 return;
10762 }
10763 switch (size) {
10764 case 0: /* PMULL.P8 */
10765 if (!fp_access_check(s)) {
10766 return;
10767 }
10768 /* The Q field specifies lo/hi half input for this insn. */
10769 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q,
10770 gen_helper_neon_pmull_h);
10771 break;
10772
10773 case 3: /* PMULL.P64 */
10774 if (!dc_isar_feature(aa64_pmull, s)) {
10775 unallocated_encoding(s);
10776 return;
10777 }
10778 if (!fp_access_check(s)) {
10779 return;
10780 }
10781 /* The Q field specifies lo/hi half input for this insn. */
10782 gen_gvec_op3_ool(s, true, rd, rn, rm, is_q,
10783 gen_helper_gvec_pmull_q);
10784 break;
10785
10786 default:
10787 unallocated_encoding(s);
10788 break;
10789 }
10790 return;
10791 case 9: /* SQDMLAL, SQDMLAL2 */
10792 case 11: /* SQDMLSL, SQDMLSL2 */
10793 case 13: /* SQDMULL, SQDMULL2 */
10794 if (is_u || size == 0) {
10795 unallocated_encoding(s);
10796 return;
10797 }
10798 /* fall through */
10799 case 0: /* SADDL, SADDL2, UADDL, UADDL2 */
10800 case 2: /* SSUBL, SSUBL2, USUBL, USUBL2 */
10801 case 5: /* SABAL, SABAL2, UABAL, UABAL2 */
10802 case 7: /* SABDL, SABDL2, UABDL, UABDL2 */
10803 case 8: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
10804 case 10: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
10805 case 12: /* SMULL, SMULL2, UMULL, UMULL2 */
10806 /* 64 x 64 -> 128 */
10807 if (size == 3) {
10808 unallocated_encoding(s);
10809 return;
10810 }
10811 if (!fp_access_check(s)) {
10812 return;
10813 }
10814
10815 handle_3rd_widening(s, is_q, is_u, size, opcode, rd, rn, rm);
10816 break;
10817 default:
10818 /* opcode 15 not allocated */
10819 unallocated_encoding(s);
10820 break;
10821 }
10822 }
10823
10824 /* Logic op (opcode == 3) subgroup of C3.6.16. */
disas_simd_3same_logic(DisasContext * s,uint32_t insn)10825 static void disas_simd_3same_logic(DisasContext *s, uint32_t insn)
10826 {
10827 int rd = extract32(insn, 0, 5);
10828 int rn = extract32(insn, 5, 5);
10829 int rm = extract32(insn, 16, 5);
10830 int size = extract32(insn, 22, 2);
10831 bool is_u = extract32(insn, 29, 1);
10832 bool is_q = extract32(insn, 30, 1);
10833
10834 if (!fp_access_check(s)) {
10835 return;
10836 }
10837
10838 switch (size + 4 * is_u) {
10839 case 0: /* AND */
10840 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_and, 0);
10841 return;
10842 case 1: /* BIC */
10843 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_andc, 0);
10844 return;
10845 case 2: /* ORR */
10846 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_or, 0);
10847 return;
10848 case 3: /* ORN */
10849 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_orc, 0);
10850 return;
10851 case 4: /* EOR */
10852 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_xor, 0);
10853 return;
10854
10855 case 5: /* BSL bitwise select */
10856 gen_gvec_fn4(s, is_q, rd, rd, rn, rm, tcg_gen_gvec_bitsel, 0);
10857 return;
10858 case 6: /* BIT, bitwise insert if true */
10859 gen_gvec_fn4(s, is_q, rd, rm, rn, rd, tcg_gen_gvec_bitsel, 0);
10860 return;
10861 case 7: /* BIF, bitwise insert if false */
10862 gen_gvec_fn4(s, is_q, rd, rm, rd, rn, tcg_gen_gvec_bitsel, 0);
10863 return;
10864
10865 default:
10866 g_assert_not_reached();
10867 }
10868 }
10869
10870 /* Pairwise op subgroup of C3.6.16.
10871 *
10872 * This is called directly or via the handle_3same_float for float pairwise
10873 * operations where the opcode and size are calculated differently.
10874 */
handle_simd_3same_pair(DisasContext * s,int is_q,int u,int opcode,int size,int rn,int rm,int rd)10875 static void handle_simd_3same_pair(DisasContext *s, int is_q, int u, int opcode,
10876 int size, int rn, int rm, int rd)
10877 {
10878 TCGv_ptr fpst;
10879 int pass;
10880
10881 /* Floating point operations need fpst */
10882 if (opcode >= 0x58) {
10883 fpst = fpstatus_ptr(FPST_FPCR);
10884 } else {
10885 fpst = NULL;
10886 }
10887
10888 if (!fp_access_check(s)) {
10889 return;
10890 }
10891
10892 /* These operations work on the concatenated rm:rn, with each pair of
10893 * adjacent elements being operated on to produce an element in the result.
10894 */
10895 if (size == 3) {
10896 TCGv_i64 tcg_res[2];
10897
10898 for (pass = 0; pass < 2; pass++) {
10899 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
10900 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
10901 int passreg = (pass == 0) ? rn : rm;
10902
10903 read_vec_element(s, tcg_op1, passreg, 0, MO_64);
10904 read_vec_element(s, tcg_op2, passreg, 1, MO_64);
10905 tcg_res[pass] = tcg_temp_new_i64();
10906
10907 switch (opcode) {
10908 case 0x17: /* ADDP */
10909 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2);
10910 break;
10911 case 0x58: /* FMAXNMP */
10912 gen_helper_vfp_maxnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10913 break;
10914 case 0x5a: /* FADDP */
10915 gen_helper_vfp_addd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10916 break;
10917 case 0x5e: /* FMAXP */
10918 gen_helper_vfp_maxd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10919 break;
10920 case 0x78: /* FMINNMP */
10921 gen_helper_vfp_minnumd(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10922 break;
10923 case 0x7e: /* FMINP */
10924 gen_helper_vfp_mind(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10925 break;
10926 default:
10927 g_assert_not_reached();
10928 }
10929 }
10930
10931 for (pass = 0; pass < 2; pass++) {
10932 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
10933 }
10934 } else {
10935 int maxpass = is_q ? 4 : 2;
10936 TCGv_i32 tcg_res[4];
10937
10938 for (pass = 0; pass < maxpass; pass++) {
10939 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
10940 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
10941 NeonGenTwoOpFn *genfn = NULL;
10942 int passreg = pass < (maxpass / 2) ? rn : rm;
10943 int passelt = (is_q && (pass & 1)) ? 2 : 0;
10944
10945 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_32);
10946 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_32);
10947 tcg_res[pass] = tcg_temp_new_i32();
10948
10949 switch (opcode) {
10950 case 0x17: /* ADDP */
10951 {
10952 static NeonGenTwoOpFn * const fns[3] = {
10953 gen_helper_neon_padd_u8,
10954 gen_helper_neon_padd_u16,
10955 tcg_gen_add_i32,
10956 };
10957 genfn = fns[size];
10958 break;
10959 }
10960 case 0x14: /* SMAXP, UMAXP */
10961 {
10962 static NeonGenTwoOpFn * const fns[3][2] = {
10963 { gen_helper_neon_pmax_s8, gen_helper_neon_pmax_u8 },
10964 { gen_helper_neon_pmax_s16, gen_helper_neon_pmax_u16 },
10965 { tcg_gen_smax_i32, tcg_gen_umax_i32 },
10966 };
10967 genfn = fns[size][u];
10968 break;
10969 }
10970 case 0x15: /* SMINP, UMINP */
10971 {
10972 static NeonGenTwoOpFn * const fns[3][2] = {
10973 { gen_helper_neon_pmin_s8, gen_helper_neon_pmin_u8 },
10974 { gen_helper_neon_pmin_s16, gen_helper_neon_pmin_u16 },
10975 { tcg_gen_smin_i32, tcg_gen_umin_i32 },
10976 };
10977 genfn = fns[size][u];
10978 break;
10979 }
10980 /* The FP operations are all on single floats (32 bit) */
10981 case 0x58: /* FMAXNMP */
10982 gen_helper_vfp_maxnums(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10983 break;
10984 case 0x5a: /* FADDP */
10985 gen_helper_vfp_adds(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10986 break;
10987 case 0x5e: /* FMAXP */
10988 gen_helper_vfp_maxs(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10989 break;
10990 case 0x78: /* FMINNMP */
10991 gen_helper_vfp_minnums(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10992 break;
10993 case 0x7e: /* FMINP */
10994 gen_helper_vfp_mins(tcg_res[pass], tcg_op1, tcg_op2, fpst);
10995 break;
10996 default:
10997 g_assert_not_reached();
10998 }
10999
11000 /* FP ops called directly, otherwise call now */
11001 if (genfn) {
11002 genfn(tcg_res[pass], tcg_op1, tcg_op2);
11003 }
11004 }
11005
11006 for (pass = 0; pass < maxpass; pass++) {
11007 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32);
11008 }
11009 clear_vec_high(s, is_q, rd);
11010 }
11011 }
11012
11013 /* Floating point op subgroup of C3.6.16. */
disas_simd_3same_float(DisasContext * s,uint32_t insn)11014 static void disas_simd_3same_float(DisasContext *s, uint32_t insn)
11015 {
11016 /* For floating point ops, the U, size[1] and opcode bits
11017 * together indicate the operation. size[0] indicates single
11018 * or double.
11019 */
11020 int fpopcode = extract32(insn, 11, 5)
11021 | (extract32(insn, 23, 1) << 5)
11022 | (extract32(insn, 29, 1) << 6);
11023 int is_q = extract32(insn, 30, 1);
11024 int size = extract32(insn, 22, 1);
11025 int rm = extract32(insn, 16, 5);
11026 int rn = extract32(insn, 5, 5);
11027 int rd = extract32(insn, 0, 5);
11028
11029 int datasize = is_q ? 128 : 64;
11030 int esize = 32 << size;
11031 int elements = datasize / esize;
11032
11033 if (size == 1 && !is_q) {
11034 unallocated_encoding(s);
11035 return;
11036 }
11037
11038 switch (fpopcode) {
11039 case 0x58: /* FMAXNMP */
11040 case 0x5a: /* FADDP */
11041 case 0x5e: /* FMAXP */
11042 case 0x78: /* FMINNMP */
11043 case 0x7e: /* FMINP */
11044 if (size && !is_q) {
11045 unallocated_encoding(s);
11046 return;
11047 }
11048 handle_simd_3same_pair(s, is_q, 0, fpopcode, size ? MO_64 : MO_32,
11049 rn, rm, rd);
11050 return;
11051 case 0x1b: /* FMULX */
11052 case 0x1f: /* FRECPS */
11053 case 0x3f: /* FRSQRTS */
11054 case 0x5d: /* FACGE */
11055 case 0x7d: /* FACGT */
11056 case 0x19: /* FMLA */
11057 case 0x39: /* FMLS */
11058 case 0x18: /* FMAXNM */
11059 case 0x1a: /* FADD */
11060 case 0x1c: /* FCMEQ */
11061 case 0x1e: /* FMAX */
11062 case 0x38: /* FMINNM */
11063 case 0x3a: /* FSUB */
11064 case 0x3e: /* FMIN */
11065 case 0x5b: /* FMUL */
11066 case 0x5c: /* FCMGE */
11067 case 0x5f: /* FDIV */
11068 case 0x7a: /* FABD */
11069 case 0x7c: /* FCMGT */
11070 if (!fp_access_check(s)) {
11071 return;
11072 }
11073 handle_3same_float(s, size, elements, fpopcode, rd, rn, rm);
11074 return;
11075
11076 case 0x1d: /* FMLAL */
11077 case 0x3d: /* FMLSL */
11078 case 0x59: /* FMLAL2 */
11079 case 0x79: /* FMLSL2 */
11080 if (size & 1 || !dc_isar_feature(aa64_fhm, s)) {
11081 unallocated_encoding(s);
11082 return;
11083 }
11084 if (fp_access_check(s)) {
11085 int is_s = extract32(insn, 23, 1);
11086 int is_2 = extract32(insn, 29, 1);
11087 int data = (is_2 << 1) | is_s;
11088 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
11089 vec_full_reg_offset(s, rn),
11090 vec_full_reg_offset(s, rm), tcg_env,
11091 is_q ? 16 : 8, vec_full_reg_size(s),
11092 data, gen_helper_gvec_fmlal_a64);
11093 }
11094 return;
11095
11096 default:
11097 unallocated_encoding(s);
11098 return;
11099 }
11100 }
11101
11102 /* Integer op subgroup of C3.6.16. */
disas_simd_3same_int(DisasContext * s,uint32_t insn)11103 static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
11104 {
11105 int is_q = extract32(insn, 30, 1);
11106 int u = extract32(insn, 29, 1);
11107 int size = extract32(insn, 22, 2);
11108 int opcode = extract32(insn, 11, 5);
11109 int rm = extract32(insn, 16, 5);
11110 int rn = extract32(insn, 5, 5);
11111 int rd = extract32(insn, 0, 5);
11112 int pass;
11113 TCGCond cond;
11114
11115 switch (opcode) {
11116 case 0x13: /* MUL, PMUL */
11117 if (u && size != 0) {
11118 unallocated_encoding(s);
11119 return;
11120 }
11121 /* fall through */
11122 case 0x0: /* SHADD, UHADD */
11123 case 0x2: /* SRHADD, URHADD */
11124 case 0x4: /* SHSUB, UHSUB */
11125 case 0xc: /* SMAX, UMAX */
11126 case 0xd: /* SMIN, UMIN */
11127 case 0xe: /* SABD, UABD */
11128 case 0xf: /* SABA, UABA */
11129 case 0x12: /* MLA, MLS */
11130 if (size == 3) {
11131 unallocated_encoding(s);
11132 return;
11133 }
11134 break;
11135 case 0x16: /* SQDMULH, SQRDMULH */
11136 if (size == 0 || size == 3) {
11137 unallocated_encoding(s);
11138 return;
11139 }
11140 break;
11141 default:
11142 if (size == 3 && !is_q) {
11143 unallocated_encoding(s);
11144 return;
11145 }
11146 break;
11147 }
11148
11149 if (!fp_access_check(s)) {
11150 return;
11151 }
11152
11153 switch (opcode) {
11154 case 0x01: /* SQADD, UQADD */
11155 if (u) {
11156 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqadd_qc, size);
11157 } else {
11158 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqadd_qc, size);
11159 }
11160 return;
11161 case 0x05: /* SQSUB, UQSUB */
11162 if (u) {
11163 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uqsub_qc, size);
11164 } else {
11165 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqsub_qc, size);
11166 }
11167 return;
11168 case 0x08: /* SSHL, USHL */
11169 if (u) {
11170 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_ushl, size);
11171 } else {
11172 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sshl, size);
11173 }
11174 return;
11175 case 0x0c: /* SMAX, UMAX */
11176 if (u) {
11177 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umax, size);
11178 } else {
11179 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smax, size);
11180 }
11181 return;
11182 case 0x0d: /* SMIN, UMIN */
11183 if (u) {
11184 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_umin, size);
11185 } else {
11186 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_smin, size);
11187 }
11188 return;
11189 case 0xe: /* SABD, UABD */
11190 if (u) {
11191 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uabd, size);
11192 } else {
11193 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sabd, size);
11194 }
11195 return;
11196 case 0xf: /* SABA, UABA */
11197 if (u) {
11198 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_uaba, size);
11199 } else {
11200 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_saba, size);
11201 }
11202 return;
11203 case 0x10: /* ADD, SUB */
11204 if (u) {
11205 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_sub, size);
11206 } else {
11207 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_add, size);
11208 }
11209 return;
11210 case 0x13: /* MUL, PMUL */
11211 if (!u) { /* MUL */
11212 gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_mul, size);
11213 } else { /* PMUL */
11214 gen_gvec_op3_ool(s, is_q, rd, rn, rm, 0, gen_helper_gvec_pmul_b);
11215 }
11216 return;
11217 case 0x12: /* MLA, MLS */
11218 if (u) {
11219 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mls, size);
11220 } else {
11221 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_mla, size);
11222 }
11223 return;
11224 case 0x16: /* SQDMULH, SQRDMULH */
11225 {
11226 static gen_helper_gvec_3_ptr * const fns[2][2] = {
11227 { gen_helper_neon_sqdmulh_h, gen_helper_neon_sqrdmulh_h },
11228 { gen_helper_neon_sqdmulh_s, gen_helper_neon_sqrdmulh_s },
11229 };
11230 gen_gvec_op3_qc(s, is_q, rd, rn, rm, fns[size - 1][u]);
11231 }
11232 return;
11233 case 0x11:
11234 if (!u) { /* CMTST */
11235 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_cmtst, size);
11236 return;
11237 }
11238 /* else CMEQ */
11239 cond = TCG_COND_EQ;
11240 goto do_gvec_cmp;
11241 case 0x06: /* CMGT, CMHI */
11242 cond = u ? TCG_COND_GTU : TCG_COND_GT;
11243 goto do_gvec_cmp;
11244 case 0x07: /* CMGE, CMHS */
11245 cond = u ? TCG_COND_GEU : TCG_COND_GE;
11246 do_gvec_cmp:
11247 tcg_gen_gvec_cmp(cond, size, vec_full_reg_offset(s, rd),
11248 vec_full_reg_offset(s, rn),
11249 vec_full_reg_offset(s, rm),
11250 is_q ? 16 : 8, vec_full_reg_size(s));
11251 return;
11252 }
11253
11254 if (size == 3) {
11255 assert(is_q);
11256 for (pass = 0; pass < 2; pass++) {
11257 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
11258 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
11259 TCGv_i64 tcg_res = tcg_temp_new_i64();
11260
11261 read_vec_element(s, tcg_op1, rn, pass, MO_64);
11262 read_vec_element(s, tcg_op2, rm, pass, MO_64);
11263
11264 handle_3same_64(s, opcode, u, tcg_res, tcg_op1, tcg_op2);
11265
11266 write_vec_element(s, tcg_res, rd, pass, MO_64);
11267 }
11268 } else {
11269 for (pass = 0; pass < (is_q ? 4 : 2); pass++) {
11270 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11271 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11272 TCGv_i32 tcg_res = tcg_temp_new_i32();
11273 NeonGenTwoOpFn *genfn = NULL;
11274 NeonGenTwoOpEnvFn *genenvfn = NULL;
11275
11276 read_vec_element_i32(s, tcg_op1, rn, pass, MO_32);
11277 read_vec_element_i32(s, tcg_op2, rm, pass, MO_32);
11278
11279 switch (opcode) {
11280 case 0x0: /* SHADD, UHADD */
11281 {
11282 static NeonGenTwoOpFn * const fns[3][2] = {
11283 { gen_helper_neon_hadd_s8, gen_helper_neon_hadd_u8 },
11284 { gen_helper_neon_hadd_s16, gen_helper_neon_hadd_u16 },
11285 { gen_helper_neon_hadd_s32, gen_helper_neon_hadd_u32 },
11286 };
11287 genfn = fns[size][u];
11288 break;
11289 }
11290 case 0x2: /* SRHADD, URHADD */
11291 {
11292 static NeonGenTwoOpFn * const fns[3][2] = {
11293 { gen_helper_neon_rhadd_s8, gen_helper_neon_rhadd_u8 },
11294 { gen_helper_neon_rhadd_s16, gen_helper_neon_rhadd_u16 },
11295 { gen_helper_neon_rhadd_s32, gen_helper_neon_rhadd_u32 },
11296 };
11297 genfn = fns[size][u];
11298 break;
11299 }
11300 case 0x4: /* SHSUB, UHSUB */
11301 {
11302 static NeonGenTwoOpFn * const fns[3][2] = {
11303 { gen_helper_neon_hsub_s8, gen_helper_neon_hsub_u8 },
11304 { gen_helper_neon_hsub_s16, gen_helper_neon_hsub_u16 },
11305 { gen_helper_neon_hsub_s32, gen_helper_neon_hsub_u32 },
11306 };
11307 genfn = fns[size][u];
11308 break;
11309 }
11310 case 0x9: /* SQSHL, UQSHL */
11311 {
11312 static NeonGenTwoOpEnvFn * const fns[3][2] = {
11313 { gen_helper_neon_qshl_s8, gen_helper_neon_qshl_u8 },
11314 { gen_helper_neon_qshl_s16, gen_helper_neon_qshl_u16 },
11315 { gen_helper_neon_qshl_s32, gen_helper_neon_qshl_u32 },
11316 };
11317 genenvfn = fns[size][u];
11318 break;
11319 }
11320 case 0xa: /* SRSHL, URSHL */
11321 {
11322 static NeonGenTwoOpFn * const fns[3][2] = {
11323 { gen_helper_neon_rshl_s8, gen_helper_neon_rshl_u8 },
11324 { gen_helper_neon_rshl_s16, gen_helper_neon_rshl_u16 },
11325 { gen_helper_neon_rshl_s32, gen_helper_neon_rshl_u32 },
11326 };
11327 genfn = fns[size][u];
11328 break;
11329 }
11330 case 0xb: /* SQRSHL, UQRSHL */
11331 {
11332 static NeonGenTwoOpEnvFn * const fns[3][2] = {
11333 { gen_helper_neon_qrshl_s8, gen_helper_neon_qrshl_u8 },
11334 { gen_helper_neon_qrshl_s16, gen_helper_neon_qrshl_u16 },
11335 { gen_helper_neon_qrshl_s32, gen_helper_neon_qrshl_u32 },
11336 };
11337 genenvfn = fns[size][u];
11338 break;
11339 }
11340 default:
11341 g_assert_not_reached();
11342 }
11343
11344 if (genenvfn) {
11345 genenvfn(tcg_res, tcg_env, tcg_op1, tcg_op2);
11346 } else {
11347 genfn(tcg_res, tcg_op1, tcg_op2);
11348 }
11349
11350 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
11351 }
11352 }
11353 clear_vec_high(s, is_q, rd);
11354 }
11355
11356 /* AdvSIMD three same
11357 * 31 30 29 28 24 23 22 21 20 16 15 11 10 9 5 4 0
11358 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11359 * | 0 | Q | U | 0 1 1 1 0 | size | 1 | Rm | opcode | 1 | Rn | Rd |
11360 * +---+---+---+-----------+------+---+------+--------+---+------+------+
11361 */
disas_simd_three_reg_same(DisasContext * s,uint32_t insn)11362 static void disas_simd_three_reg_same(DisasContext *s, uint32_t insn)
11363 {
11364 int opcode = extract32(insn, 11, 5);
11365
11366 switch (opcode) {
11367 case 0x3: /* logic ops */
11368 disas_simd_3same_logic(s, insn);
11369 break;
11370 case 0x17: /* ADDP */
11371 case 0x14: /* SMAXP, UMAXP */
11372 case 0x15: /* SMINP, UMINP */
11373 {
11374 /* Pairwise operations */
11375 int is_q = extract32(insn, 30, 1);
11376 int u = extract32(insn, 29, 1);
11377 int size = extract32(insn, 22, 2);
11378 int rm = extract32(insn, 16, 5);
11379 int rn = extract32(insn, 5, 5);
11380 int rd = extract32(insn, 0, 5);
11381 if (opcode == 0x17) {
11382 if (u || (size == 3 && !is_q)) {
11383 unallocated_encoding(s);
11384 return;
11385 }
11386 } else {
11387 if (size == 3) {
11388 unallocated_encoding(s);
11389 return;
11390 }
11391 }
11392 handle_simd_3same_pair(s, is_q, u, opcode, size, rn, rm, rd);
11393 break;
11394 }
11395 case 0x18 ... 0x31:
11396 /* floating point ops, sz[1] and U are part of opcode */
11397 disas_simd_3same_float(s, insn);
11398 break;
11399 default:
11400 disas_simd_3same_int(s, insn);
11401 break;
11402 }
11403 }
11404
11405 /*
11406 * Advanced SIMD three same (ARMv8.2 FP16 variants)
11407 *
11408 * 31 30 29 28 24 23 22 21 20 16 15 14 13 11 10 9 5 4 0
11409 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+
11410 * | 0 | Q | U | 0 1 1 1 0 | a | 1 0 | Rm | 0 0 | opcode | 1 | Rn | Rd |
11411 * +---+---+---+-----------+---------+------+-----+--------+---+------+------+
11412 *
11413 * This includes FMULX, FCMEQ (register), FRECPS, FRSQRTS, FCMGE
11414 * (register), FACGE, FABD, FCMGT (register) and FACGT.
11415 *
11416 */
disas_simd_three_reg_same_fp16(DisasContext * s,uint32_t insn)11417 static void disas_simd_three_reg_same_fp16(DisasContext *s, uint32_t insn)
11418 {
11419 int opcode = extract32(insn, 11, 3);
11420 int u = extract32(insn, 29, 1);
11421 int a = extract32(insn, 23, 1);
11422 int is_q = extract32(insn, 30, 1);
11423 int rm = extract32(insn, 16, 5);
11424 int rn = extract32(insn, 5, 5);
11425 int rd = extract32(insn, 0, 5);
11426 /*
11427 * For these floating point ops, the U, a and opcode bits
11428 * together indicate the operation.
11429 */
11430 int fpopcode = opcode | (a << 3) | (u << 4);
11431 int datasize = is_q ? 128 : 64;
11432 int elements = datasize / 16;
11433 bool pairwise;
11434 TCGv_ptr fpst;
11435 int pass;
11436
11437 switch (fpopcode) {
11438 case 0x0: /* FMAXNM */
11439 case 0x1: /* FMLA */
11440 case 0x2: /* FADD */
11441 case 0x3: /* FMULX */
11442 case 0x4: /* FCMEQ */
11443 case 0x6: /* FMAX */
11444 case 0x7: /* FRECPS */
11445 case 0x8: /* FMINNM */
11446 case 0x9: /* FMLS */
11447 case 0xa: /* FSUB */
11448 case 0xe: /* FMIN */
11449 case 0xf: /* FRSQRTS */
11450 case 0x13: /* FMUL */
11451 case 0x14: /* FCMGE */
11452 case 0x15: /* FACGE */
11453 case 0x17: /* FDIV */
11454 case 0x1a: /* FABD */
11455 case 0x1c: /* FCMGT */
11456 case 0x1d: /* FACGT */
11457 pairwise = false;
11458 break;
11459 case 0x10: /* FMAXNMP */
11460 case 0x12: /* FADDP */
11461 case 0x16: /* FMAXP */
11462 case 0x18: /* FMINNMP */
11463 case 0x1e: /* FMINP */
11464 pairwise = true;
11465 break;
11466 default:
11467 unallocated_encoding(s);
11468 return;
11469 }
11470
11471 if (!dc_isar_feature(aa64_fp16, s)) {
11472 unallocated_encoding(s);
11473 return;
11474 }
11475
11476 if (!fp_access_check(s)) {
11477 return;
11478 }
11479
11480 fpst = fpstatus_ptr(FPST_FPCR_F16);
11481
11482 if (pairwise) {
11483 int maxpass = is_q ? 8 : 4;
11484 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11485 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11486 TCGv_i32 tcg_res[8];
11487
11488 for (pass = 0; pass < maxpass; pass++) {
11489 int passreg = pass < (maxpass / 2) ? rn : rm;
11490 int passelt = (pass << 1) & (maxpass - 1);
11491
11492 read_vec_element_i32(s, tcg_op1, passreg, passelt, MO_16);
11493 read_vec_element_i32(s, tcg_op2, passreg, passelt + 1, MO_16);
11494 tcg_res[pass] = tcg_temp_new_i32();
11495
11496 switch (fpopcode) {
11497 case 0x10: /* FMAXNMP */
11498 gen_helper_advsimd_maxnumh(tcg_res[pass], tcg_op1, tcg_op2,
11499 fpst);
11500 break;
11501 case 0x12: /* FADDP */
11502 gen_helper_advsimd_addh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11503 break;
11504 case 0x16: /* FMAXP */
11505 gen_helper_advsimd_maxh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11506 break;
11507 case 0x18: /* FMINNMP */
11508 gen_helper_advsimd_minnumh(tcg_res[pass], tcg_op1, tcg_op2,
11509 fpst);
11510 break;
11511 case 0x1e: /* FMINP */
11512 gen_helper_advsimd_minh(tcg_res[pass], tcg_op1, tcg_op2, fpst);
11513 break;
11514 default:
11515 g_assert_not_reached();
11516 }
11517 }
11518
11519 for (pass = 0; pass < maxpass; pass++) {
11520 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_16);
11521 }
11522 } else {
11523 for (pass = 0; pass < elements; pass++) {
11524 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
11525 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
11526 TCGv_i32 tcg_res = tcg_temp_new_i32();
11527
11528 read_vec_element_i32(s, tcg_op1, rn, pass, MO_16);
11529 read_vec_element_i32(s, tcg_op2, rm, pass, MO_16);
11530
11531 switch (fpopcode) {
11532 case 0x0: /* FMAXNM */
11533 gen_helper_advsimd_maxnumh(tcg_res, tcg_op1, tcg_op2, fpst);
11534 break;
11535 case 0x1: /* FMLA */
11536 read_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11537 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res,
11538 fpst);
11539 break;
11540 case 0x2: /* FADD */
11541 gen_helper_advsimd_addh(tcg_res, tcg_op1, tcg_op2, fpst);
11542 break;
11543 case 0x3: /* FMULX */
11544 gen_helper_advsimd_mulxh(tcg_res, tcg_op1, tcg_op2, fpst);
11545 break;
11546 case 0x4: /* FCMEQ */
11547 gen_helper_advsimd_ceq_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11548 break;
11549 case 0x6: /* FMAX */
11550 gen_helper_advsimd_maxh(tcg_res, tcg_op1, tcg_op2, fpst);
11551 break;
11552 case 0x7: /* FRECPS */
11553 gen_helper_recpsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11554 break;
11555 case 0x8: /* FMINNM */
11556 gen_helper_advsimd_minnumh(tcg_res, tcg_op1, tcg_op2, fpst);
11557 break;
11558 case 0x9: /* FMLS */
11559 /* As usual for ARM, separate negation for fused multiply-add */
11560 tcg_gen_xori_i32(tcg_op1, tcg_op1, 0x8000);
11561 read_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11562 gen_helper_advsimd_muladdh(tcg_res, tcg_op1, tcg_op2, tcg_res,
11563 fpst);
11564 break;
11565 case 0xa: /* FSUB */
11566 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
11567 break;
11568 case 0xe: /* FMIN */
11569 gen_helper_advsimd_minh(tcg_res, tcg_op1, tcg_op2, fpst);
11570 break;
11571 case 0xf: /* FRSQRTS */
11572 gen_helper_rsqrtsf_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11573 break;
11574 case 0x13: /* FMUL */
11575 gen_helper_advsimd_mulh(tcg_res, tcg_op1, tcg_op2, fpst);
11576 break;
11577 case 0x14: /* FCMGE */
11578 gen_helper_advsimd_cge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11579 break;
11580 case 0x15: /* FACGE */
11581 gen_helper_advsimd_acge_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11582 break;
11583 case 0x17: /* FDIV */
11584 gen_helper_advsimd_divh(tcg_res, tcg_op1, tcg_op2, fpst);
11585 break;
11586 case 0x1a: /* FABD */
11587 gen_helper_advsimd_subh(tcg_res, tcg_op1, tcg_op2, fpst);
11588 tcg_gen_andi_i32(tcg_res, tcg_res, 0x7fff);
11589 break;
11590 case 0x1c: /* FCMGT */
11591 gen_helper_advsimd_cgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11592 break;
11593 case 0x1d: /* FACGT */
11594 gen_helper_advsimd_acgt_f16(tcg_res, tcg_op1, tcg_op2, fpst);
11595 break;
11596 default:
11597 g_assert_not_reached();
11598 }
11599
11600 write_vec_element_i32(s, tcg_res, rd, pass, MO_16);
11601 }
11602 }
11603
11604 clear_vec_high(s, is_q, rd);
11605 }
11606
11607 /* AdvSIMD three same extra
11608 * 31 30 29 28 24 23 22 21 20 16 15 14 11 10 9 5 4 0
11609 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11610 * | 0 | Q | U | 0 1 1 1 0 | size | 0 | Rm | 1 | opcode | 1 | Rn | Rd |
11611 * +---+---+---+-----------+------+---+------+---+--------+---+----+----+
11612 */
disas_simd_three_reg_same_extra(DisasContext * s,uint32_t insn)11613 static void disas_simd_three_reg_same_extra(DisasContext *s, uint32_t insn)
11614 {
11615 int rd = extract32(insn, 0, 5);
11616 int rn = extract32(insn, 5, 5);
11617 int opcode = extract32(insn, 11, 4);
11618 int rm = extract32(insn, 16, 5);
11619 int size = extract32(insn, 22, 2);
11620 bool u = extract32(insn, 29, 1);
11621 bool is_q = extract32(insn, 30, 1);
11622 bool feature;
11623 int rot;
11624
11625 switch (u * 16 + opcode) {
11626 case 0x10: /* SQRDMLAH (vector) */
11627 case 0x11: /* SQRDMLSH (vector) */
11628 if (size != 1 && size != 2) {
11629 unallocated_encoding(s);
11630 return;
11631 }
11632 feature = dc_isar_feature(aa64_rdm, s);
11633 break;
11634 case 0x02: /* SDOT (vector) */
11635 case 0x12: /* UDOT (vector) */
11636 if (size != MO_32) {
11637 unallocated_encoding(s);
11638 return;
11639 }
11640 feature = dc_isar_feature(aa64_dp, s);
11641 break;
11642 case 0x03: /* USDOT */
11643 if (size != MO_32) {
11644 unallocated_encoding(s);
11645 return;
11646 }
11647 feature = dc_isar_feature(aa64_i8mm, s);
11648 break;
11649 case 0x04: /* SMMLA */
11650 case 0x14: /* UMMLA */
11651 case 0x05: /* USMMLA */
11652 if (!is_q || size != MO_32) {
11653 unallocated_encoding(s);
11654 return;
11655 }
11656 feature = dc_isar_feature(aa64_i8mm, s);
11657 break;
11658 case 0x18: /* FCMLA, #0 */
11659 case 0x19: /* FCMLA, #90 */
11660 case 0x1a: /* FCMLA, #180 */
11661 case 0x1b: /* FCMLA, #270 */
11662 case 0x1c: /* FCADD, #90 */
11663 case 0x1e: /* FCADD, #270 */
11664 if (size == 0
11665 || (size == 1 && !dc_isar_feature(aa64_fp16, s))
11666 || (size == 3 && !is_q)) {
11667 unallocated_encoding(s);
11668 return;
11669 }
11670 feature = dc_isar_feature(aa64_fcma, s);
11671 break;
11672 case 0x1d: /* BFMMLA */
11673 if (size != MO_16 || !is_q) {
11674 unallocated_encoding(s);
11675 return;
11676 }
11677 feature = dc_isar_feature(aa64_bf16, s);
11678 break;
11679 case 0x1f:
11680 switch (size) {
11681 case 1: /* BFDOT */
11682 case 3: /* BFMLAL{B,T} */
11683 feature = dc_isar_feature(aa64_bf16, s);
11684 break;
11685 default:
11686 unallocated_encoding(s);
11687 return;
11688 }
11689 break;
11690 default:
11691 unallocated_encoding(s);
11692 return;
11693 }
11694 if (!feature) {
11695 unallocated_encoding(s);
11696 return;
11697 }
11698 if (!fp_access_check(s)) {
11699 return;
11700 }
11701
11702 switch (opcode) {
11703 case 0x0: /* SQRDMLAH (vector) */
11704 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlah_qc, size);
11705 return;
11706
11707 case 0x1: /* SQRDMLSH (vector) */
11708 gen_gvec_fn3(s, is_q, rd, rn, rm, gen_gvec_sqrdmlsh_qc, size);
11709 return;
11710
11711 case 0x2: /* SDOT / UDOT */
11712 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0,
11713 u ? gen_helper_gvec_udot_b : gen_helper_gvec_sdot_b);
11714 return;
11715
11716 case 0x3: /* USDOT */
11717 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_usdot_b);
11718 return;
11719
11720 case 0x04: /* SMMLA, UMMLA */
11721 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0,
11722 u ? gen_helper_gvec_ummla_b
11723 : gen_helper_gvec_smmla_b);
11724 return;
11725 case 0x05: /* USMMLA */
11726 gen_gvec_op4_ool(s, 1, rd, rn, rm, rd, 0, gen_helper_gvec_usmmla_b);
11727 return;
11728
11729 case 0x8: /* FCMLA, #0 */
11730 case 0x9: /* FCMLA, #90 */
11731 case 0xa: /* FCMLA, #180 */
11732 case 0xb: /* FCMLA, #270 */
11733 rot = extract32(opcode, 0, 2);
11734 switch (size) {
11735 case 1:
11736 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, true, rot,
11737 gen_helper_gvec_fcmlah);
11738 break;
11739 case 2:
11740 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
11741 gen_helper_gvec_fcmlas);
11742 break;
11743 case 3:
11744 gen_gvec_op4_fpst(s, is_q, rd, rn, rm, rd, false, rot,
11745 gen_helper_gvec_fcmlad);
11746 break;
11747 default:
11748 g_assert_not_reached();
11749 }
11750 return;
11751
11752 case 0xc: /* FCADD, #90 */
11753 case 0xe: /* FCADD, #270 */
11754 rot = extract32(opcode, 1, 1);
11755 switch (size) {
11756 case 1:
11757 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11758 gen_helper_gvec_fcaddh);
11759 break;
11760 case 2:
11761 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11762 gen_helper_gvec_fcadds);
11763 break;
11764 case 3:
11765 gen_gvec_op3_fpst(s, is_q, rd, rn, rm, size == 1, rot,
11766 gen_helper_gvec_fcaddd);
11767 break;
11768 default:
11769 g_assert_not_reached();
11770 }
11771 return;
11772
11773 case 0xd: /* BFMMLA */
11774 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfmmla);
11775 return;
11776 case 0xf:
11777 switch (size) {
11778 case 1: /* BFDOT */
11779 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, 0, gen_helper_gvec_bfdot);
11780 break;
11781 case 3: /* BFMLAL{B,T} */
11782 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, false, is_q,
11783 gen_helper_gvec_bfmlal);
11784 break;
11785 default:
11786 g_assert_not_reached();
11787 }
11788 return;
11789
11790 default:
11791 g_assert_not_reached();
11792 }
11793 }
11794
handle_2misc_widening(DisasContext * s,int opcode,bool is_q,int size,int rn,int rd)11795 static void handle_2misc_widening(DisasContext *s, int opcode, bool is_q,
11796 int size, int rn, int rd)
11797 {
11798 /* Handle 2-reg-misc ops which are widening (so each size element
11799 * in the source becomes a 2*size element in the destination.
11800 * The only instruction like this is FCVTL.
11801 */
11802 int pass;
11803
11804 if (size == 3) {
11805 /* 32 -> 64 bit fp conversion */
11806 TCGv_i64 tcg_res[2];
11807 int srcelt = is_q ? 2 : 0;
11808
11809 for (pass = 0; pass < 2; pass++) {
11810 TCGv_i32 tcg_op = tcg_temp_new_i32();
11811 tcg_res[pass] = tcg_temp_new_i64();
11812
11813 read_vec_element_i32(s, tcg_op, rn, srcelt + pass, MO_32);
11814 gen_helper_vfp_fcvtds(tcg_res[pass], tcg_op, tcg_env);
11815 }
11816 for (pass = 0; pass < 2; pass++) {
11817 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
11818 }
11819 } else {
11820 /* 16 -> 32 bit fp conversion */
11821 int srcelt = is_q ? 4 : 0;
11822 TCGv_i32 tcg_res[4];
11823 TCGv_ptr fpst = fpstatus_ptr(FPST_FPCR);
11824 TCGv_i32 ahp = get_ahp_flag();
11825
11826 for (pass = 0; pass < 4; pass++) {
11827 tcg_res[pass] = tcg_temp_new_i32();
11828
11829 read_vec_element_i32(s, tcg_res[pass], rn, srcelt + pass, MO_16);
11830 gen_helper_vfp_fcvt_f16_to_f32(tcg_res[pass], tcg_res[pass],
11831 fpst, ahp);
11832 }
11833 for (pass = 0; pass < 4; pass++) {
11834 write_vec_element_i32(s, tcg_res[pass], rd, pass, MO_32);
11835 }
11836 }
11837 }
11838
handle_rev(DisasContext * s,int opcode,bool u,bool is_q,int size,int rn,int rd)11839 static void handle_rev(DisasContext *s, int opcode, bool u,
11840 bool is_q, int size, int rn, int rd)
11841 {
11842 int op = (opcode << 1) | u;
11843 int opsz = op + size;
11844 int grp_size = 3 - opsz;
11845 int dsize = is_q ? 128 : 64;
11846 int i;
11847
11848 if (opsz >= 3) {
11849 unallocated_encoding(s);
11850 return;
11851 }
11852
11853 if (!fp_access_check(s)) {
11854 return;
11855 }
11856
11857 if (size == 0) {
11858 /* Special case bytes, use bswap op on each group of elements */
11859 int groups = dsize / (8 << grp_size);
11860
11861 for (i = 0; i < groups; i++) {
11862 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
11863
11864 read_vec_element(s, tcg_tmp, rn, i, grp_size);
11865 switch (grp_size) {
11866 case MO_16:
11867 tcg_gen_bswap16_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
11868 break;
11869 case MO_32:
11870 tcg_gen_bswap32_i64(tcg_tmp, tcg_tmp, TCG_BSWAP_IZ);
11871 break;
11872 case MO_64:
11873 tcg_gen_bswap64_i64(tcg_tmp, tcg_tmp);
11874 break;
11875 default:
11876 g_assert_not_reached();
11877 }
11878 write_vec_element(s, tcg_tmp, rd, i, grp_size);
11879 }
11880 clear_vec_high(s, is_q, rd);
11881 } else {
11882 int revmask = (1 << grp_size) - 1;
11883 int esize = 8 << size;
11884 int elements = dsize / esize;
11885 TCGv_i64 tcg_rn = tcg_temp_new_i64();
11886 TCGv_i64 tcg_rd[2];
11887
11888 for (i = 0; i < 2; i++) {
11889 tcg_rd[i] = tcg_temp_new_i64();
11890 tcg_gen_movi_i64(tcg_rd[i], 0);
11891 }
11892
11893 for (i = 0; i < elements; i++) {
11894 int e_rev = (i & 0xf) ^ revmask;
11895 int w = (e_rev * esize) / 64;
11896 int o = (e_rev * esize) % 64;
11897
11898 read_vec_element(s, tcg_rn, rn, i, size);
11899 tcg_gen_deposit_i64(tcg_rd[w], tcg_rd[w], tcg_rn, o, esize);
11900 }
11901
11902 for (i = 0; i < 2; i++) {
11903 write_vec_element(s, tcg_rd[i], rd, i, MO_64);
11904 }
11905 clear_vec_high(s, true, rd);
11906 }
11907 }
11908
handle_2misc_pairwise(DisasContext * s,int opcode,bool u,bool is_q,int size,int rn,int rd)11909 static void handle_2misc_pairwise(DisasContext *s, int opcode, bool u,
11910 bool is_q, int size, int rn, int rd)
11911 {
11912 /* Implement the pairwise operations from 2-misc:
11913 * SADDLP, UADDLP, SADALP, UADALP.
11914 * These all add pairs of elements in the input to produce a
11915 * double-width result element in the output (possibly accumulating).
11916 */
11917 bool accum = (opcode == 0x6);
11918 int maxpass = is_q ? 2 : 1;
11919 int pass;
11920 TCGv_i64 tcg_res[2];
11921
11922 if (size == 2) {
11923 /* 32 + 32 -> 64 op */
11924 MemOp memop = size + (u ? 0 : MO_SIGN);
11925
11926 for (pass = 0; pass < maxpass; pass++) {
11927 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
11928 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
11929
11930 tcg_res[pass] = tcg_temp_new_i64();
11931
11932 read_vec_element(s, tcg_op1, rn, pass * 2, memop);
11933 read_vec_element(s, tcg_op2, rn, pass * 2 + 1, memop);
11934 tcg_gen_add_i64(tcg_res[pass], tcg_op1, tcg_op2);
11935 if (accum) {
11936 read_vec_element(s, tcg_op1, rd, pass, MO_64);
11937 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
11938 }
11939 }
11940 } else {
11941 for (pass = 0; pass < maxpass; pass++) {
11942 TCGv_i64 tcg_op = tcg_temp_new_i64();
11943 NeonGenOne64OpFn *genfn;
11944 static NeonGenOne64OpFn * const fns[2][2] = {
11945 { gen_helper_neon_addlp_s8, gen_helper_neon_addlp_u8 },
11946 { gen_helper_neon_addlp_s16, gen_helper_neon_addlp_u16 },
11947 };
11948
11949 genfn = fns[size][u];
11950
11951 tcg_res[pass] = tcg_temp_new_i64();
11952
11953 read_vec_element(s, tcg_op, rn, pass, MO_64);
11954 genfn(tcg_res[pass], tcg_op);
11955
11956 if (accum) {
11957 read_vec_element(s, tcg_op, rd, pass, MO_64);
11958 if (size == 0) {
11959 gen_helper_neon_addl_u16(tcg_res[pass],
11960 tcg_res[pass], tcg_op);
11961 } else {
11962 gen_helper_neon_addl_u32(tcg_res[pass],
11963 tcg_res[pass], tcg_op);
11964 }
11965 }
11966 }
11967 }
11968 if (!is_q) {
11969 tcg_res[1] = tcg_constant_i64(0);
11970 }
11971 for (pass = 0; pass < 2; pass++) {
11972 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
11973 }
11974 }
11975
handle_shll(DisasContext * s,bool is_q,int size,int rn,int rd)11976 static void handle_shll(DisasContext *s, bool is_q, int size, int rn, int rd)
11977 {
11978 /* Implement SHLL and SHLL2 */
11979 int pass;
11980 int part = is_q ? 2 : 0;
11981 TCGv_i64 tcg_res[2];
11982
11983 for (pass = 0; pass < 2; pass++) {
11984 static NeonGenWidenFn * const widenfns[3] = {
11985 gen_helper_neon_widen_u8,
11986 gen_helper_neon_widen_u16,
11987 tcg_gen_extu_i32_i64,
11988 };
11989 NeonGenWidenFn *widenfn = widenfns[size];
11990 TCGv_i32 tcg_op = tcg_temp_new_i32();
11991
11992 read_vec_element_i32(s, tcg_op, rn, part + pass, MO_32);
11993 tcg_res[pass] = tcg_temp_new_i64();
11994 widenfn(tcg_res[pass], tcg_op);
11995 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << size);
11996 }
11997
11998 for (pass = 0; pass < 2; pass++) {
11999 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
12000 }
12001 }
12002
12003 /* AdvSIMD two reg misc
12004 * 31 30 29 28 24 23 22 21 17 16 12 11 10 9 5 4 0
12005 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
12006 * | 0 | Q | U | 0 1 1 1 0 | size | 1 0 0 0 0 | opcode | 1 0 | Rn | Rd |
12007 * +---+---+---+-----------+------+-----------+--------+-----+------+------+
12008 */
disas_simd_two_reg_misc(DisasContext * s,uint32_t insn)12009 static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
12010 {
12011 int size = extract32(insn, 22, 2);
12012 int opcode = extract32(insn, 12, 5);
12013 bool u = extract32(insn, 29, 1);
12014 bool is_q = extract32(insn, 30, 1);
12015 int rn = extract32(insn, 5, 5);
12016 int rd = extract32(insn, 0, 5);
12017 bool need_fpstatus = false;
12018 int rmode = -1;
12019 TCGv_i32 tcg_rmode;
12020 TCGv_ptr tcg_fpstatus;
12021
12022 switch (opcode) {
12023 case 0x0: /* REV64, REV32 */
12024 case 0x1: /* REV16 */
12025 handle_rev(s, opcode, u, is_q, size, rn, rd);
12026 return;
12027 case 0x5: /* CNT, NOT, RBIT */
12028 if (u && size == 0) {
12029 /* NOT */
12030 break;
12031 } else if (u && size == 1) {
12032 /* RBIT */
12033 break;
12034 } else if (!u && size == 0) {
12035 /* CNT */
12036 break;
12037 }
12038 unallocated_encoding(s);
12039 return;
12040 case 0x12: /* XTN, XTN2, SQXTUN, SQXTUN2 */
12041 case 0x14: /* SQXTN, SQXTN2, UQXTN, UQXTN2 */
12042 if (size == 3) {
12043 unallocated_encoding(s);
12044 return;
12045 }
12046 if (!fp_access_check(s)) {
12047 return;
12048 }
12049
12050 handle_2misc_narrow(s, false, opcode, u, is_q, size, rn, rd);
12051 return;
12052 case 0x4: /* CLS, CLZ */
12053 if (size == 3) {
12054 unallocated_encoding(s);
12055 return;
12056 }
12057 break;
12058 case 0x2: /* SADDLP, UADDLP */
12059 case 0x6: /* SADALP, UADALP */
12060 if (size == 3) {
12061 unallocated_encoding(s);
12062 return;
12063 }
12064 if (!fp_access_check(s)) {
12065 return;
12066 }
12067 handle_2misc_pairwise(s, opcode, u, is_q, size, rn, rd);
12068 return;
12069 case 0x13: /* SHLL, SHLL2 */
12070 if (u == 0 || size == 3) {
12071 unallocated_encoding(s);
12072 return;
12073 }
12074 if (!fp_access_check(s)) {
12075 return;
12076 }
12077 handle_shll(s, is_q, size, rn, rd);
12078 return;
12079 case 0xa: /* CMLT */
12080 if (u == 1) {
12081 unallocated_encoding(s);
12082 return;
12083 }
12084 /* fall through */
12085 case 0x8: /* CMGT, CMGE */
12086 case 0x9: /* CMEQ, CMLE */
12087 case 0xb: /* ABS, NEG */
12088 if (size == 3 && !is_q) {
12089 unallocated_encoding(s);
12090 return;
12091 }
12092 break;
12093 case 0x3: /* SUQADD, USQADD */
12094 if (size == 3 && !is_q) {
12095 unallocated_encoding(s);
12096 return;
12097 }
12098 if (!fp_access_check(s)) {
12099 return;
12100 }
12101 handle_2misc_satacc(s, false, u, is_q, size, rn, rd);
12102 return;
12103 case 0x7: /* SQABS, SQNEG */
12104 if (size == 3 && !is_q) {
12105 unallocated_encoding(s);
12106 return;
12107 }
12108 break;
12109 case 0xc ... 0xf:
12110 case 0x16 ... 0x1f:
12111 {
12112 /* Floating point: U, size[1] and opcode indicate operation;
12113 * size[0] indicates single or double precision.
12114 */
12115 int is_double = extract32(size, 0, 1);
12116 opcode |= (extract32(size, 1, 1) << 5) | (u << 6);
12117 size = is_double ? 3 : 2;
12118 switch (opcode) {
12119 case 0x2f: /* FABS */
12120 case 0x6f: /* FNEG */
12121 if (size == 3 && !is_q) {
12122 unallocated_encoding(s);
12123 return;
12124 }
12125 break;
12126 case 0x1d: /* SCVTF */
12127 case 0x5d: /* UCVTF */
12128 {
12129 bool is_signed = (opcode == 0x1d) ? true : false;
12130 int elements = is_double ? 2 : is_q ? 4 : 2;
12131 if (is_double && !is_q) {
12132 unallocated_encoding(s);
12133 return;
12134 }
12135 if (!fp_access_check(s)) {
12136 return;
12137 }
12138 handle_simd_intfp_conv(s, rd, rn, elements, is_signed, 0, size);
12139 return;
12140 }
12141 case 0x2c: /* FCMGT (zero) */
12142 case 0x2d: /* FCMEQ (zero) */
12143 case 0x2e: /* FCMLT (zero) */
12144 case 0x6c: /* FCMGE (zero) */
12145 case 0x6d: /* FCMLE (zero) */
12146 if (size == 3 && !is_q) {
12147 unallocated_encoding(s);
12148 return;
12149 }
12150 handle_2misc_fcmp_zero(s, opcode, false, u, is_q, size, rn, rd);
12151 return;
12152 case 0x7f: /* FSQRT */
12153 if (size == 3 && !is_q) {
12154 unallocated_encoding(s);
12155 return;
12156 }
12157 break;
12158 case 0x1a: /* FCVTNS */
12159 case 0x1b: /* FCVTMS */
12160 case 0x3a: /* FCVTPS */
12161 case 0x3b: /* FCVTZS */
12162 case 0x5a: /* FCVTNU */
12163 case 0x5b: /* FCVTMU */
12164 case 0x7a: /* FCVTPU */
12165 case 0x7b: /* FCVTZU */
12166 need_fpstatus = true;
12167 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
12168 if (size == 3 && !is_q) {
12169 unallocated_encoding(s);
12170 return;
12171 }
12172 break;
12173 case 0x5c: /* FCVTAU */
12174 case 0x1c: /* FCVTAS */
12175 need_fpstatus = true;
12176 rmode = FPROUNDING_TIEAWAY;
12177 if (size == 3 && !is_q) {
12178 unallocated_encoding(s);
12179 return;
12180 }
12181 break;
12182 case 0x3c: /* URECPE */
12183 if (size == 3) {
12184 unallocated_encoding(s);
12185 return;
12186 }
12187 /* fall through */
12188 case 0x3d: /* FRECPE */
12189 case 0x7d: /* FRSQRTE */
12190 if (size == 3 && !is_q) {
12191 unallocated_encoding(s);
12192 return;
12193 }
12194 if (!fp_access_check(s)) {
12195 return;
12196 }
12197 handle_2misc_reciprocal(s, opcode, false, u, is_q, size, rn, rd);
12198 return;
12199 case 0x56: /* FCVTXN, FCVTXN2 */
12200 if (size == 2) {
12201 unallocated_encoding(s);
12202 return;
12203 }
12204 /* fall through */
12205 case 0x16: /* FCVTN, FCVTN2 */
12206 /* handle_2misc_narrow does a 2*size -> size operation, but these
12207 * instructions encode the source size rather than dest size.
12208 */
12209 if (!fp_access_check(s)) {
12210 return;
12211 }
12212 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
12213 return;
12214 case 0x36: /* BFCVTN, BFCVTN2 */
12215 if (!dc_isar_feature(aa64_bf16, s) || size != 2) {
12216 unallocated_encoding(s);
12217 return;
12218 }
12219 if (!fp_access_check(s)) {
12220 return;
12221 }
12222 handle_2misc_narrow(s, false, opcode, 0, is_q, size - 1, rn, rd);
12223 return;
12224 case 0x17: /* FCVTL, FCVTL2 */
12225 if (!fp_access_check(s)) {
12226 return;
12227 }
12228 handle_2misc_widening(s, opcode, is_q, size, rn, rd);
12229 return;
12230 case 0x18: /* FRINTN */
12231 case 0x19: /* FRINTM */
12232 case 0x38: /* FRINTP */
12233 case 0x39: /* FRINTZ */
12234 rmode = extract32(opcode, 5, 1) | (extract32(opcode, 0, 1) << 1);
12235 /* fall through */
12236 case 0x59: /* FRINTX */
12237 case 0x79: /* FRINTI */
12238 need_fpstatus = true;
12239 if (size == 3 && !is_q) {
12240 unallocated_encoding(s);
12241 return;
12242 }
12243 break;
12244 case 0x58: /* FRINTA */
12245 rmode = FPROUNDING_TIEAWAY;
12246 need_fpstatus = true;
12247 if (size == 3 && !is_q) {
12248 unallocated_encoding(s);
12249 return;
12250 }
12251 break;
12252 case 0x7c: /* URSQRTE */
12253 if (size == 3) {
12254 unallocated_encoding(s);
12255 return;
12256 }
12257 break;
12258 case 0x1e: /* FRINT32Z */
12259 case 0x1f: /* FRINT64Z */
12260 rmode = FPROUNDING_ZERO;
12261 /* fall through */
12262 case 0x5e: /* FRINT32X */
12263 case 0x5f: /* FRINT64X */
12264 need_fpstatus = true;
12265 if ((size == 3 && !is_q) || !dc_isar_feature(aa64_frint, s)) {
12266 unallocated_encoding(s);
12267 return;
12268 }
12269 break;
12270 default:
12271 unallocated_encoding(s);
12272 return;
12273 }
12274 break;
12275 }
12276 default:
12277 unallocated_encoding(s);
12278 return;
12279 }
12280
12281 if (!fp_access_check(s)) {
12282 return;
12283 }
12284
12285 if (need_fpstatus || rmode >= 0) {
12286 tcg_fpstatus = fpstatus_ptr(FPST_FPCR);
12287 } else {
12288 tcg_fpstatus = NULL;
12289 }
12290 if (rmode >= 0) {
12291 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
12292 } else {
12293 tcg_rmode = NULL;
12294 }
12295
12296 switch (opcode) {
12297 case 0x5:
12298 if (u && size == 0) { /* NOT */
12299 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_not, 0);
12300 return;
12301 }
12302 break;
12303 case 0x8: /* CMGT, CMGE */
12304 if (u) {
12305 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cge0, size);
12306 } else {
12307 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cgt0, size);
12308 }
12309 return;
12310 case 0x9: /* CMEQ, CMLE */
12311 if (u) {
12312 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_cle0, size);
12313 } else {
12314 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_ceq0, size);
12315 }
12316 return;
12317 case 0xa: /* CMLT */
12318 gen_gvec_fn2(s, is_q, rd, rn, gen_gvec_clt0, size);
12319 return;
12320 case 0xb:
12321 if (u) { /* ABS, NEG */
12322 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_neg, size);
12323 } else {
12324 gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_abs, size);
12325 }
12326 return;
12327 }
12328
12329 if (size == 3) {
12330 /* All 64-bit element operations can be shared with scalar 2misc */
12331 int pass;
12332
12333 /* Coverity claims (size == 3 && !is_q) has been eliminated
12334 * from all paths leading to here.
12335 */
12336 tcg_debug_assert(is_q);
12337 for (pass = 0; pass < 2; pass++) {
12338 TCGv_i64 tcg_op = tcg_temp_new_i64();
12339 TCGv_i64 tcg_res = tcg_temp_new_i64();
12340
12341 read_vec_element(s, tcg_op, rn, pass, MO_64);
12342
12343 handle_2misc_64(s, opcode, u, tcg_res, tcg_op,
12344 tcg_rmode, tcg_fpstatus);
12345
12346 write_vec_element(s, tcg_res, rd, pass, MO_64);
12347 }
12348 } else {
12349 int pass;
12350
12351 for (pass = 0; pass < (is_q ? 4 : 2); pass++) {
12352 TCGv_i32 tcg_op = tcg_temp_new_i32();
12353 TCGv_i32 tcg_res = tcg_temp_new_i32();
12354
12355 read_vec_element_i32(s, tcg_op, rn, pass, MO_32);
12356
12357 if (size == 2) {
12358 /* Special cases for 32 bit elements */
12359 switch (opcode) {
12360 case 0x4: /* CLS */
12361 if (u) {
12362 tcg_gen_clzi_i32(tcg_res, tcg_op, 32);
12363 } else {
12364 tcg_gen_clrsb_i32(tcg_res, tcg_op);
12365 }
12366 break;
12367 case 0x7: /* SQABS, SQNEG */
12368 if (u) {
12369 gen_helper_neon_qneg_s32(tcg_res, tcg_env, tcg_op);
12370 } else {
12371 gen_helper_neon_qabs_s32(tcg_res, tcg_env, tcg_op);
12372 }
12373 break;
12374 case 0x2f: /* FABS */
12375 gen_helper_vfp_abss(tcg_res, tcg_op);
12376 break;
12377 case 0x6f: /* FNEG */
12378 gen_helper_vfp_negs(tcg_res, tcg_op);
12379 break;
12380 case 0x7f: /* FSQRT */
12381 gen_helper_vfp_sqrts(tcg_res, tcg_op, tcg_env);
12382 break;
12383 case 0x1a: /* FCVTNS */
12384 case 0x1b: /* FCVTMS */
12385 case 0x1c: /* FCVTAS */
12386 case 0x3a: /* FCVTPS */
12387 case 0x3b: /* FCVTZS */
12388 gen_helper_vfp_tosls(tcg_res, tcg_op,
12389 tcg_constant_i32(0), tcg_fpstatus);
12390 break;
12391 case 0x5a: /* FCVTNU */
12392 case 0x5b: /* FCVTMU */
12393 case 0x5c: /* FCVTAU */
12394 case 0x7a: /* FCVTPU */
12395 case 0x7b: /* FCVTZU */
12396 gen_helper_vfp_touls(tcg_res, tcg_op,
12397 tcg_constant_i32(0), tcg_fpstatus);
12398 break;
12399 case 0x18: /* FRINTN */
12400 case 0x19: /* FRINTM */
12401 case 0x38: /* FRINTP */
12402 case 0x39: /* FRINTZ */
12403 case 0x58: /* FRINTA */
12404 case 0x79: /* FRINTI */
12405 gen_helper_rints(tcg_res, tcg_op, tcg_fpstatus);
12406 break;
12407 case 0x59: /* FRINTX */
12408 gen_helper_rints_exact(tcg_res, tcg_op, tcg_fpstatus);
12409 break;
12410 case 0x7c: /* URSQRTE */
12411 gen_helper_rsqrte_u32(tcg_res, tcg_op);
12412 break;
12413 case 0x1e: /* FRINT32Z */
12414 case 0x5e: /* FRINT32X */
12415 gen_helper_frint32_s(tcg_res, tcg_op, tcg_fpstatus);
12416 break;
12417 case 0x1f: /* FRINT64Z */
12418 case 0x5f: /* FRINT64X */
12419 gen_helper_frint64_s(tcg_res, tcg_op, tcg_fpstatus);
12420 break;
12421 default:
12422 g_assert_not_reached();
12423 }
12424 } else {
12425 /* Use helpers for 8 and 16 bit elements */
12426 switch (opcode) {
12427 case 0x5: /* CNT, RBIT */
12428 /* For these two insns size is part of the opcode specifier
12429 * (handled earlier); they always operate on byte elements.
12430 */
12431 if (u) {
12432 gen_helper_neon_rbit_u8(tcg_res, tcg_op);
12433 } else {
12434 gen_helper_neon_cnt_u8(tcg_res, tcg_op);
12435 }
12436 break;
12437 case 0x7: /* SQABS, SQNEG */
12438 {
12439 NeonGenOneOpEnvFn *genfn;
12440 static NeonGenOneOpEnvFn * const fns[2][2] = {
12441 { gen_helper_neon_qabs_s8, gen_helper_neon_qneg_s8 },
12442 { gen_helper_neon_qabs_s16, gen_helper_neon_qneg_s16 },
12443 };
12444 genfn = fns[size][u];
12445 genfn(tcg_res, tcg_env, tcg_op);
12446 break;
12447 }
12448 case 0x4: /* CLS, CLZ */
12449 if (u) {
12450 if (size == 0) {
12451 gen_helper_neon_clz_u8(tcg_res, tcg_op);
12452 } else {
12453 gen_helper_neon_clz_u16(tcg_res, tcg_op);
12454 }
12455 } else {
12456 if (size == 0) {
12457 gen_helper_neon_cls_s8(tcg_res, tcg_op);
12458 } else {
12459 gen_helper_neon_cls_s16(tcg_res, tcg_op);
12460 }
12461 }
12462 break;
12463 default:
12464 g_assert_not_reached();
12465 }
12466 }
12467
12468 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
12469 }
12470 }
12471 clear_vec_high(s, is_q, rd);
12472
12473 if (tcg_rmode) {
12474 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
12475 }
12476 }
12477
12478 /* AdvSIMD [scalar] two register miscellaneous (FP16)
12479 *
12480 * 31 30 29 28 27 24 23 22 21 17 16 12 11 10 9 5 4 0
12481 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
12482 * | 0 | Q | U | S | 1 1 1 0 | a | 1 1 1 1 0 0 | opcode | 1 0 | Rn | Rd |
12483 * +---+---+---+---+---------+---+-------------+--------+-----+------+------+
12484 * mask: 1000 1111 0111 1110 0000 1100 0000 0000 0x8f7e 0c00
12485 * val: 0000 1110 0111 1000 0000 1000 0000 0000 0x0e78 0800
12486 *
12487 * This actually covers two groups where scalar access is governed by
12488 * bit 28. A bunch of the instructions (float to integral) only exist
12489 * in the vector form and are un-allocated for the scalar decode. Also
12490 * in the scalar decode Q is always 1.
12491 */
disas_simd_two_reg_misc_fp16(DisasContext * s,uint32_t insn)12492 static void disas_simd_two_reg_misc_fp16(DisasContext *s, uint32_t insn)
12493 {
12494 int fpop, opcode, a, u;
12495 int rn, rd;
12496 bool is_q;
12497 bool is_scalar;
12498 bool only_in_vector = false;
12499
12500 int pass;
12501 TCGv_i32 tcg_rmode = NULL;
12502 TCGv_ptr tcg_fpstatus = NULL;
12503 bool need_fpst = true;
12504 int rmode = -1;
12505
12506 if (!dc_isar_feature(aa64_fp16, s)) {
12507 unallocated_encoding(s);
12508 return;
12509 }
12510
12511 rd = extract32(insn, 0, 5);
12512 rn = extract32(insn, 5, 5);
12513
12514 a = extract32(insn, 23, 1);
12515 u = extract32(insn, 29, 1);
12516 is_scalar = extract32(insn, 28, 1);
12517 is_q = extract32(insn, 30, 1);
12518
12519 opcode = extract32(insn, 12, 5);
12520 fpop = deposit32(opcode, 5, 1, a);
12521 fpop = deposit32(fpop, 6, 1, u);
12522
12523 switch (fpop) {
12524 case 0x1d: /* SCVTF */
12525 case 0x5d: /* UCVTF */
12526 {
12527 int elements;
12528
12529 if (is_scalar) {
12530 elements = 1;
12531 } else {
12532 elements = (is_q ? 8 : 4);
12533 }
12534
12535 if (!fp_access_check(s)) {
12536 return;
12537 }
12538 handle_simd_intfp_conv(s, rd, rn, elements, !u, 0, MO_16);
12539 return;
12540 }
12541 break;
12542 case 0x2c: /* FCMGT (zero) */
12543 case 0x2d: /* FCMEQ (zero) */
12544 case 0x2e: /* FCMLT (zero) */
12545 case 0x6c: /* FCMGE (zero) */
12546 case 0x6d: /* FCMLE (zero) */
12547 handle_2misc_fcmp_zero(s, fpop, is_scalar, 0, is_q, MO_16, rn, rd);
12548 return;
12549 case 0x3d: /* FRECPE */
12550 case 0x3f: /* FRECPX */
12551 break;
12552 case 0x18: /* FRINTN */
12553 only_in_vector = true;
12554 rmode = FPROUNDING_TIEEVEN;
12555 break;
12556 case 0x19: /* FRINTM */
12557 only_in_vector = true;
12558 rmode = FPROUNDING_NEGINF;
12559 break;
12560 case 0x38: /* FRINTP */
12561 only_in_vector = true;
12562 rmode = FPROUNDING_POSINF;
12563 break;
12564 case 0x39: /* FRINTZ */
12565 only_in_vector = true;
12566 rmode = FPROUNDING_ZERO;
12567 break;
12568 case 0x58: /* FRINTA */
12569 only_in_vector = true;
12570 rmode = FPROUNDING_TIEAWAY;
12571 break;
12572 case 0x59: /* FRINTX */
12573 case 0x79: /* FRINTI */
12574 only_in_vector = true;
12575 /* current rounding mode */
12576 break;
12577 case 0x1a: /* FCVTNS */
12578 rmode = FPROUNDING_TIEEVEN;
12579 break;
12580 case 0x1b: /* FCVTMS */
12581 rmode = FPROUNDING_NEGINF;
12582 break;
12583 case 0x1c: /* FCVTAS */
12584 rmode = FPROUNDING_TIEAWAY;
12585 break;
12586 case 0x3a: /* FCVTPS */
12587 rmode = FPROUNDING_POSINF;
12588 break;
12589 case 0x3b: /* FCVTZS */
12590 rmode = FPROUNDING_ZERO;
12591 break;
12592 case 0x5a: /* FCVTNU */
12593 rmode = FPROUNDING_TIEEVEN;
12594 break;
12595 case 0x5b: /* FCVTMU */
12596 rmode = FPROUNDING_NEGINF;
12597 break;
12598 case 0x5c: /* FCVTAU */
12599 rmode = FPROUNDING_TIEAWAY;
12600 break;
12601 case 0x7a: /* FCVTPU */
12602 rmode = FPROUNDING_POSINF;
12603 break;
12604 case 0x7b: /* FCVTZU */
12605 rmode = FPROUNDING_ZERO;
12606 break;
12607 case 0x2f: /* FABS */
12608 case 0x6f: /* FNEG */
12609 need_fpst = false;
12610 break;
12611 case 0x7d: /* FRSQRTE */
12612 case 0x7f: /* FSQRT (vector) */
12613 break;
12614 default:
12615 unallocated_encoding(s);
12616 return;
12617 }
12618
12619
12620 /* Check additional constraints for the scalar encoding */
12621 if (is_scalar) {
12622 if (!is_q) {
12623 unallocated_encoding(s);
12624 return;
12625 }
12626 /* FRINTxx is only in the vector form */
12627 if (only_in_vector) {
12628 unallocated_encoding(s);
12629 return;
12630 }
12631 }
12632
12633 if (!fp_access_check(s)) {
12634 return;
12635 }
12636
12637 if (rmode >= 0 || need_fpst) {
12638 tcg_fpstatus = fpstatus_ptr(FPST_FPCR_F16);
12639 }
12640
12641 if (rmode >= 0) {
12642 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
12643 }
12644
12645 if (is_scalar) {
12646 TCGv_i32 tcg_op = read_fp_hreg(s, rn);
12647 TCGv_i32 tcg_res = tcg_temp_new_i32();
12648
12649 switch (fpop) {
12650 case 0x1a: /* FCVTNS */
12651 case 0x1b: /* FCVTMS */
12652 case 0x1c: /* FCVTAS */
12653 case 0x3a: /* FCVTPS */
12654 case 0x3b: /* FCVTZS */
12655 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
12656 break;
12657 case 0x3d: /* FRECPE */
12658 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
12659 break;
12660 case 0x3f: /* FRECPX */
12661 gen_helper_frecpx_f16(tcg_res, tcg_op, tcg_fpstatus);
12662 break;
12663 case 0x5a: /* FCVTNU */
12664 case 0x5b: /* FCVTMU */
12665 case 0x5c: /* FCVTAU */
12666 case 0x7a: /* FCVTPU */
12667 case 0x7b: /* FCVTZU */
12668 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
12669 break;
12670 case 0x6f: /* FNEG */
12671 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
12672 break;
12673 case 0x7d: /* FRSQRTE */
12674 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
12675 break;
12676 default:
12677 g_assert_not_reached();
12678 }
12679
12680 /* limit any sign extension going on */
12681 tcg_gen_andi_i32(tcg_res, tcg_res, 0xffff);
12682 write_fp_sreg(s, rd, tcg_res);
12683 } else {
12684 for (pass = 0; pass < (is_q ? 8 : 4); pass++) {
12685 TCGv_i32 tcg_op = tcg_temp_new_i32();
12686 TCGv_i32 tcg_res = tcg_temp_new_i32();
12687
12688 read_vec_element_i32(s, tcg_op, rn, pass, MO_16);
12689
12690 switch (fpop) {
12691 case 0x1a: /* FCVTNS */
12692 case 0x1b: /* FCVTMS */
12693 case 0x1c: /* FCVTAS */
12694 case 0x3a: /* FCVTPS */
12695 case 0x3b: /* FCVTZS */
12696 gen_helper_advsimd_f16tosinth(tcg_res, tcg_op, tcg_fpstatus);
12697 break;
12698 case 0x3d: /* FRECPE */
12699 gen_helper_recpe_f16(tcg_res, tcg_op, tcg_fpstatus);
12700 break;
12701 case 0x5a: /* FCVTNU */
12702 case 0x5b: /* FCVTMU */
12703 case 0x5c: /* FCVTAU */
12704 case 0x7a: /* FCVTPU */
12705 case 0x7b: /* FCVTZU */
12706 gen_helper_advsimd_f16touinth(tcg_res, tcg_op, tcg_fpstatus);
12707 break;
12708 case 0x18: /* FRINTN */
12709 case 0x19: /* FRINTM */
12710 case 0x38: /* FRINTP */
12711 case 0x39: /* FRINTZ */
12712 case 0x58: /* FRINTA */
12713 case 0x79: /* FRINTI */
12714 gen_helper_advsimd_rinth(tcg_res, tcg_op, tcg_fpstatus);
12715 break;
12716 case 0x59: /* FRINTX */
12717 gen_helper_advsimd_rinth_exact(tcg_res, tcg_op, tcg_fpstatus);
12718 break;
12719 case 0x2f: /* FABS */
12720 tcg_gen_andi_i32(tcg_res, tcg_op, 0x7fff);
12721 break;
12722 case 0x6f: /* FNEG */
12723 tcg_gen_xori_i32(tcg_res, tcg_op, 0x8000);
12724 break;
12725 case 0x7d: /* FRSQRTE */
12726 gen_helper_rsqrte_f16(tcg_res, tcg_op, tcg_fpstatus);
12727 break;
12728 case 0x7f: /* FSQRT */
12729 gen_helper_sqrt_f16(tcg_res, tcg_op, tcg_fpstatus);
12730 break;
12731 default:
12732 g_assert_not_reached();
12733 }
12734
12735 write_vec_element_i32(s, tcg_res, rd, pass, MO_16);
12736 }
12737
12738 clear_vec_high(s, is_q, rd);
12739 }
12740
12741 if (tcg_rmode) {
12742 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
12743 }
12744 }
12745
12746 /* AdvSIMD scalar x indexed element
12747 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12748 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12749 * | 0 1 | U | 1 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12750 * +-----+---+-----------+------+---+---+------+-----+---+---+------+------+
12751 * AdvSIMD vector x indexed element
12752 * 31 30 29 28 24 23 22 21 20 19 16 15 12 11 10 9 5 4 0
12753 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12754 * | 0 | Q | U | 0 1 1 1 1 | size | L | M | Rm | opc | H | 0 | Rn | Rd |
12755 * +---+---+---+-----------+------+---+---+------+-----+---+---+------+------+
12756 */
disas_simd_indexed(DisasContext * s,uint32_t insn)12757 static void disas_simd_indexed(DisasContext *s, uint32_t insn)
12758 {
12759 /* This encoding has two kinds of instruction:
12760 * normal, where we perform elt x idxelt => elt for each
12761 * element in the vector
12762 * long, where we perform elt x idxelt and generate a result of
12763 * double the width of the input element
12764 * The long ops have a 'part' specifier (ie come in INSN, INSN2 pairs).
12765 */
12766 bool is_scalar = extract32(insn, 28, 1);
12767 bool is_q = extract32(insn, 30, 1);
12768 bool u = extract32(insn, 29, 1);
12769 int size = extract32(insn, 22, 2);
12770 int l = extract32(insn, 21, 1);
12771 int m = extract32(insn, 20, 1);
12772 /* Note that the Rm field here is only 4 bits, not 5 as it usually is */
12773 int rm = extract32(insn, 16, 4);
12774 int opcode = extract32(insn, 12, 4);
12775 int h = extract32(insn, 11, 1);
12776 int rn = extract32(insn, 5, 5);
12777 int rd = extract32(insn, 0, 5);
12778 bool is_long = false;
12779 int is_fp = 0;
12780 bool is_fp16 = false;
12781 int index;
12782 TCGv_ptr fpst;
12783
12784 switch (16 * u + opcode) {
12785 case 0x08: /* MUL */
12786 case 0x10: /* MLA */
12787 case 0x14: /* MLS */
12788 if (is_scalar) {
12789 unallocated_encoding(s);
12790 return;
12791 }
12792 break;
12793 case 0x02: /* SMLAL, SMLAL2 */
12794 case 0x12: /* UMLAL, UMLAL2 */
12795 case 0x06: /* SMLSL, SMLSL2 */
12796 case 0x16: /* UMLSL, UMLSL2 */
12797 case 0x0a: /* SMULL, SMULL2 */
12798 case 0x1a: /* UMULL, UMULL2 */
12799 if (is_scalar) {
12800 unallocated_encoding(s);
12801 return;
12802 }
12803 is_long = true;
12804 break;
12805 case 0x03: /* SQDMLAL, SQDMLAL2 */
12806 case 0x07: /* SQDMLSL, SQDMLSL2 */
12807 case 0x0b: /* SQDMULL, SQDMULL2 */
12808 is_long = true;
12809 break;
12810 case 0x0c: /* SQDMULH */
12811 case 0x0d: /* SQRDMULH */
12812 break;
12813 case 0x01: /* FMLA */
12814 case 0x05: /* FMLS */
12815 case 0x09: /* FMUL */
12816 case 0x19: /* FMULX */
12817 is_fp = 1;
12818 break;
12819 case 0x1d: /* SQRDMLAH */
12820 case 0x1f: /* SQRDMLSH */
12821 if (!dc_isar_feature(aa64_rdm, s)) {
12822 unallocated_encoding(s);
12823 return;
12824 }
12825 break;
12826 case 0x0e: /* SDOT */
12827 case 0x1e: /* UDOT */
12828 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_dp, s)) {
12829 unallocated_encoding(s);
12830 return;
12831 }
12832 break;
12833 case 0x0f:
12834 switch (size) {
12835 case 0: /* SUDOT */
12836 case 2: /* USDOT */
12837 if (is_scalar || !dc_isar_feature(aa64_i8mm, s)) {
12838 unallocated_encoding(s);
12839 return;
12840 }
12841 size = MO_32;
12842 break;
12843 case 1: /* BFDOT */
12844 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
12845 unallocated_encoding(s);
12846 return;
12847 }
12848 size = MO_32;
12849 break;
12850 case 3: /* BFMLAL{B,T} */
12851 if (is_scalar || !dc_isar_feature(aa64_bf16, s)) {
12852 unallocated_encoding(s);
12853 return;
12854 }
12855 /* can't set is_fp without other incorrect size checks */
12856 size = MO_16;
12857 break;
12858 default:
12859 unallocated_encoding(s);
12860 return;
12861 }
12862 break;
12863 case 0x11: /* FCMLA #0 */
12864 case 0x13: /* FCMLA #90 */
12865 case 0x15: /* FCMLA #180 */
12866 case 0x17: /* FCMLA #270 */
12867 if (is_scalar || !dc_isar_feature(aa64_fcma, s)) {
12868 unallocated_encoding(s);
12869 return;
12870 }
12871 is_fp = 2;
12872 break;
12873 case 0x00: /* FMLAL */
12874 case 0x04: /* FMLSL */
12875 case 0x18: /* FMLAL2 */
12876 case 0x1c: /* FMLSL2 */
12877 if (is_scalar || size != MO_32 || !dc_isar_feature(aa64_fhm, s)) {
12878 unallocated_encoding(s);
12879 return;
12880 }
12881 size = MO_16;
12882 /* is_fp, but we pass tcg_env not fp_status. */
12883 break;
12884 default:
12885 unallocated_encoding(s);
12886 return;
12887 }
12888
12889 switch (is_fp) {
12890 case 1: /* normal fp */
12891 /* convert insn encoded size to MemOp size */
12892 switch (size) {
12893 case 0: /* half-precision */
12894 size = MO_16;
12895 is_fp16 = true;
12896 break;
12897 case MO_32: /* single precision */
12898 case MO_64: /* double precision */
12899 break;
12900 default:
12901 unallocated_encoding(s);
12902 return;
12903 }
12904 break;
12905
12906 case 2: /* complex fp */
12907 /* Each indexable element is a complex pair. */
12908 size += 1;
12909 switch (size) {
12910 case MO_32:
12911 if (h && !is_q) {
12912 unallocated_encoding(s);
12913 return;
12914 }
12915 is_fp16 = true;
12916 break;
12917 case MO_64:
12918 break;
12919 default:
12920 unallocated_encoding(s);
12921 return;
12922 }
12923 break;
12924
12925 default: /* integer */
12926 switch (size) {
12927 case MO_8:
12928 case MO_64:
12929 unallocated_encoding(s);
12930 return;
12931 }
12932 break;
12933 }
12934 if (is_fp16 && !dc_isar_feature(aa64_fp16, s)) {
12935 unallocated_encoding(s);
12936 return;
12937 }
12938
12939 /* Given MemOp size, adjust register and indexing. */
12940 switch (size) {
12941 case MO_16:
12942 index = h << 2 | l << 1 | m;
12943 break;
12944 case MO_32:
12945 index = h << 1 | l;
12946 rm |= m << 4;
12947 break;
12948 case MO_64:
12949 if (l || !is_q) {
12950 unallocated_encoding(s);
12951 return;
12952 }
12953 index = h;
12954 rm |= m << 4;
12955 break;
12956 default:
12957 g_assert_not_reached();
12958 }
12959
12960 if (!fp_access_check(s)) {
12961 return;
12962 }
12963
12964 if (is_fp) {
12965 fpst = fpstatus_ptr(is_fp16 ? FPST_FPCR_F16 : FPST_FPCR);
12966 } else {
12967 fpst = NULL;
12968 }
12969
12970 switch (16 * u + opcode) {
12971 case 0x0e: /* SDOT */
12972 case 0x1e: /* UDOT */
12973 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12974 u ? gen_helper_gvec_udot_idx_b
12975 : gen_helper_gvec_sdot_idx_b);
12976 return;
12977 case 0x0f:
12978 switch (extract32(insn, 22, 2)) {
12979 case 0: /* SUDOT */
12980 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12981 gen_helper_gvec_sudot_idx_b);
12982 return;
12983 case 1: /* BFDOT */
12984 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12985 gen_helper_gvec_bfdot_idx);
12986 return;
12987 case 2: /* USDOT */
12988 gen_gvec_op4_ool(s, is_q, rd, rn, rm, rd, index,
12989 gen_helper_gvec_usdot_idx_b);
12990 return;
12991 case 3: /* BFMLAL{B,T} */
12992 gen_gvec_op4_fpst(s, 1, rd, rn, rm, rd, 0, (index << 1) | is_q,
12993 gen_helper_gvec_bfmlal_idx);
12994 return;
12995 }
12996 g_assert_not_reached();
12997 case 0x11: /* FCMLA #0 */
12998 case 0x13: /* FCMLA #90 */
12999 case 0x15: /* FCMLA #180 */
13000 case 0x17: /* FCMLA #270 */
13001 {
13002 int rot = extract32(insn, 13, 2);
13003 int data = (index << 2) | rot;
13004 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
13005 vec_full_reg_offset(s, rn),
13006 vec_full_reg_offset(s, rm),
13007 vec_full_reg_offset(s, rd), fpst,
13008 is_q ? 16 : 8, vec_full_reg_size(s), data,
13009 size == MO_64
13010 ? gen_helper_gvec_fcmlas_idx
13011 : gen_helper_gvec_fcmlah_idx);
13012 }
13013 return;
13014
13015 case 0x00: /* FMLAL */
13016 case 0x04: /* FMLSL */
13017 case 0x18: /* FMLAL2 */
13018 case 0x1c: /* FMLSL2 */
13019 {
13020 int is_s = extract32(opcode, 2, 1);
13021 int is_2 = u;
13022 int data = (index << 2) | (is_2 << 1) | is_s;
13023 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
13024 vec_full_reg_offset(s, rn),
13025 vec_full_reg_offset(s, rm), tcg_env,
13026 is_q ? 16 : 8, vec_full_reg_size(s),
13027 data, gen_helper_gvec_fmlal_idx_a64);
13028 }
13029 return;
13030
13031 case 0x08: /* MUL */
13032 if (!is_long && !is_scalar) {
13033 static gen_helper_gvec_3 * const fns[3] = {
13034 gen_helper_gvec_mul_idx_h,
13035 gen_helper_gvec_mul_idx_s,
13036 gen_helper_gvec_mul_idx_d,
13037 };
13038 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
13039 vec_full_reg_offset(s, rn),
13040 vec_full_reg_offset(s, rm),
13041 is_q ? 16 : 8, vec_full_reg_size(s),
13042 index, fns[size - 1]);
13043 return;
13044 }
13045 break;
13046
13047 case 0x10: /* MLA */
13048 if (!is_long && !is_scalar) {
13049 static gen_helper_gvec_4 * const fns[3] = {
13050 gen_helper_gvec_mla_idx_h,
13051 gen_helper_gvec_mla_idx_s,
13052 gen_helper_gvec_mla_idx_d,
13053 };
13054 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
13055 vec_full_reg_offset(s, rn),
13056 vec_full_reg_offset(s, rm),
13057 vec_full_reg_offset(s, rd),
13058 is_q ? 16 : 8, vec_full_reg_size(s),
13059 index, fns[size - 1]);
13060 return;
13061 }
13062 break;
13063
13064 case 0x14: /* MLS */
13065 if (!is_long && !is_scalar) {
13066 static gen_helper_gvec_4 * const fns[3] = {
13067 gen_helper_gvec_mls_idx_h,
13068 gen_helper_gvec_mls_idx_s,
13069 gen_helper_gvec_mls_idx_d,
13070 };
13071 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
13072 vec_full_reg_offset(s, rn),
13073 vec_full_reg_offset(s, rm),
13074 vec_full_reg_offset(s, rd),
13075 is_q ? 16 : 8, vec_full_reg_size(s),
13076 index, fns[size - 1]);
13077 return;
13078 }
13079 break;
13080 }
13081
13082 if (size == 3) {
13083 TCGv_i64 tcg_idx = tcg_temp_new_i64();
13084 int pass;
13085
13086 assert(is_fp && is_q && !is_long);
13087
13088 read_vec_element(s, tcg_idx, rm, index, MO_64);
13089
13090 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
13091 TCGv_i64 tcg_op = tcg_temp_new_i64();
13092 TCGv_i64 tcg_res = tcg_temp_new_i64();
13093
13094 read_vec_element(s, tcg_op, rn, pass, MO_64);
13095
13096 switch (16 * u + opcode) {
13097 case 0x05: /* FMLS */
13098 /* As usual for ARM, separate negation for fused multiply-add */
13099 gen_helper_vfp_negd(tcg_op, tcg_op);
13100 /* fall through */
13101 case 0x01: /* FMLA */
13102 read_vec_element(s, tcg_res, rd, pass, MO_64);
13103 gen_helper_vfp_muladdd(tcg_res, tcg_op, tcg_idx, tcg_res, fpst);
13104 break;
13105 case 0x09: /* FMUL */
13106 gen_helper_vfp_muld(tcg_res, tcg_op, tcg_idx, fpst);
13107 break;
13108 case 0x19: /* FMULX */
13109 gen_helper_vfp_mulxd(tcg_res, tcg_op, tcg_idx, fpst);
13110 break;
13111 default:
13112 g_assert_not_reached();
13113 }
13114
13115 write_vec_element(s, tcg_res, rd, pass, MO_64);
13116 }
13117
13118 clear_vec_high(s, !is_scalar, rd);
13119 } else if (!is_long) {
13120 /* 32 bit floating point, or 16 or 32 bit integer.
13121 * For the 16 bit scalar case we use the usual Neon helpers and
13122 * rely on the fact that 0 op 0 == 0 with no side effects.
13123 */
13124 TCGv_i32 tcg_idx = tcg_temp_new_i32();
13125 int pass, maxpasses;
13126
13127 if (is_scalar) {
13128 maxpasses = 1;
13129 } else {
13130 maxpasses = is_q ? 4 : 2;
13131 }
13132
13133 read_vec_element_i32(s, tcg_idx, rm, index, size);
13134
13135 if (size == 1 && !is_scalar) {
13136 /* The simplest way to handle the 16x16 indexed ops is to duplicate
13137 * the index into both halves of the 32 bit tcg_idx and then use
13138 * the usual Neon helpers.
13139 */
13140 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16);
13141 }
13142
13143 for (pass = 0; pass < maxpasses; pass++) {
13144 TCGv_i32 tcg_op = tcg_temp_new_i32();
13145 TCGv_i32 tcg_res = tcg_temp_new_i32();
13146
13147 read_vec_element_i32(s, tcg_op, rn, pass, is_scalar ? size : MO_32);
13148
13149 switch (16 * u + opcode) {
13150 case 0x08: /* MUL */
13151 case 0x10: /* MLA */
13152 case 0x14: /* MLS */
13153 {
13154 static NeonGenTwoOpFn * const fns[2][2] = {
13155 { gen_helper_neon_add_u16, gen_helper_neon_sub_u16 },
13156 { tcg_gen_add_i32, tcg_gen_sub_i32 },
13157 };
13158 NeonGenTwoOpFn *genfn;
13159 bool is_sub = opcode == 0x4;
13160
13161 if (size == 1) {
13162 gen_helper_neon_mul_u16(tcg_res, tcg_op, tcg_idx);
13163 } else {
13164 tcg_gen_mul_i32(tcg_res, tcg_op, tcg_idx);
13165 }
13166 if (opcode == 0x8) {
13167 break;
13168 }
13169 read_vec_element_i32(s, tcg_op, rd, pass, MO_32);
13170 genfn = fns[size - 1][is_sub];
13171 genfn(tcg_res, tcg_op, tcg_res);
13172 break;
13173 }
13174 case 0x05: /* FMLS */
13175 case 0x01: /* FMLA */
13176 read_vec_element_i32(s, tcg_res, rd, pass,
13177 is_scalar ? size : MO_32);
13178 switch (size) {
13179 case 1:
13180 if (opcode == 0x5) {
13181 /* As usual for ARM, separate negation for fused
13182 * multiply-add */
13183 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80008000);
13184 }
13185 if (is_scalar) {
13186 gen_helper_advsimd_muladdh(tcg_res, tcg_op, tcg_idx,
13187 tcg_res, fpst);
13188 } else {
13189 gen_helper_advsimd_muladd2h(tcg_res, tcg_op, tcg_idx,
13190 tcg_res, fpst);
13191 }
13192 break;
13193 case 2:
13194 if (opcode == 0x5) {
13195 /* As usual for ARM, separate negation for
13196 * fused multiply-add */
13197 tcg_gen_xori_i32(tcg_op, tcg_op, 0x80000000);
13198 }
13199 gen_helper_vfp_muladds(tcg_res, tcg_op, tcg_idx,
13200 tcg_res, fpst);
13201 break;
13202 default:
13203 g_assert_not_reached();
13204 }
13205 break;
13206 case 0x09: /* FMUL */
13207 switch (size) {
13208 case 1:
13209 if (is_scalar) {
13210 gen_helper_advsimd_mulh(tcg_res, tcg_op,
13211 tcg_idx, fpst);
13212 } else {
13213 gen_helper_advsimd_mul2h(tcg_res, tcg_op,
13214 tcg_idx, fpst);
13215 }
13216 break;
13217 case 2:
13218 gen_helper_vfp_muls(tcg_res, tcg_op, tcg_idx, fpst);
13219 break;
13220 default:
13221 g_assert_not_reached();
13222 }
13223 break;
13224 case 0x19: /* FMULX */
13225 switch (size) {
13226 case 1:
13227 if (is_scalar) {
13228 gen_helper_advsimd_mulxh(tcg_res, tcg_op,
13229 tcg_idx, fpst);
13230 } else {
13231 gen_helper_advsimd_mulx2h(tcg_res, tcg_op,
13232 tcg_idx, fpst);
13233 }
13234 break;
13235 case 2:
13236 gen_helper_vfp_mulxs(tcg_res, tcg_op, tcg_idx, fpst);
13237 break;
13238 default:
13239 g_assert_not_reached();
13240 }
13241 break;
13242 case 0x0c: /* SQDMULH */
13243 if (size == 1) {
13244 gen_helper_neon_qdmulh_s16(tcg_res, tcg_env,
13245 tcg_op, tcg_idx);
13246 } else {
13247 gen_helper_neon_qdmulh_s32(tcg_res, tcg_env,
13248 tcg_op, tcg_idx);
13249 }
13250 break;
13251 case 0x0d: /* SQRDMULH */
13252 if (size == 1) {
13253 gen_helper_neon_qrdmulh_s16(tcg_res, tcg_env,
13254 tcg_op, tcg_idx);
13255 } else {
13256 gen_helper_neon_qrdmulh_s32(tcg_res, tcg_env,
13257 tcg_op, tcg_idx);
13258 }
13259 break;
13260 case 0x1d: /* SQRDMLAH */
13261 read_vec_element_i32(s, tcg_res, rd, pass,
13262 is_scalar ? size : MO_32);
13263 if (size == 1) {
13264 gen_helper_neon_qrdmlah_s16(tcg_res, tcg_env,
13265 tcg_op, tcg_idx, tcg_res);
13266 } else {
13267 gen_helper_neon_qrdmlah_s32(tcg_res, tcg_env,
13268 tcg_op, tcg_idx, tcg_res);
13269 }
13270 break;
13271 case 0x1f: /* SQRDMLSH */
13272 read_vec_element_i32(s, tcg_res, rd, pass,
13273 is_scalar ? size : MO_32);
13274 if (size == 1) {
13275 gen_helper_neon_qrdmlsh_s16(tcg_res, tcg_env,
13276 tcg_op, tcg_idx, tcg_res);
13277 } else {
13278 gen_helper_neon_qrdmlsh_s32(tcg_res, tcg_env,
13279 tcg_op, tcg_idx, tcg_res);
13280 }
13281 break;
13282 default:
13283 g_assert_not_reached();
13284 }
13285
13286 if (is_scalar) {
13287 write_fp_sreg(s, rd, tcg_res);
13288 } else {
13289 write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
13290 }
13291 }
13292
13293 clear_vec_high(s, is_q, rd);
13294 } else {
13295 /* long ops: 16x16->32 or 32x32->64 */
13296 TCGv_i64 tcg_res[2];
13297 int pass;
13298 bool satop = extract32(opcode, 0, 1);
13299 MemOp memop = MO_32;
13300
13301 if (satop || !u) {
13302 memop |= MO_SIGN;
13303 }
13304
13305 if (size == 2) {
13306 TCGv_i64 tcg_idx = tcg_temp_new_i64();
13307
13308 read_vec_element(s, tcg_idx, rm, index, memop);
13309
13310 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
13311 TCGv_i64 tcg_op = tcg_temp_new_i64();
13312 TCGv_i64 tcg_passres;
13313 int passelt;
13314
13315 if (is_scalar) {
13316 passelt = 0;
13317 } else {
13318 passelt = pass + (is_q * 2);
13319 }
13320
13321 read_vec_element(s, tcg_op, rn, passelt, memop);
13322
13323 tcg_res[pass] = tcg_temp_new_i64();
13324
13325 if (opcode == 0xa || opcode == 0xb) {
13326 /* Non-accumulating ops */
13327 tcg_passres = tcg_res[pass];
13328 } else {
13329 tcg_passres = tcg_temp_new_i64();
13330 }
13331
13332 tcg_gen_mul_i64(tcg_passres, tcg_op, tcg_idx);
13333
13334 if (satop) {
13335 /* saturating, doubling */
13336 gen_helper_neon_addl_saturate_s64(tcg_passres, tcg_env,
13337 tcg_passres, tcg_passres);
13338 }
13339
13340 if (opcode == 0xa || opcode == 0xb) {
13341 continue;
13342 }
13343
13344 /* Accumulating op: handle accumulate step */
13345 read_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13346
13347 switch (opcode) {
13348 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
13349 tcg_gen_add_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
13350 break;
13351 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
13352 tcg_gen_sub_i64(tcg_res[pass], tcg_res[pass], tcg_passres);
13353 break;
13354 case 0x7: /* SQDMLSL, SQDMLSL2 */
13355 tcg_gen_neg_i64(tcg_passres, tcg_passres);
13356 /* fall through */
13357 case 0x3: /* SQDMLAL, SQDMLAL2 */
13358 gen_helper_neon_addl_saturate_s64(tcg_res[pass], tcg_env,
13359 tcg_res[pass],
13360 tcg_passres);
13361 break;
13362 default:
13363 g_assert_not_reached();
13364 }
13365 }
13366
13367 clear_vec_high(s, !is_scalar, rd);
13368 } else {
13369 TCGv_i32 tcg_idx = tcg_temp_new_i32();
13370
13371 assert(size == 1);
13372 read_vec_element_i32(s, tcg_idx, rm, index, size);
13373
13374 if (!is_scalar) {
13375 /* The simplest way to handle the 16x16 indexed ops is to
13376 * duplicate the index into both halves of the 32 bit tcg_idx
13377 * and then use the usual Neon helpers.
13378 */
13379 tcg_gen_deposit_i32(tcg_idx, tcg_idx, tcg_idx, 16, 16);
13380 }
13381
13382 for (pass = 0; pass < (is_scalar ? 1 : 2); pass++) {
13383 TCGv_i32 tcg_op = tcg_temp_new_i32();
13384 TCGv_i64 tcg_passres;
13385
13386 if (is_scalar) {
13387 read_vec_element_i32(s, tcg_op, rn, pass, size);
13388 } else {
13389 read_vec_element_i32(s, tcg_op, rn,
13390 pass + (is_q * 2), MO_32);
13391 }
13392
13393 tcg_res[pass] = tcg_temp_new_i64();
13394
13395 if (opcode == 0xa || opcode == 0xb) {
13396 /* Non-accumulating ops */
13397 tcg_passres = tcg_res[pass];
13398 } else {
13399 tcg_passres = tcg_temp_new_i64();
13400 }
13401
13402 if (memop & MO_SIGN) {
13403 gen_helper_neon_mull_s16(tcg_passres, tcg_op, tcg_idx);
13404 } else {
13405 gen_helper_neon_mull_u16(tcg_passres, tcg_op, tcg_idx);
13406 }
13407 if (satop) {
13408 gen_helper_neon_addl_saturate_s32(tcg_passres, tcg_env,
13409 tcg_passres, tcg_passres);
13410 }
13411
13412 if (opcode == 0xa || opcode == 0xb) {
13413 continue;
13414 }
13415
13416 /* Accumulating op: handle accumulate step */
13417 read_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13418
13419 switch (opcode) {
13420 case 0x2: /* SMLAL, SMLAL2, UMLAL, UMLAL2 */
13421 gen_helper_neon_addl_u32(tcg_res[pass], tcg_res[pass],
13422 tcg_passres);
13423 break;
13424 case 0x6: /* SMLSL, SMLSL2, UMLSL, UMLSL2 */
13425 gen_helper_neon_subl_u32(tcg_res[pass], tcg_res[pass],
13426 tcg_passres);
13427 break;
13428 case 0x7: /* SQDMLSL, SQDMLSL2 */
13429 gen_helper_neon_negl_u32(tcg_passres, tcg_passres);
13430 /* fall through */
13431 case 0x3: /* SQDMLAL, SQDMLAL2 */
13432 gen_helper_neon_addl_saturate_s32(tcg_res[pass], tcg_env,
13433 tcg_res[pass],
13434 tcg_passres);
13435 break;
13436 default:
13437 g_assert_not_reached();
13438 }
13439 }
13440
13441 if (is_scalar) {
13442 tcg_gen_ext32u_i64(tcg_res[0], tcg_res[0]);
13443 }
13444 }
13445
13446 if (is_scalar) {
13447 tcg_res[1] = tcg_constant_i64(0);
13448 }
13449
13450 for (pass = 0; pass < 2; pass++) {
13451 write_vec_element(s, tcg_res[pass], rd, pass, MO_64);
13452 }
13453 }
13454 }
13455
13456 /* Crypto AES
13457 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0
13458 * +-----------------+------+-----------+--------+-----+------+------+
13459 * | 0 1 0 0 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd |
13460 * +-----------------+------+-----------+--------+-----+------+------+
13461 */
disas_crypto_aes(DisasContext * s,uint32_t insn)13462 static void disas_crypto_aes(DisasContext *s, uint32_t insn)
13463 {
13464 int size = extract32(insn, 22, 2);
13465 int opcode = extract32(insn, 12, 5);
13466 int rn = extract32(insn, 5, 5);
13467 int rd = extract32(insn, 0, 5);
13468 gen_helper_gvec_2 *genfn2 = NULL;
13469 gen_helper_gvec_3 *genfn3 = NULL;
13470
13471 if (!dc_isar_feature(aa64_aes, s) || size != 0) {
13472 unallocated_encoding(s);
13473 return;
13474 }
13475
13476 switch (opcode) {
13477 case 0x4: /* AESE */
13478 genfn3 = gen_helper_crypto_aese;
13479 break;
13480 case 0x6: /* AESMC */
13481 genfn2 = gen_helper_crypto_aesmc;
13482 break;
13483 case 0x5: /* AESD */
13484 genfn3 = gen_helper_crypto_aesd;
13485 break;
13486 case 0x7: /* AESIMC */
13487 genfn2 = gen_helper_crypto_aesimc;
13488 break;
13489 default:
13490 unallocated_encoding(s);
13491 return;
13492 }
13493
13494 if (!fp_access_check(s)) {
13495 return;
13496 }
13497 if (genfn2) {
13498 gen_gvec_op2_ool(s, true, rd, rn, 0, genfn2);
13499 } else {
13500 gen_gvec_op3_ool(s, true, rd, rd, rn, 0, genfn3);
13501 }
13502 }
13503
13504 /* Crypto three-reg SHA
13505 * 31 24 23 22 21 20 16 15 14 12 11 10 9 5 4 0
13506 * +-----------------+------+---+------+---+--------+-----+------+------+
13507 * | 0 1 0 1 1 1 1 0 | size | 0 | Rm | 0 | opcode | 0 0 | Rn | Rd |
13508 * +-----------------+------+---+------+---+--------+-----+------+------+
13509 */
disas_crypto_three_reg_sha(DisasContext * s,uint32_t insn)13510 static void disas_crypto_three_reg_sha(DisasContext *s, uint32_t insn)
13511 {
13512 int size = extract32(insn, 22, 2);
13513 int opcode = extract32(insn, 12, 3);
13514 int rm = extract32(insn, 16, 5);
13515 int rn = extract32(insn, 5, 5);
13516 int rd = extract32(insn, 0, 5);
13517 gen_helper_gvec_3 *genfn;
13518 bool feature;
13519
13520 if (size != 0) {
13521 unallocated_encoding(s);
13522 return;
13523 }
13524
13525 switch (opcode) {
13526 case 0: /* SHA1C */
13527 genfn = gen_helper_crypto_sha1c;
13528 feature = dc_isar_feature(aa64_sha1, s);
13529 break;
13530 case 1: /* SHA1P */
13531 genfn = gen_helper_crypto_sha1p;
13532 feature = dc_isar_feature(aa64_sha1, s);
13533 break;
13534 case 2: /* SHA1M */
13535 genfn = gen_helper_crypto_sha1m;
13536 feature = dc_isar_feature(aa64_sha1, s);
13537 break;
13538 case 3: /* SHA1SU0 */
13539 genfn = gen_helper_crypto_sha1su0;
13540 feature = dc_isar_feature(aa64_sha1, s);
13541 break;
13542 case 4: /* SHA256H */
13543 genfn = gen_helper_crypto_sha256h;
13544 feature = dc_isar_feature(aa64_sha256, s);
13545 break;
13546 case 5: /* SHA256H2 */
13547 genfn = gen_helper_crypto_sha256h2;
13548 feature = dc_isar_feature(aa64_sha256, s);
13549 break;
13550 case 6: /* SHA256SU1 */
13551 genfn = gen_helper_crypto_sha256su1;
13552 feature = dc_isar_feature(aa64_sha256, s);
13553 break;
13554 default:
13555 unallocated_encoding(s);
13556 return;
13557 }
13558
13559 if (!feature) {
13560 unallocated_encoding(s);
13561 return;
13562 }
13563
13564 if (!fp_access_check(s)) {
13565 return;
13566 }
13567 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, genfn);
13568 }
13569
13570 /* Crypto two-reg SHA
13571 * 31 24 23 22 21 17 16 12 11 10 9 5 4 0
13572 * +-----------------+------+-----------+--------+-----+------+------+
13573 * | 0 1 0 1 1 1 1 0 | size | 1 0 1 0 0 | opcode | 1 0 | Rn | Rd |
13574 * +-----------------+------+-----------+--------+-----+------+------+
13575 */
disas_crypto_two_reg_sha(DisasContext * s,uint32_t insn)13576 static void disas_crypto_two_reg_sha(DisasContext *s, uint32_t insn)
13577 {
13578 int size = extract32(insn, 22, 2);
13579 int opcode = extract32(insn, 12, 5);
13580 int rn = extract32(insn, 5, 5);
13581 int rd = extract32(insn, 0, 5);
13582 gen_helper_gvec_2 *genfn;
13583 bool feature;
13584
13585 if (size != 0) {
13586 unallocated_encoding(s);
13587 return;
13588 }
13589
13590 switch (opcode) {
13591 case 0: /* SHA1H */
13592 feature = dc_isar_feature(aa64_sha1, s);
13593 genfn = gen_helper_crypto_sha1h;
13594 break;
13595 case 1: /* SHA1SU1 */
13596 feature = dc_isar_feature(aa64_sha1, s);
13597 genfn = gen_helper_crypto_sha1su1;
13598 break;
13599 case 2: /* SHA256SU0 */
13600 feature = dc_isar_feature(aa64_sha256, s);
13601 genfn = gen_helper_crypto_sha256su0;
13602 break;
13603 default:
13604 unallocated_encoding(s);
13605 return;
13606 }
13607
13608 if (!feature) {
13609 unallocated_encoding(s);
13610 return;
13611 }
13612
13613 if (!fp_access_check(s)) {
13614 return;
13615 }
13616 gen_gvec_op2_ool(s, true, rd, rn, 0, genfn);
13617 }
13618
gen_rax1_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)13619 static void gen_rax1_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
13620 {
13621 tcg_gen_rotli_i64(d, m, 1);
13622 tcg_gen_xor_i64(d, d, n);
13623 }
13624
gen_rax1_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m)13625 static void gen_rax1_vec(unsigned vece, TCGv_vec d, TCGv_vec n, TCGv_vec m)
13626 {
13627 tcg_gen_rotli_vec(vece, d, m, 1);
13628 tcg_gen_xor_vec(vece, d, d, n);
13629 }
13630
gen_gvec_rax1(unsigned vece,uint32_t rd_ofs,uint32_t rn_ofs,uint32_t rm_ofs,uint32_t opr_sz,uint32_t max_sz)13631 void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
13632 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
13633 {
13634 static const TCGOpcode vecop_list[] = { INDEX_op_rotli_vec, 0 };
13635 static const GVecGen3 op = {
13636 .fni8 = gen_rax1_i64,
13637 .fniv = gen_rax1_vec,
13638 .opt_opc = vecop_list,
13639 .fno = gen_helper_crypto_rax1,
13640 .vece = MO_64,
13641 };
13642 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &op);
13643 }
13644
13645 /* Crypto three-reg SHA512
13646 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
13647 * +-----------------------+------+---+---+-----+--------+------+------+
13648 * | 1 1 0 0 1 1 1 0 0 1 1 | Rm | 1 | O | 0 0 | opcode | Rn | Rd |
13649 * +-----------------------+------+---+---+-----+--------+------+------+
13650 */
disas_crypto_three_reg_sha512(DisasContext * s,uint32_t insn)13651 static void disas_crypto_three_reg_sha512(DisasContext *s, uint32_t insn)
13652 {
13653 int opcode = extract32(insn, 10, 2);
13654 int o = extract32(insn, 14, 1);
13655 int rm = extract32(insn, 16, 5);
13656 int rn = extract32(insn, 5, 5);
13657 int rd = extract32(insn, 0, 5);
13658 bool feature;
13659 gen_helper_gvec_3 *oolfn = NULL;
13660 GVecGen3Fn *gvecfn = NULL;
13661
13662 if (o == 0) {
13663 switch (opcode) {
13664 case 0: /* SHA512H */
13665 feature = dc_isar_feature(aa64_sha512, s);
13666 oolfn = gen_helper_crypto_sha512h;
13667 break;
13668 case 1: /* SHA512H2 */
13669 feature = dc_isar_feature(aa64_sha512, s);
13670 oolfn = gen_helper_crypto_sha512h2;
13671 break;
13672 case 2: /* SHA512SU1 */
13673 feature = dc_isar_feature(aa64_sha512, s);
13674 oolfn = gen_helper_crypto_sha512su1;
13675 break;
13676 case 3: /* RAX1 */
13677 feature = dc_isar_feature(aa64_sha3, s);
13678 gvecfn = gen_gvec_rax1;
13679 break;
13680 default:
13681 g_assert_not_reached();
13682 }
13683 } else {
13684 switch (opcode) {
13685 case 0: /* SM3PARTW1 */
13686 feature = dc_isar_feature(aa64_sm3, s);
13687 oolfn = gen_helper_crypto_sm3partw1;
13688 break;
13689 case 1: /* SM3PARTW2 */
13690 feature = dc_isar_feature(aa64_sm3, s);
13691 oolfn = gen_helper_crypto_sm3partw2;
13692 break;
13693 case 2: /* SM4EKEY */
13694 feature = dc_isar_feature(aa64_sm4, s);
13695 oolfn = gen_helper_crypto_sm4ekey;
13696 break;
13697 default:
13698 unallocated_encoding(s);
13699 return;
13700 }
13701 }
13702
13703 if (!feature) {
13704 unallocated_encoding(s);
13705 return;
13706 }
13707
13708 if (!fp_access_check(s)) {
13709 return;
13710 }
13711
13712 if (oolfn) {
13713 gen_gvec_op3_ool(s, true, rd, rn, rm, 0, oolfn);
13714 } else {
13715 gen_gvec_fn3(s, true, rd, rn, rm, gvecfn, MO_64);
13716 }
13717 }
13718
13719 /* Crypto two-reg SHA512
13720 * 31 12 11 10 9 5 4 0
13721 * +-----------------------------------------+--------+------+------+
13722 * | 1 1 0 0 1 1 1 0 1 1 0 0 0 0 0 0 1 0 0 0 | opcode | Rn | Rd |
13723 * +-----------------------------------------+--------+------+------+
13724 */
disas_crypto_two_reg_sha512(DisasContext * s,uint32_t insn)13725 static void disas_crypto_two_reg_sha512(DisasContext *s, uint32_t insn)
13726 {
13727 int opcode = extract32(insn, 10, 2);
13728 int rn = extract32(insn, 5, 5);
13729 int rd = extract32(insn, 0, 5);
13730 bool feature;
13731
13732 switch (opcode) {
13733 case 0: /* SHA512SU0 */
13734 feature = dc_isar_feature(aa64_sha512, s);
13735 break;
13736 case 1: /* SM4E */
13737 feature = dc_isar_feature(aa64_sm4, s);
13738 break;
13739 default:
13740 unallocated_encoding(s);
13741 return;
13742 }
13743
13744 if (!feature) {
13745 unallocated_encoding(s);
13746 return;
13747 }
13748
13749 if (!fp_access_check(s)) {
13750 return;
13751 }
13752
13753 switch (opcode) {
13754 case 0: /* SHA512SU0 */
13755 gen_gvec_op2_ool(s, true, rd, rn, 0, gen_helper_crypto_sha512su0);
13756 break;
13757 case 1: /* SM4E */
13758 gen_gvec_op3_ool(s, true, rd, rd, rn, 0, gen_helper_crypto_sm4e);
13759 break;
13760 default:
13761 g_assert_not_reached();
13762 }
13763 }
13764
13765 /* Crypto four-register
13766 * 31 23 22 21 20 16 15 14 10 9 5 4 0
13767 * +-------------------+-----+------+---+------+------+------+
13768 * | 1 1 0 0 1 1 1 0 0 | Op0 | Rm | 0 | Ra | Rn | Rd |
13769 * +-------------------+-----+------+---+------+------+------+
13770 */
disas_crypto_four_reg(DisasContext * s,uint32_t insn)13771 static void disas_crypto_four_reg(DisasContext *s, uint32_t insn)
13772 {
13773 int op0 = extract32(insn, 21, 2);
13774 int rm = extract32(insn, 16, 5);
13775 int ra = extract32(insn, 10, 5);
13776 int rn = extract32(insn, 5, 5);
13777 int rd = extract32(insn, 0, 5);
13778 bool feature;
13779
13780 switch (op0) {
13781 case 0: /* EOR3 */
13782 case 1: /* BCAX */
13783 feature = dc_isar_feature(aa64_sha3, s);
13784 break;
13785 case 2: /* SM3SS1 */
13786 feature = dc_isar_feature(aa64_sm3, s);
13787 break;
13788 default:
13789 unallocated_encoding(s);
13790 return;
13791 }
13792
13793 if (!feature) {
13794 unallocated_encoding(s);
13795 return;
13796 }
13797
13798 if (!fp_access_check(s)) {
13799 return;
13800 }
13801
13802 if (op0 < 2) {
13803 TCGv_i64 tcg_op1, tcg_op2, tcg_op3, tcg_res[2];
13804 int pass;
13805
13806 tcg_op1 = tcg_temp_new_i64();
13807 tcg_op2 = tcg_temp_new_i64();
13808 tcg_op3 = tcg_temp_new_i64();
13809 tcg_res[0] = tcg_temp_new_i64();
13810 tcg_res[1] = tcg_temp_new_i64();
13811
13812 for (pass = 0; pass < 2; pass++) {
13813 read_vec_element(s, tcg_op1, rn, pass, MO_64);
13814 read_vec_element(s, tcg_op2, rm, pass, MO_64);
13815 read_vec_element(s, tcg_op3, ra, pass, MO_64);
13816
13817 if (op0 == 0) {
13818 /* EOR3 */
13819 tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op3);
13820 } else {
13821 /* BCAX */
13822 tcg_gen_andc_i64(tcg_res[pass], tcg_op2, tcg_op3);
13823 }
13824 tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
13825 }
13826 write_vec_element(s, tcg_res[0], rd, 0, MO_64);
13827 write_vec_element(s, tcg_res[1], rd, 1, MO_64);
13828 } else {
13829 TCGv_i32 tcg_op1, tcg_op2, tcg_op3, tcg_res, tcg_zero;
13830
13831 tcg_op1 = tcg_temp_new_i32();
13832 tcg_op2 = tcg_temp_new_i32();
13833 tcg_op3 = tcg_temp_new_i32();
13834 tcg_res = tcg_temp_new_i32();
13835 tcg_zero = tcg_constant_i32(0);
13836
13837 read_vec_element_i32(s, tcg_op1, rn, 3, MO_32);
13838 read_vec_element_i32(s, tcg_op2, rm, 3, MO_32);
13839 read_vec_element_i32(s, tcg_op3, ra, 3, MO_32);
13840
13841 tcg_gen_rotri_i32(tcg_res, tcg_op1, 20);
13842 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2);
13843 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3);
13844 tcg_gen_rotri_i32(tcg_res, tcg_res, 25);
13845
13846 write_vec_element_i32(s, tcg_zero, rd, 0, MO_32);
13847 write_vec_element_i32(s, tcg_zero, rd, 1, MO_32);
13848 write_vec_element_i32(s, tcg_zero, rd, 2, MO_32);
13849 write_vec_element_i32(s, tcg_res, rd, 3, MO_32);
13850 }
13851 }
13852
13853 /* Crypto XAR
13854 * 31 21 20 16 15 10 9 5 4 0
13855 * +-----------------------+------+--------+------+------+
13856 * | 1 1 0 0 1 1 1 0 1 0 0 | Rm | imm6 | Rn | Rd |
13857 * +-----------------------+------+--------+------+------+
13858 */
disas_crypto_xar(DisasContext * s,uint32_t insn)13859 static void disas_crypto_xar(DisasContext *s, uint32_t insn)
13860 {
13861 int rm = extract32(insn, 16, 5);
13862 int imm6 = extract32(insn, 10, 6);
13863 int rn = extract32(insn, 5, 5);
13864 int rd = extract32(insn, 0, 5);
13865
13866 if (!dc_isar_feature(aa64_sha3, s)) {
13867 unallocated_encoding(s);
13868 return;
13869 }
13870
13871 if (!fp_access_check(s)) {
13872 return;
13873 }
13874
13875 gen_gvec_xar(MO_64, vec_full_reg_offset(s, rd),
13876 vec_full_reg_offset(s, rn),
13877 vec_full_reg_offset(s, rm), imm6, 16,
13878 vec_full_reg_size(s));
13879 }
13880
13881 /* Crypto three-reg imm2
13882 * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
13883 * +-----------------------+------+-----+------+--------+------+------+
13884 * | 1 1 0 0 1 1 1 0 0 1 0 | Rm | 1 0 | imm2 | opcode | Rn | Rd |
13885 * +-----------------------+------+-----+------+--------+------+------+
13886 */
disas_crypto_three_reg_imm2(DisasContext * s,uint32_t insn)13887 static void disas_crypto_three_reg_imm2(DisasContext *s, uint32_t insn)
13888 {
13889 static gen_helper_gvec_3 * const fns[4] = {
13890 gen_helper_crypto_sm3tt1a, gen_helper_crypto_sm3tt1b,
13891 gen_helper_crypto_sm3tt2a, gen_helper_crypto_sm3tt2b,
13892 };
13893 int opcode = extract32(insn, 10, 2);
13894 int imm2 = extract32(insn, 12, 2);
13895 int rm = extract32(insn, 16, 5);
13896 int rn = extract32(insn, 5, 5);
13897 int rd = extract32(insn, 0, 5);
13898
13899 if (!dc_isar_feature(aa64_sm3, s)) {
13900 unallocated_encoding(s);
13901 return;
13902 }
13903
13904 if (!fp_access_check(s)) {
13905 return;
13906 }
13907
13908 gen_gvec_op3_ool(s, true, rd, rn, rm, imm2, fns[opcode]);
13909 }
13910
13911 /* C3.6 Data processing - SIMD, inc Crypto
13912 *
13913 * As the decode gets a little complex we are using a table based
13914 * approach for this part of the decode.
13915 */
13916 static const AArch64DecodeTable data_proc_simd[] = {
13917 /* pattern , mask , fn */
13918 { 0x0e200400, 0x9f200400, disas_simd_three_reg_same },
13919 { 0x0e008400, 0x9f208400, disas_simd_three_reg_same_extra },
13920 { 0x0e200000, 0x9f200c00, disas_simd_three_reg_diff },
13921 { 0x0e200800, 0x9f3e0c00, disas_simd_two_reg_misc },
13922 { 0x0e300800, 0x9f3e0c00, disas_simd_across_lanes },
13923 { 0x0e000400, 0x9fe08400, disas_simd_copy },
13924 { 0x0f000000, 0x9f000400, disas_simd_indexed }, /* vector indexed */
13925 /* simd_mod_imm decode is a subset of simd_shift_imm, so must precede it */
13926 { 0x0f000400, 0x9ff80400, disas_simd_mod_imm },
13927 { 0x0f000400, 0x9f800400, disas_simd_shift_imm },
13928 { 0x0e000000, 0xbf208c00, disas_simd_tb },
13929 { 0x0e000800, 0xbf208c00, disas_simd_zip_trn },
13930 { 0x2e000000, 0xbf208400, disas_simd_ext },
13931 { 0x5e200400, 0xdf200400, disas_simd_scalar_three_reg_same },
13932 { 0x5e008400, 0xdf208400, disas_simd_scalar_three_reg_same_extra },
13933 { 0x5e200000, 0xdf200c00, disas_simd_scalar_three_reg_diff },
13934 { 0x5e200800, 0xdf3e0c00, disas_simd_scalar_two_reg_misc },
13935 { 0x5e300800, 0xdf3e0c00, disas_simd_scalar_pairwise },
13936 { 0x5e000400, 0xdfe08400, disas_simd_scalar_copy },
13937 { 0x5f000000, 0xdf000400, disas_simd_indexed }, /* scalar indexed */
13938 { 0x5f000400, 0xdf800400, disas_simd_scalar_shift_imm },
13939 { 0x4e280800, 0xff3e0c00, disas_crypto_aes },
13940 { 0x5e000000, 0xff208c00, disas_crypto_three_reg_sha },
13941 { 0x5e280800, 0xff3e0c00, disas_crypto_two_reg_sha },
13942 { 0xce608000, 0xffe0b000, disas_crypto_three_reg_sha512 },
13943 { 0xcec08000, 0xfffff000, disas_crypto_two_reg_sha512 },
13944 { 0xce000000, 0xff808000, disas_crypto_four_reg },
13945 { 0xce800000, 0xffe00000, disas_crypto_xar },
13946 { 0xce408000, 0xffe0c000, disas_crypto_three_reg_imm2 },
13947 { 0x0e400400, 0x9f60c400, disas_simd_three_reg_same_fp16 },
13948 { 0x0e780800, 0x8f7e0c00, disas_simd_two_reg_misc_fp16 },
13949 { 0x5e400400, 0xdf60c400, disas_simd_scalar_three_reg_same_fp16 },
13950 { 0x00000000, 0x00000000, NULL }
13951 };
13952
disas_data_proc_simd(DisasContext * s,uint32_t insn)13953 static void disas_data_proc_simd(DisasContext *s, uint32_t insn)
13954 {
13955 /* Note that this is called with all non-FP cases from
13956 * table C3-6 so it must UNDEF for entries not specifically
13957 * allocated to instructions in that table.
13958 */
13959 AArch64DecodeFn *fn = lookup_disas_fn(&data_proc_simd[0], insn);
13960 if (fn) {
13961 fn(s, insn);
13962 } else {
13963 unallocated_encoding(s);
13964 }
13965 }
13966
13967 /* C3.6 Data processing - SIMD and floating point */
disas_data_proc_simd_fp(DisasContext * s,uint32_t insn)13968 static void disas_data_proc_simd_fp(DisasContext *s, uint32_t insn)
13969 {
13970 if (extract32(insn, 28, 1) == 1 && extract32(insn, 30, 1) == 0) {
13971 disas_data_proc_fp(s, insn);
13972 } else {
13973 /* SIMD, including crypto */
13974 disas_data_proc_simd(s, insn);
13975 }
13976 }
13977
trans_OK(DisasContext * s,arg_OK * a)13978 static bool trans_OK(DisasContext *s, arg_OK *a)
13979 {
13980 return true;
13981 }
13982
trans_FAIL(DisasContext * s,arg_OK * a)13983 static bool trans_FAIL(DisasContext *s, arg_OK *a)
13984 {
13985 s->is_nonstreaming = true;
13986 return true;
13987 }
13988
13989 /**
13990 * is_guarded_page:
13991 * @env: The cpu environment
13992 * @s: The DisasContext
13993 *
13994 * Return true if the page is guarded.
13995 */
is_guarded_page(CPUARMState * env,DisasContext * s)13996 static bool is_guarded_page(CPUARMState *env, DisasContext *s)
13997 {
13998 uint64_t addr = s->base.pc_first;
13999 #ifdef CONFIG_USER_ONLY
14000 return page_get_flags(addr) & PAGE_BTI;
14001 #else
14002 CPUTLBEntryFull *full;
14003 void *host;
14004 int mmu_idx = arm_to_core_mmu_idx(s->mmu_idx);
14005 int flags;
14006
14007 /*
14008 * We test this immediately after reading an insn, which means
14009 * that the TLB entry must be present and valid, and thus this
14010 * access will never raise an exception.
14011 */
14012 flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx,
14013 false, &host, &full, 0);
14014 assert(!(flags & TLB_INVALID_MASK));
14015
14016 return full->extra.arm.guarded;
14017 #endif
14018 }
14019
14020 /**
14021 * btype_destination_ok:
14022 * @insn: The instruction at the branch destination
14023 * @bt: SCTLR_ELx.BT
14024 * @btype: PSTATE.BTYPE, and is non-zero
14025 *
14026 * On a guarded page, there are a limited number of insns
14027 * that may be present at the branch target:
14028 * - branch target identifiers,
14029 * - paciasp, pacibsp,
14030 * - BRK insn
14031 * - HLT insn
14032 * Anything else causes a Branch Target Exception.
14033 *
14034 * Return true if the branch is compatible, false to raise BTITRAP.
14035 */
btype_destination_ok(uint32_t insn,bool bt,int btype)14036 static bool btype_destination_ok(uint32_t insn, bool bt, int btype)
14037 {
14038 if ((insn & 0xfffff01fu) == 0xd503201fu) {
14039 /* HINT space */
14040 switch (extract32(insn, 5, 7)) {
14041 case 0b011001: /* PACIASP */
14042 case 0b011011: /* PACIBSP */
14043 /*
14044 * If SCTLR_ELx.BT, then PACI*SP are not compatible
14045 * with btype == 3. Otherwise all btype are ok.
14046 */
14047 return !bt || btype != 3;
14048 case 0b100000: /* BTI */
14049 /* Not compatible with any btype. */
14050 return false;
14051 case 0b100010: /* BTI c */
14052 /* Not compatible with btype == 3 */
14053 return btype != 3;
14054 case 0b100100: /* BTI j */
14055 /* Not compatible with btype == 2 */
14056 return btype != 2;
14057 case 0b100110: /* BTI jc */
14058 /* Compatible with any btype. */
14059 return true;
14060 }
14061 } else {
14062 switch (insn & 0xffe0001fu) {
14063 case 0xd4200000u: /* BRK */
14064 case 0xd4400000u: /* HLT */
14065 /* Give priority to the breakpoint exception. */
14066 return true;
14067 }
14068 }
14069 return false;
14070 }
14071
14072 /* C3.1 A64 instruction index by encoding */
disas_a64_legacy(DisasContext * s,uint32_t insn)14073 static void disas_a64_legacy(DisasContext *s, uint32_t insn)
14074 {
14075 switch (extract32(insn, 25, 4)) {
14076 case 0x5:
14077 case 0xd: /* Data processing - register */
14078 disas_data_proc_reg(s, insn);
14079 break;
14080 case 0x7:
14081 case 0xf: /* Data processing - SIMD and floating point */
14082 disas_data_proc_simd_fp(s, insn);
14083 break;
14084 default:
14085 unallocated_encoding(s);
14086 break;
14087 }
14088 }
14089
aarch64_tr_init_disas_context(DisasContextBase * dcbase,CPUState * cpu)14090 static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
14091 CPUState *cpu)
14092 {
14093 DisasContext *dc = container_of(dcbase, DisasContext, base);
14094 CPUARMState *env = cpu_env(cpu);
14095 ARMCPU *arm_cpu = env_archcpu(env);
14096 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb);
14097 int bound, core_mmu_idx;
14098
14099 dc->isar = &arm_cpu->isar;
14100 dc->condjmp = 0;
14101 dc->pc_save = dc->base.pc_first;
14102 dc->aarch64 = true;
14103 dc->thumb = false;
14104 dc->sctlr_b = 0;
14105 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE;
14106 dc->condexec_mask = 0;
14107 dc->condexec_cond = 0;
14108 core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX);
14109 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx);
14110 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII);
14111 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID);
14112 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA);
14113 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
14114 #if !defined(CONFIG_USER_ONLY)
14115 dc->user = (dc->current_el == 0);
14116 #endif
14117 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL);
14118 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM);
14119 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL);
14120 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE);
14121 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC);
14122 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET);
14123 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL);
14124 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL);
14125 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16;
14126 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16;
14127 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE);
14128 dc->bt = EX_TBFLAG_A64(tb_flags, BT);
14129 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE);
14130 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV);
14131 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA);
14132 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0);
14133 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE);
14134 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
14135 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
14136 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
14137 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING);
14138 dc->naa = EX_TBFLAG_A64(tb_flags, NAA);
14139 dc->nv = EX_TBFLAG_A64(tb_flags, NV);
14140 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1);
14141 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2);
14142 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20);
14143 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE);
14144 dc->vec_len = 0;
14145 dc->vec_stride = 0;
14146 dc->cp_regs = arm_cpu->cp_regs;
14147 dc->features = env->features;
14148 dc->dcz_blocksize = arm_cpu->dcz_blocksize;
14149 dc->gm_blocksize = arm_cpu->gm_blocksize;
14150
14151 #ifdef CONFIG_USER_ONLY
14152 /* In sve_probe_page, we assume TBI is enabled. */
14153 tcg_debug_assert(dc->tbid & 1);
14154 #endif
14155
14156 dc->lse2 = dc_isar_feature(aa64_lse2, dc);
14157
14158 /* Single step state. The code-generation logic here is:
14159 * SS_ACTIVE == 0:
14160 * generate code with no special handling for single-stepping (except
14161 * that anything that can make us go to SS_ACTIVE == 1 must end the TB;
14162 * this happens anyway because those changes are all system register or
14163 * PSTATE writes).
14164 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
14165 * emit code for one insn
14166 * emit code to clear PSTATE.SS
14167 * emit code to generate software step exception for completed step
14168 * end TB (as usual for having generated an exception)
14169 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
14170 * emit code to generate a software step exception
14171 * end the TB
14172 */
14173 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE);
14174 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS);
14175 dc->is_ldex = false;
14176
14177 /* Bound the number of insns to execute to those left on the page. */
14178 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4;
14179
14180 /* If architectural single step active, limit to 1. */
14181 if (dc->ss_active) {
14182 bound = 1;
14183 }
14184 dc->base.max_insns = MIN(dc->base.max_insns, bound);
14185 }
14186
aarch64_tr_tb_start(DisasContextBase * db,CPUState * cpu)14187 static void aarch64_tr_tb_start(DisasContextBase *db, CPUState *cpu)
14188 {
14189 }
14190
aarch64_tr_insn_start(DisasContextBase * dcbase,CPUState * cpu)14191 static void aarch64_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
14192 {
14193 DisasContext *dc = container_of(dcbase, DisasContext, base);
14194 target_ulong pc_arg = dc->base.pc_next;
14195
14196 if (tb_cflags(dcbase->tb) & CF_PCREL) {
14197 pc_arg &= ~TARGET_PAGE_MASK;
14198 }
14199 tcg_gen_insn_start(pc_arg, 0, 0);
14200 dc->insn_start_updated = false;
14201 }
14202
aarch64_tr_translate_insn(DisasContextBase * dcbase,CPUState * cpu)14203 static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
14204 {
14205 DisasContext *s = container_of(dcbase, DisasContext, base);
14206 CPUARMState *env = cpu_env(cpu);
14207 uint64_t pc = s->base.pc_next;
14208 uint32_t insn;
14209
14210 /* Singlestep exceptions have the highest priority. */
14211 if (s->ss_active && !s->pstate_ss) {
14212 /* Singlestep state is Active-pending.
14213 * If we're in this state at the start of a TB then either
14214 * a) we just took an exception to an EL which is being debugged
14215 * and this is the first insn in the exception handler
14216 * b) debug exceptions were masked and we just unmasked them
14217 * without changing EL (eg by clearing PSTATE.D)
14218 * In either case we're going to take a swstep exception in the
14219 * "did not step an insn" case, and so the syndrome ISV and EX
14220 * bits should be zero.
14221 */
14222 assert(s->base.num_insns == 1);
14223 gen_swstep_exception(s, 0, 0);
14224 s->base.is_jmp = DISAS_NORETURN;
14225 s->base.pc_next = pc + 4;
14226 return;
14227 }
14228
14229 if (pc & 3) {
14230 /*
14231 * PC alignment fault. This has priority over the instruction abort
14232 * that we would receive from a translation fault via arm_ldl_code.
14233 * This should only be possible after an indirect branch, at the
14234 * start of the TB.
14235 */
14236 assert(s->base.num_insns == 1);
14237 gen_helper_exception_pc_alignment(tcg_env, tcg_constant_tl(pc));
14238 s->base.is_jmp = DISAS_NORETURN;
14239 s->base.pc_next = QEMU_ALIGN_UP(pc, 4);
14240 return;
14241 }
14242
14243 s->pc_curr = pc;
14244 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b);
14245 s->insn = insn;
14246 s->base.pc_next = pc + 4;
14247
14248 s->fp_access_checked = false;
14249 s->sve_access_checked = false;
14250
14251 if (s->pstate_il) {
14252 /*
14253 * Illegal execution state. This has priority over BTI
14254 * exceptions, but comes after instruction abort exceptions.
14255 */
14256 gen_exception_insn(s, 0, EXCP_UDEF, syn_illegalstate());
14257 return;
14258 }
14259
14260 if (dc_isar_feature(aa64_bti, s)) {
14261 if (s->base.num_insns == 1) {
14262 /*
14263 * At the first insn of the TB, compute s->guarded_page.
14264 * We delayed computing this until successfully reading
14265 * the first insn of the TB, above. This (mostly) ensures
14266 * that the softmmu tlb entry has been populated, and the
14267 * page table GP bit is available.
14268 *
14269 * Note that we need to compute this even if btype == 0,
14270 * because this value is used for BR instructions later
14271 * where ENV is not available.
14272 */
14273 s->guarded_page = is_guarded_page(env, s);
14274
14275 /* First insn can have btype set to non-zero. */
14276 tcg_debug_assert(s->btype >= 0);
14277
14278 /*
14279 * Note that the Branch Target Exception has fairly high
14280 * priority -- below debugging exceptions but above most
14281 * everything else. This allows us to handle this now
14282 * instead of waiting until the insn is otherwise decoded.
14283 */
14284 if (s->btype != 0
14285 && s->guarded_page
14286 && !btype_destination_ok(insn, s->bt, s->btype)) {
14287 gen_exception_insn(s, 0, EXCP_UDEF, syn_btitrap(s->btype));
14288 return;
14289 }
14290 } else {
14291 /* Not the first insn: btype must be 0. */
14292 tcg_debug_assert(s->btype == 0);
14293 }
14294 }
14295
14296 s->is_nonstreaming = false;
14297 if (s->sme_trap_nonstreaming) {
14298 disas_sme_fa64(s, insn);
14299 }
14300
14301 if (!disas_a64(s, insn) &&
14302 !disas_sme(s, insn) &&
14303 !disas_sve(s, insn)) {
14304 disas_a64_legacy(s, insn);
14305 }
14306
14307 /*
14308 * After execution of most insns, btype is reset to 0.
14309 * Note that we set btype == -1 when the insn sets btype.
14310 */
14311 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) {
14312 reset_btype(s);
14313 }
14314 }
14315
aarch64_tr_tb_stop(DisasContextBase * dcbase,CPUState * cpu)14316 static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
14317 {
14318 DisasContext *dc = container_of(dcbase, DisasContext, base);
14319
14320 if (unlikely(dc->ss_active)) {
14321 /* Note that this means single stepping WFI doesn't halt the CPU.
14322 * For conditional branch insns this is harmless unreachable code as
14323 * gen_goto_tb() has already handled emitting the debug exception
14324 * (and thus a tb-jump is not possible when singlestepping).
14325 */
14326 switch (dc->base.is_jmp) {
14327 default:
14328 gen_a64_update_pc(dc, 4);
14329 /* fall through */
14330 case DISAS_EXIT:
14331 case DISAS_JUMP:
14332 gen_step_complete_exception(dc);
14333 break;
14334 case DISAS_NORETURN:
14335 break;
14336 }
14337 } else {
14338 switch (dc->base.is_jmp) {
14339 case DISAS_NEXT:
14340 case DISAS_TOO_MANY:
14341 gen_goto_tb(dc, 1, 4);
14342 break;
14343 default:
14344 case DISAS_UPDATE_EXIT:
14345 gen_a64_update_pc(dc, 4);
14346 /* fall through */
14347 case DISAS_EXIT:
14348 tcg_gen_exit_tb(NULL, 0);
14349 break;
14350 case DISAS_UPDATE_NOCHAIN:
14351 gen_a64_update_pc(dc, 4);
14352 /* fall through */
14353 case DISAS_JUMP:
14354 tcg_gen_lookup_and_goto_ptr();
14355 break;
14356 case DISAS_NORETURN:
14357 case DISAS_SWI:
14358 break;
14359 case DISAS_WFE:
14360 gen_a64_update_pc(dc, 4);
14361 gen_helper_wfe(tcg_env);
14362 break;
14363 case DISAS_YIELD:
14364 gen_a64_update_pc(dc, 4);
14365 gen_helper_yield(tcg_env);
14366 break;
14367 case DISAS_WFI:
14368 /*
14369 * This is a special case because we don't want to just halt
14370 * the CPU if trying to debug across a WFI.
14371 */
14372 gen_a64_update_pc(dc, 4);
14373 gen_helper_wfi(tcg_env, tcg_constant_i32(4));
14374 /*
14375 * The helper doesn't necessarily throw an exception, but we
14376 * must go back to the main loop to check for interrupts anyway.
14377 */
14378 tcg_gen_exit_tb(NULL, 0);
14379 break;
14380 }
14381 }
14382 }
14383
14384 const TranslatorOps aarch64_translator_ops = {
14385 .init_disas_context = aarch64_tr_init_disas_context,
14386 .tb_start = aarch64_tr_tb_start,
14387 .insn_start = aarch64_tr_insn_start,
14388 .translate_insn = aarch64_tr_translate_insn,
14389 .tb_stop = aarch64_tr_tb_stop,
14390 };
14391