1 /*
2 * ARM AdvSIMD / SVE Vector Operations
3 *
4 * Copyright (c) 2018 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "cpu.h"
22 #include "exec/helper-proto.h"
23 #include "tcg/tcg-gvec-desc.h"
24 #include "fpu/softfloat.h"
25 #include "qemu/int128.h"
26 #include "crypto/clmul.h"
27 #include "vec_internal.h"
28
29 /*
30 * Data for expanding active predicate bits to bytes, for byte elements.
31 *
32 * for (i = 0; i < 256; ++i) {
33 * unsigned long m = 0;
34 * for (j = 0; j < 8; j++) {
35 * if ((i >> j) & 1) {
36 * m |= 0xfful << (j << 3);
37 * }
38 * }
39 * printf("0x%016lx,\n", m);
40 * }
41 */
42 const uint64_t expand_pred_b_data[256] = {
43 0x0000000000000000, 0x00000000000000ff, 0x000000000000ff00,
44 0x000000000000ffff, 0x0000000000ff0000, 0x0000000000ff00ff,
45 0x0000000000ffff00, 0x0000000000ffffff, 0x00000000ff000000,
46 0x00000000ff0000ff, 0x00000000ff00ff00, 0x00000000ff00ffff,
47 0x00000000ffff0000, 0x00000000ffff00ff, 0x00000000ffffff00,
48 0x00000000ffffffff, 0x000000ff00000000, 0x000000ff000000ff,
49 0x000000ff0000ff00, 0x000000ff0000ffff, 0x000000ff00ff0000,
50 0x000000ff00ff00ff, 0x000000ff00ffff00, 0x000000ff00ffffff,
51 0x000000ffff000000, 0x000000ffff0000ff, 0x000000ffff00ff00,
52 0x000000ffff00ffff, 0x000000ffffff0000, 0x000000ffffff00ff,
53 0x000000ffffffff00, 0x000000ffffffffff, 0x0000ff0000000000,
54 0x0000ff00000000ff, 0x0000ff000000ff00, 0x0000ff000000ffff,
55 0x0000ff0000ff0000, 0x0000ff0000ff00ff, 0x0000ff0000ffff00,
56 0x0000ff0000ffffff, 0x0000ff00ff000000, 0x0000ff00ff0000ff,
57 0x0000ff00ff00ff00, 0x0000ff00ff00ffff, 0x0000ff00ffff0000,
58 0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0x0000ff00ffffffff,
59 0x0000ffff00000000, 0x0000ffff000000ff, 0x0000ffff0000ff00,
60 0x0000ffff0000ffff, 0x0000ffff00ff0000, 0x0000ffff00ff00ff,
61 0x0000ffff00ffff00, 0x0000ffff00ffffff, 0x0000ffffff000000,
62 0x0000ffffff0000ff, 0x0000ffffff00ff00, 0x0000ffffff00ffff,
63 0x0000ffffffff0000, 0x0000ffffffff00ff, 0x0000ffffffffff00,
64 0x0000ffffffffffff, 0x00ff000000000000, 0x00ff0000000000ff,
65 0x00ff00000000ff00, 0x00ff00000000ffff, 0x00ff000000ff0000,
66 0x00ff000000ff00ff, 0x00ff000000ffff00, 0x00ff000000ffffff,
67 0x00ff0000ff000000, 0x00ff0000ff0000ff, 0x00ff0000ff00ff00,
68 0x00ff0000ff00ffff, 0x00ff0000ffff0000, 0x00ff0000ffff00ff,
69 0x00ff0000ffffff00, 0x00ff0000ffffffff, 0x00ff00ff00000000,
70 0x00ff00ff000000ff, 0x00ff00ff0000ff00, 0x00ff00ff0000ffff,
71 0x00ff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00,
72 0x00ff00ff00ffffff, 0x00ff00ffff000000, 0x00ff00ffff0000ff,
73 0x00ff00ffff00ff00, 0x00ff00ffff00ffff, 0x00ff00ffffff0000,
74 0x00ff00ffffff00ff, 0x00ff00ffffffff00, 0x00ff00ffffffffff,
75 0x00ffff0000000000, 0x00ffff00000000ff, 0x00ffff000000ff00,
76 0x00ffff000000ffff, 0x00ffff0000ff0000, 0x00ffff0000ff00ff,
77 0x00ffff0000ffff00, 0x00ffff0000ffffff, 0x00ffff00ff000000,
78 0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0x00ffff00ff00ffff,
79 0x00ffff00ffff0000, 0x00ffff00ffff00ff, 0x00ffff00ffffff00,
80 0x00ffff00ffffffff, 0x00ffffff00000000, 0x00ffffff000000ff,
81 0x00ffffff0000ff00, 0x00ffffff0000ffff, 0x00ffffff00ff0000,
82 0x00ffffff00ff00ff, 0x00ffffff00ffff00, 0x00ffffff00ffffff,
83 0x00ffffffff000000, 0x00ffffffff0000ff, 0x00ffffffff00ff00,
84 0x00ffffffff00ffff, 0x00ffffffffff0000, 0x00ffffffffff00ff,
85 0x00ffffffffffff00, 0x00ffffffffffffff, 0xff00000000000000,
86 0xff000000000000ff, 0xff0000000000ff00, 0xff0000000000ffff,
87 0xff00000000ff0000, 0xff00000000ff00ff, 0xff00000000ffff00,
88 0xff00000000ffffff, 0xff000000ff000000, 0xff000000ff0000ff,
89 0xff000000ff00ff00, 0xff000000ff00ffff, 0xff000000ffff0000,
90 0xff000000ffff00ff, 0xff000000ffffff00, 0xff000000ffffffff,
91 0xff0000ff00000000, 0xff0000ff000000ff, 0xff0000ff0000ff00,
92 0xff0000ff0000ffff, 0xff0000ff00ff0000, 0xff0000ff00ff00ff,
93 0xff0000ff00ffff00, 0xff0000ff00ffffff, 0xff0000ffff000000,
94 0xff0000ffff0000ff, 0xff0000ffff00ff00, 0xff0000ffff00ffff,
95 0xff0000ffffff0000, 0xff0000ffffff00ff, 0xff0000ffffffff00,
96 0xff0000ffffffffff, 0xff00ff0000000000, 0xff00ff00000000ff,
97 0xff00ff000000ff00, 0xff00ff000000ffff, 0xff00ff0000ff0000,
98 0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0xff00ff0000ffffff,
99 0xff00ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00,
100 0xff00ff00ff00ffff, 0xff00ff00ffff0000, 0xff00ff00ffff00ff,
101 0xff00ff00ffffff00, 0xff00ff00ffffffff, 0xff00ffff00000000,
102 0xff00ffff000000ff, 0xff00ffff0000ff00, 0xff00ffff0000ffff,
103 0xff00ffff00ff0000, 0xff00ffff00ff00ff, 0xff00ffff00ffff00,
104 0xff00ffff00ffffff, 0xff00ffffff000000, 0xff00ffffff0000ff,
105 0xff00ffffff00ff00, 0xff00ffffff00ffff, 0xff00ffffffff0000,
106 0xff00ffffffff00ff, 0xff00ffffffffff00, 0xff00ffffffffffff,
107 0xffff000000000000, 0xffff0000000000ff, 0xffff00000000ff00,
108 0xffff00000000ffff, 0xffff000000ff0000, 0xffff000000ff00ff,
109 0xffff000000ffff00, 0xffff000000ffffff, 0xffff0000ff000000,
110 0xffff0000ff0000ff, 0xffff0000ff00ff00, 0xffff0000ff00ffff,
111 0xffff0000ffff0000, 0xffff0000ffff00ff, 0xffff0000ffffff00,
112 0xffff0000ffffffff, 0xffff00ff00000000, 0xffff00ff000000ff,
113 0xffff00ff0000ff00, 0xffff00ff0000ffff, 0xffff00ff00ff0000,
114 0xffff00ff00ff00ff, 0xffff00ff00ffff00, 0xffff00ff00ffffff,
115 0xffff00ffff000000, 0xffff00ffff0000ff, 0xffff00ffff00ff00,
116 0xffff00ffff00ffff, 0xffff00ffffff0000, 0xffff00ffffff00ff,
117 0xffff00ffffffff00, 0xffff00ffffffffff, 0xffffff0000000000,
118 0xffffff00000000ff, 0xffffff000000ff00, 0xffffff000000ffff,
119 0xffffff0000ff0000, 0xffffff0000ff00ff, 0xffffff0000ffff00,
120 0xffffff0000ffffff, 0xffffff00ff000000, 0xffffff00ff0000ff,
121 0xffffff00ff00ff00, 0xffffff00ff00ffff, 0xffffff00ffff0000,
122 0xffffff00ffff00ff, 0xffffff00ffffff00, 0xffffff00ffffffff,
123 0xffffffff00000000, 0xffffffff000000ff, 0xffffffff0000ff00,
124 0xffffffff0000ffff, 0xffffffff00ff0000, 0xffffffff00ff00ff,
125 0xffffffff00ffff00, 0xffffffff00ffffff, 0xffffffffff000000,
126 0xffffffffff0000ff, 0xffffffffff00ff00, 0xffffffffff00ffff,
127 0xffffffffffff0000, 0xffffffffffff00ff, 0xffffffffffffff00,
128 0xffffffffffffffff,
129 };
130
131 /*
132 * Similarly for half-word elements.
133 * for (i = 0; i < 256; ++i) {
134 * unsigned long m = 0;
135 * if (i & 0xaa) {
136 * continue;
137 * }
138 * for (j = 0; j < 8; j += 2) {
139 * if ((i >> j) & 1) {
140 * m |= 0xfffful << (j << 3);
141 * }
142 * }
143 * printf("[0x%x] = 0x%016lx,\n", i, m);
144 * }
145 */
146 const uint64_t expand_pred_h_data[0x55 + 1] = {
147 [0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000,
148 [0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000,
149 [0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000,
150 [0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000,
151 [0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000,
152 [0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000,
153 [0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000,
154 [0x55] = 0xffffffffffffffff,
155 };
156
157 /* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */
do_sqrdmlah_b(int8_t src1,int8_t src2,int8_t src3,bool neg,bool round)158 int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3,
159 bool neg, bool round)
160 {
161 /*
162 * Simplify:
163 * = ((a3 << 8) + ((e1 * e2) << 1) + (round << 7)) >> 8
164 * = ((a3 << 7) + (e1 * e2) + (round << 6)) >> 7
165 */
166 int32_t ret = (int32_t)src1 * src2;
167 if (neg) {
168 ret = -ret;
169 }
170 ret += ((int32_t)src3 << 7) + (round << 6);
171 ret >>= 7;
172
173 if (ret != (int8_t)ret) {
174 ret = (ret < 0 ? INT8_MIN : INT8_MAX);
175 }
176 return ret;
177 }
178
HELPER(sve2_sqrdmlah_b)179 void HELPER(sve2_sqrdmlah_b)(void *vd, void *vn, void *vm,
180 void *va, uint32_t desc)
181 {
182 intptr_t i, opr_sz = simd_oprsz(desc);
183 int8_t *d = vd, *n = vn, *m = vm, *a = va;
184
185 for (i = 0; i < opr_sz; ++i) {
186 d[i] = do_sqrdmlah_b(n[i], m[i], a[i], false, true);
187 }
188 }
189
HELPER(sve2_sqrdmlsh_b)190 void HELPER(sve2_sqrdmlsh_b)(void *vd, void *vn, void *vm,
191 void *va, uint32_t desc)
192 {
193 intptr_t i, opr_sz = simd_oprsz(desc);
194 int8_t *d = vd, *n = vn, *m = vm, *a = va;
195
196 for (i = 0; i < opr_sz; ++i) {
197 d[i] = do_sqrdmlah_b(n[i], m[i], a[i], true, true);
198 }
199 }
200
HELPER(sve2_sqdmulh_b)201 void HELPER(sve2_sqdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
202 {
203 intptr_t i, opr_sz = simd_oprsz(desc);
204 int8_t *d = vd, *n = vn, *m = vm;
205
206 for (i = 0; i < opr_sz; ++i) {
207 d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, false);
208 }
209 }
210
HELPER(sve2_sqrdmulh_b)211 void HELPER(sve2_sqrdmulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
212 {
213 intptr_t i, opr_sz = simd_oprsz(desc);
214 int8_t *d = vd, *n = vn, *m = vm;
215
216 for (i = 0; i < opr_sz; ++i) {
217 d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, true);
218 }
219 }
220
221 /* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */
do_sqrdmlah_h(int16_t src1,int16_t src2,int16_t src3,bool neg,bool round,uint32_t * sat)222 int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
223 bool neg, bool round, uint32_t *sat)
224 {
225 /* Simplify similarly to do_sqrdmlah_b above. */
226 int32_t ret = (int32_t)src1 * src2;
227 if (neg) {
228 ret = -ret;
229 }
230 ret += ((int32_t)src3 << 15) + (round << 14);
231 ret >>= 15;
232
233 if (ret != (int16_t)ret) {
234 *sat = 1;
235 ret = (ret < 0 ? INT16_MIN : INT16_MAX);
236 }
237 return ret;
238 }
239
HELPER(neon_qrdmlah_s16)240 uint32_t HELPER(neon_qrdmlah_s16)(CPUARMState *env, uint32_t src1,
241 uint32_t src2, uint32_t src3)
242 {
243 uint32_t *sat = &env->vfp.qc[0];
244 uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, false, true, sat);
245 uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16,
246 false, true, sat);
247 return deposit32(e1, 16, 16, e2);
248 }
249
HELPER(gvec_qrdmlah_s16)250 void HELPER(gvec_qrdmlah_s16)(void *vd, void *vn, void *vm,
251 void *vq, uint32_t desc)
252 {
253 uintptr_t opr_sz = simd_oprsz(desc);
254 int16_t *d = vd;
255 int16_t *n = vn;
256 int16_t *m = vm;
257 uintptr_t i;
258
259 for (i = 0; i < opr_sz / 2; ++i) {
260 d[i] = do_sqrdmlah_h(n[i], m[i], d[i], false, true, vq);
261 }
262 clear_tail(d, opr_sz, simd_maxsz(desc));
263 }
264
HELPER(neon_qrdmlsh_s16)265 uint32_t HELPER(neon_qrdmlsh_s16)(CPUARMState *env, uint32_t src1,
266 uint32_t src2, uint32_t src3)
267 {
268 uint32_t *sat = &env->vfp.qc[0];
269 uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, true, true, sat);
270 uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16,
271 true, true, sat);
272 return deposit32(e1, 16, 16, e2);
273 }
274
HELPER(gvec_qrdmlsh_s16)275 void HELPER(gvec_qrdmlsh_s16)(void *vd, void *vn, void *vm,
276 void *vq, uint32_t desc)
277 {
278 uintptr_t opr_sz = simd_oprsz(desc);
279 int16_t *d = vd;
280 int16_t *n = vn;
281 int16_t *m = vm;
282 uintptr_t i;
283
284 for (i = 0; i < opr_sz / 2; ++i) {
285 d[i] = do_sqrdmlah_h(n[i], m[i], d[i], true, true, vq);
286 }
287 clear_tail(d, opr_sz, simd_maxsz(desc));
288 }
289
HELPER(neon_sqdmulh_h)290 void HELPER(neon_sqdmulh_h)(void *vd, void *vn, void *vm,
291 void *vq, uint32_t desc)
292 {
293 intptr_t i, opr_sz = simd_oprsz(desc);
294 int16_t *d = vd, *n = vn, *m = vm;
295
296 for (i = 0; i < opr_sz / 2; ++i) {
297 d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, vq);
298 }
299 clear_tail(d, opr_sz, simd_maxsz(desc));
300 }
301
HELPER(neon_sqrdmulh_h)302 void HELPER(neon_sqrdmulh_h)(void *vd, void *vn, void *vm,
303 void *vq, uint32_t desc)
304 {
305 intptr_t i, opr_sz = simd_oprsz(desc);
306 int16_t *d = vd, *n = vn, *m = vm;
307
308 for (i = 0; i < opr_sz / 2; ++i) {
309 d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, vq);
310 }
311 clear_tail(d, opr_sz, simd_maxsz(desc));
312 }
313
HELPER(neon_sqdmulh_idx_h)314 void HELPER(neon_sqdmulh_idx_h)(void *vd, void *vn, void *vm,
315 void *vq, uint32_t desc)
316 {
317 intptr_t i, j, opr_sz = simd_oprsz(desc);
318 int idx = simd_data(desc);
319 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
320 intptr_t elements = opr_sz / 2;
321 intptr_t eltspersegment = MIN(16 / 2, elements);
322
323 for (i = 0; i < elements; i += 16 / 2) {
324 int16_t mm = m[i];
325 for (j = 0; j < eltspersegment; ++j) {
326 d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, vq);
327 }
328 }
329 clear_tail(d, opr_sz, simd_maxsz(desc));
330 }
331
HELPER(neon_sqrdmulh_idx_h)332 void HELPER(neon_sqrdmulh_idx_h)(void *vd, void *vn, void *vm,
333 void *vq, uint32_t desc)
334 {
335 intptr_t i, j, opr_sz = simd_oprsz(desc);
336 int idx = simd_data(desc);
337 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
338 intptr_t elements = opr_sz / 2;
339 intptr_t eltspersegment = MIN(16 / 2, elements);
340
341 for (i = 0; i < elements; i += 16 / 2) {
342 int16_t mm = m[i];
343 for (j = 0; j < eltspersegment; ++j) {
344 d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, vq);
345 }
346 }
347 clear_tail(d, opr_sz, simd_maxsz(desc));
348 }
349
HELPER(neon_sqrdmlah_idx_h)350 void HELPER(neon_sqrdmlah_idx_h)(void *vd, void *vn, void *vm,
351 void *vq, uint32_t desc)
352 {
353 intptr_t i, j, opr_sz = simd_oprsz(desc);
354 int idx = simd_data(desc);
355 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
356 intptr_t elements = opr_sz / 2;
357 intptr_t eltspersegment = MIN(16 / 2, elements);
358
359 for (i = 0; i < elements; i += 16 / 2) {
360 int16_t mm = m[i];
361 for (j = 0; j < eltspersegment; ++j) {
362 d[i + j] = do_sqrdmlah_h(n[i + j], mm, d[i + j], false, true, vq);
363 }
364 }
365 clear_tail(d, opr_sz, simd_maxsz(desc));
366 }
367
HELPER(neon_sqrdmlsh_idx_h)368 void HELPER(neon_sqrdmlsh_idx_h)(void *vd, void *vn, void *vm,
369 void *vq, uint32_t desc)
370 {
371 intptr_t i, j, opr_sz = simd_oprsz(desc);
372 int idx = simd_data(desc);
373 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
374 intptr_t elements = opr_sz / 2;
375 intptr_t eltspersegment = MIN(16 / 2, elements);
376
377 for (i = 0; i < elements; i += 16 / 2) {
378 int16_t mm = m[i];
379 for (j = 0; j < eltspersegment; ++j) {
380 d[i + j] = do_sqrdmlah_h(n[i + j], mm, d[i + j], true, true, vq);
381 }
382 }
383 clear_tail(d, opr_sz, simd_maxsz(desc));
384 }
385
HELPER(sve2_sqrdmlah_h)386 void HELPER(sve2_sqrdmlah_h)(void *vd, void *vn, void *vm,
387 void *va, uint32_t desc)
388 {
389 intptr_t i, opr_sz = simd_oprsz(desc);
390 int16_t *d = vd, *n = vn, *m = vm, *a = va;
391 uint32_t discard;
392
393 for (i = 0; i < opr_sz / 2; ++i) {
394 d[i] = do_sqrdmlah_h(n[i], m[i], a[i], false, true, &discard);
395 }
396 }
397
HELPER(sve2_sqrdmlsh_h)398 void HELPER(sve2_sqrdmlsh_h)(void *vd, void *vn, void *vm,
399 void *va, uint32_t desc)
400 {
401 intptr_t i, opr_sz = simd_oprsz(desc);
402 int16_t *d = vd, *n = vn, *m = vm, *a = va;
403 uint32_t discard;
404
405 for (i = 0; i < opr_sz / 2; ++i) {
406 d[i] = do_sqrdmlah_h(n[i], m[i], a[i], true, true, &discard);
407 }
408 }
409
HELPER(sve2_sqdmulh_h)410 void HELPER(sve2_sqdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
411 {
412 intptr_t i, opr_sz = simd_oprsz(desc);
413 int16_t *d = vd, *n = vn, *m = vm;
414 uint32_t discard;
415
416 for (i = 0; i < opr_sz / 2; ++i) {
417 d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, &discard);
418 }
419 }
420
HELPER(sve2_sqrdmulh_h)421 void HELPER(sve2_sqrdmulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
422 {
423 intptr_t i, opr_sz = simd_oprsz(desc);
424 int16_t *d = vd, *n = vn, *m = vm;
425 uint32_t discard;
426
427 for (i = 0; i < opr_sz / 2; ++i) {
428 d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, &discard);
429 }
430 }
431
HELPER(sve2_sqdmulh_idx_h)432 void HELPER(sve2_sqdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
433 {
434 intptr_t i, j, opr_sz = simd_oprsz(desc);
435 int idx = simd_data(desc);
436 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
437 uint32_t discard;
438
439 for (i = 0; i < opr_sz / 2; i += 16 / 2) {
440 int16_t mm = m[i];
441 for (j = 0; j < 16 / 2; ++j) {
442 d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, &discard);
443 }
444 }
445 }
446
HELPER(sve2_sqrdmulh_idx_h)447 void HELPER(sve2_sqrdmulh_idx_h)(void *vd, void *vn, void *vm, uint32_t desc)
448 {
449 intptr_t i, j, opr_sz = simd_oprsz(desc);
450 int idx = simd_data(desc);
451 int16_t *d = vd, *n = vn, *m = (int16_t *)vm + H2(idx);
452 uint32_t discard;
453
454 for (i = 0; i < opr_sz / 2; i += 16 / 2) {
455 int16_t mm = m[i];
456 for (j = 0; j < 16 / 2; ++j) {
457 d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, &discard);
458 }
459 }
460 }
461
462 /* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
do_sqrdmlah_s(int32_t src1,int32_t src2,int32_t src3,bool neg,bool round,uint32_t * sat)463 int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
464 bool neg, bool round, uint32_t *sat)
465 {
466 /* Simplify similarly to do_sqrdmlah_b above. */
467 int64_t ret = (int64_t)src1 * src2;
468 if (neg) {
469 ret = -ret;
470 }
471 ret += ((int64_t)src3 << 31) + (round << 30);
472 ret >>= 31;
473
474 if (ret != (int32_t)ret) {
475 *sat = 1;
476 ret = (ret < 0 ? INT32_MIN : INT32_MAX);
477 }
478 return ret;
479 }
480
HELPER(neon_qrdmlah_s32)481 uint32_t HELPER(neon_qrdmlah_s32)(CPUARMState *env, int32_t src1,
482 int32_t src2, int32_t src3)
483 {
484 uint32_t *sat = &env->vfp.qc[0];
485 return do_sqrdmlah_s(src1, src2, src3, false, true, sat);
486 }
487
HELPER(gvec_qrdmlah_s32)488 void HELPER(gvec_qrdmlah_s32)(void *vd, void *vn, void *vm,
489 void *vq, uint32_t desc)
490 {
491 uintptr_t opr_sz = simd_oprsz(desc);
492 int32_t *d = vd;
493 int32_t *n = vn;
494 int32_t *m = vm;
495 uintptr_t i;
496
497 for (i = 0; i < opr_sz / 4; ++i) {
498 d[i] = do_sqrdmlah_s(n[i], m[i], d[i], false, true, vq);
499 }
500 clear_tail(d, opr_sz, simd_maxsz(desc));
501 }
502
HELPER(neon_qrdmlsh_s32)503 uint32_t HELPER(neon_qrdmlsh_s32)(CPUARMState *env, int32_t src1,
504 int32_t src2, int32_t src3)
505 {
506 uint32_t *sat = &env->vfp.qc[0];
507 return do_sqrdmlah_s(src1, src2, src3, true, true, sat);
508 }
509
HELPER(gvec_qrdmlsh_s32)510 void HELPER(gvec_qrdmlsh_s32)(void *vd, void *vn, void *vm,
511 void *vq, uint32_t desc)
512 {
513 uintptr_t opr_sz = simd_oprsz(desc);
514 int32_t *d = vd;
515 int32_t *n = vn;
516 int32_t *m = vm;
517 uintptr_t i;
518
519 for (i = 0; i < opr_sz / 4; ++i) {
520 d[i] = do_sqrdmlah_s(n[i], m[i], d[i], true, true, vq);
521 }
522 clear_tail(d, opr_sz, simd_maxsz(desc));
523 }
524
HELPER(neon_sqdmulh_s)525 void HELPER(neon_sqdmulh_s)(void *vd, void *vn, void *vm,
526 void *vq, uint32_t desc)
527 {
528 intptr_t i, opr_sz = simd_oprsz(desc);
529 int32_t *d = vd, *n = vn, *m = vm;
530
531 for (i = 0; i < opr_sz / 4; ++i) {
532 d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, vq);
533 }
534 clear_tail(d, opr_sz, simd_maxsz(desc));
535 }
536
HELPER(neon_sqrdmulh_s)537 void HELPER(neon_sqrdmulh_s)(void *vd, void *vn, void *vm,
538 void *vq, uint32_t desc)
539 {
540 intptr_t i, opr_sz = simd_oprsz(desc);
541 int32_t *d = vd, *n = vn, *m = vm;
542
543 for (i = 0; i < opr_sz / 4; ++i) {
544 d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, vq);
545 }
546 clear_tail(d, opr_sz, simd_maxsz(desc));
547 }
548
HELPER(neon_sqdmulh_idx_s)549 void HELPER(neon_sqdmulh_idx_s)(void *vd, void *vn, void *vm,
550 void *vq, uint32_t desc)
551 {
552 intptr_t i, j, opr_sz = simd_oprsz(desc);
553 int idx = simd_data(desc);
554 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
555 intptr_t elements = opr_sz / 4;
556 intptr_t eltspersegment = MIN(16 / 4, elements);
557
558 for (i = 0; i < elements; i += 16 / 4) {
559 int32_t mm = m[i];
560 for (j = 0; j < eltspersegment; ++j) {
561 d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, vq);
562 }
563 }
564 clear_tail(d, opr_sz, simd_maxsz(desc));
565 }
566
HELPER(neon_sqrdmulh_idx_s)567 void HELPER(neon_sqrdmulh_idx_s)(void *vd, void *vn, void *vm,
568 void *vq, uint32_t desc)
569 {
570 intptr_t i, j, opr_sz = simd_oprsz(desc);
571 int idx = simd_data(desc);
572 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
573 intptr_t elements = opr_sz / 4;
574 intptr_t eltspersegment = MIN(16 / 4, elements);
575
576 for (i = 0; i < elements; i += 16 / 4) {
577 int32_t mm = m[i];
578 for (j = 0; j < eltspersegment; ++j) {
579 d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, vq);
580 }
581 }
582 clear_tail(d, opr_sz, simd_maxsz(desc));
583 }
584
HELPER(neon_sqrdmlah_idx_s)585 void HELPER(neon_sqrdmlah_idx_s)(void *vd, void *vn, void *vm,
586 void *vq, uint32_t desc)
587 {
588 intptr_t i, j, opr_sz = simd_oprsz(desc);
589 int idx = simd_data(desc);
590 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
591 intptr_t elements = opr_sz / 4;
592 intptr_t eltspersegment = MIN(16 / 4, elements);
593
594 for (i = 0; i < elements; i += 16 / 4) {
595 int32_t mm = m[i];
596 for (j = 0; j < eltspersegment; ++j) {
597 d[i + j] = do_sqrdmlah_s(n[i + j], mm, d[i + j], false, true, vq);
598 }
599 }
600 clear_tail(d, opr_sz, simd_maxsz(desc));
601 }
602
HELPER(neon_sqrdmlsh_idx_s)603 void HELPER(neon_sqrdmlsh_idx_s)(void *vd, void *vn, void *vm,
604 void *vq, uint32_t desc)
605 {
606 intptr_t i, j, opr_sz = simd_oprsz(desc);
607 int idx = simd_data(desc);
608 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
609 intptr_t elements = opr_sz / 4;
610 intptr_t eltspersegment = MIN(16 / 4, elements);
611
612 for (i = 0; i < elements; i += 16 / 4) {
613 int32_t mm = m[i];
614 for (j = 0; j < eltspersegment; ++j) {
615 d[i + j] = do_sqrdmlah_s(n[i + j], mm, d[i + j], true, true, vq);
616 }
617 }
618 clear_tail(d, opr_sz, simd_maxsz(desc));
619 }
620
HELPER(sve2_sqrdmlah_s)621 void HELPER(sve2_sqrdmlah_s)(void *vd, void *vn, void *vm,
622 void *va, uint32_t desc)
623 {
624 intptr_t i, opr_sz = simd_oprsz(desc);
625 int32_t *d = vd, *n = vn, *m = vm, *a = va;
626 uint32_t discard;
627
628 for (i = 0; i < opr_sz / 4; ++i) {
629 d[i] = do_sqrdmlah_s(n[i], m[i], a[i], false, true, &discard);
630 }
631 }
632
HELPER(sve2_sqrdmlsh_s)633 void HELPER(sve2_sqrdmlsh_s)(void *vd, void *vn, void *vm,
634 void *va, uint32_t desc)
635 {
636 intptr_t i, opr_sz = simd_oprsz(desc);
637 int32_t *d = vd, *n = vn, *m = vm, *a = va;
638 uint32_t discard;
639
640 for (i = 0; i < opr_sz / 4; ++i) {
641 d[i] = do_sqrdmlah_s(n[i], m[i], a[i], true, true, &discard);
642 }
643 }
644
HELPER(sve2_sqdmulh_s)645 void HELPER(sve2_sqdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
646 {
647 intptr_t i, opr_sz = simd_oprsz(desc);
648 int32_t *d = vd, *n = vn, *m = vm;
649 uint32_t discard;
650
651 for (i = 0; i < opr_sz / 4; ++i) {
652 d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, &discard);
653 }
654 }
655
HELPER(sve2_sqrdmulh_s)656 void HELPER(sve2_sqrdmulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
657 {
658 intptr_t i, opr_sz = simd_oprsz(desc);
659 int32_t *d = vd, *n = vn, *m = vm;
660 uint32_t discard;
661
662 for (i = 0; i < opr_sz / 4; ++i) {
663 d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, &discard);
664 }
665 }
666
HELPER(sve2_sqdmulh_idx_s)667 void HELPER(sve2_sqdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc)
668 {
669 intptr_t i, j, opr_sz = simd_oprsz(desc);
670 int idx = simd_data(desc);
671 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
672 uint32_t discard;
673
674 for (i = 0; i < opr_sz / 4; i += 16 / 4) {
675 int32_t mm = m[i];
676 for (j = 0; j < 16 / 4; ++j) {
677 d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, &discard);
678 }
679 }
680 }
681
HELPER(sve2_sqrdmulh_idx_s)682 void HELPER(sve2_sqrdmulh_idx_s)(void *vd, void *vn, void *vm, uint32_t desc)
683 {
684 intptr_t i, j, opr_sz = simd_oprsz(desc);
685 int idx = simd_data(desc);
686 int32_t *d = vd, *n = vn, *m = (int32_t *)vm + H4(idx);
687 uint32_t discard;
688
689 for (i = 0; i < opr_sz / 4; i += 16 / 4) {
690 int32_t mm = m[i];
691 for (j = 0; j < 16 / 4; ++j) {
692 d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, &discard);
693 }
694 }
695 }
696
697 /* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */
do_sat128_d(Int128 r)698 static int64_t do_sat128_d(Int128 r)
699 {
700 int64_t ls = int128_getlo(r);
701 int64_t hs = int128_gethi(r);
702
703 if (unlikely(hs != (ls >> 63))) {
704 return hs < 0 ? INT64_MIN : INT64_MAX;
705 }
706 return ls;
707 }
708
do_sqrdmlah_d(int64_t n,int64_t m,int64_t a,bool neg,bool round)709 int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, bool neg, bool round)
710 {
711 uint64_t l, h;
712 Int128 r, t;
713
714 /* As in do_sqrdmlah_b, but with 128-bit arithmetic. */
715 muls64(&l, &h, m, n);
716 r = int128_make128(l, h);
717 if (neg) {
718 r = int128_neg(r);
719 }
720 if (a) {
721 t = int128_exts64(a);
722 t = int128_lshift(t, 63);
723 r = int128_add(r, t);
724 }
725 if (round) {
726 t = int128_exts64(1ll << 62);
727 r = int128_add(r, t);
728 }
729 r = int128_rshift(r, 63);
730
731 return do_sat128_d(r);
732 }
733
HELPER(sve2_sqrdmlah_d)734 void HELPER(sve2_sqrdmlah_d)(void *vd, void *vn, void *vm,
735 void *va, uint32_t desc)
736 {
737 intptr_t i, opr_sz = simd_oprsz(desc);
738 int64_t *d = vd, *n = vn, *m = vm, *a = va;
739
740 for (i = 0; i < opr_sz / 8; ++i) {
741 d[i] = do_sqrdmlah_d(n[i], m[i], a[i], false, true);
742 }
743 }
744
HELPER(sve2_sqrdmlsh_d)745 void HELPER(sve2_sqrdmlsh_d)(void *vd, void *vn, void *vm,
746 void *va, uint32_t desc)
747 {
748 intptr_t i, opr_sz = simd_oprsz(desc);
749 int64_t *d = vd, *n = vn, *m = vm, *a = va;
750
751 for (i = 0; i < opr_sz / 8; ++i) {
752 d[i] = do_sqrdmlah_d(n[i], m[i], a[i], true, true);
753 }
754 }
755
HELPER(sve2_sqdmulh_d)756 void HELPER(sve2_sqdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
757 {
758 intptr_t i, opr_sz = simd_oprsz(desc);
759 int64_t *d = vd, *n = vn, *m = vm;
760
761 for (i = 0; i < opr_sz / 8; ++i) {
762 d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, false);
763 }
764 }
765
HELPER(sve2_sqrdmulh_d)766 void HELPER(sve2_sqrdmulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
767 {
768 intptr_t i, opr_sz = simd_oprsz(desc);
769 int64_t *d = vd, *n = vn, *m = vm;
770
771 for (i = 0; i < opr_sz / 8; ++i) {
772 d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, true);
773 }
774 }
775
HELPER(sve2_sqdmulh_idx_d)776 void HELPER(sve2_sqdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc)
777 {
778 intptr_t i, j, opr_sz = simd_oprsz(desc);
779 int idx = simd_data(desc);
780 int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx;
781
782 for (i = 0; i < opr_sz / 8; i += 16 / 8) {
783 int64_t mm = m[i];
784 for (j = 0; j < 16 / 8; ++j) {
785 d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, false);
786 }
787 }
788 }
789
HELPER(sve2_sqrdmulh_idx_d)790 void HELPER(sve2_sqrdmulh_idx_d)(void *vd, void *vn, void *vm, uint32_t desc)
791 {
792 intptr_t i, j, opr_sz = simd_oprsz(desc);
793 int idx = simd_data(desc);
794 int64_t *d = vd, *n = vn, *m = (int64_t *)vm + idx;
795
796 for (i = 0; i < opr_sz / 8; i += 16 / 8) {
797 int64_t mm = m[i];
798 for (j = 0; j < 16 / 8; ++j) {
799 d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, true);
800 }
801 }
802 }
803
804 /* Integer 8 and 16-bit dot-product.
805 *
806 * Note that for the loops herein, host endianness does not matter
807 * with respect to the ordering of data within the quad-width lanes.
808 * All elements are treated equally, no matter where they are.
809 */
810
811 #define DO_DOT(NAME, TYPED, TYPEN, TYPEM) \
812 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
813 { \
814 intptr_t i, opr_sz = simd_oprsz(desc); \
815 TYPED *d = vd, *a = va; \
816 TYPEN *n = vn; \
817 TYPEM *m = vm; \
818 for (i = 0; i < opr_sz / sizeof(TYPED); ++i) { \
819 d[i] = (a[i] + \
820 (TYPED)n[i * 4 + 0] * m[i * 4 + 0] + \
821 (TYPED)n[i * 4 + 1] * m[i * 4 + 1] + \
822 (TYPED)n[i * 4 + 2] * m[i * 4 + 2] + \
823 (TYPED)n[i * 4 + 3] * m[i * 4 + 3]); \
824 } \
825 clear_tail(d, opr_sz, simd_maxsz(desc)); \
826 }
827
DO_DOT(gvec_sdot_b,int32_t,int8_t,int8_t)828 DO_DOT(gvec_sdot_b, int32_t, int8_t, int8_t)
829 DO_DOT(gvec_udot_b, uint32_t, uint8_t, uint8_t)
830 DO_DOT(gvec_usdot_b, uint32_t, uint8_t, int8_t)
831 DO_DOT(gvec_sdot_h, int64_t, int16_t, int16_t)
832 DO_DOT(gvec_udot_h, uint64_t, uint16_t, uint16_t)
833
834 #define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, HD) \
835 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
836 { \
837 intptr_t i = 0, opr_sz = simd_oprsz(desc); \
838 intptr_t opr_sz_n = opr_sz / sizeof(TYPED); \
839 intptr_t segend = MIN(16 / sizeof(TYPED), opr_sz_n); \
840 intptr_t index = simd_data(desc); \
841 TYPED *d = vd, *a = va; \
842 TYPEN *n = vn; \
843 TYPEM *m_indexed = (TYPEM *)vm + HD(index) * 4; \
844 do { \
845 TYPED m0 = m_indexed[i * 4 + 0]; \
846 TYPED m1 = m_indexed[i * 4 + 1]; \
847 TYPED m2 = m_indexed[i * 4 + 2]; \
848 TYPED m3 = m_indexed[i * 4 + 3]; \
849 do { \
850 d[i] = (a[i] + \
851 n[i * 4 + 0] * m0 + \
852 n[i * 4 + 1] * m1 + \
853 n[i * 4 + 2] * m2 + \
854 n[i * 4 + 3] * m3); \
855 } while (++i < segend); \
856 segend = i + 4; \
857 } while (i < opr_sz_n); \
858 clear_tail(d, opr_sz, simd_maxsz(desc)); \
859 }
860
861 DO_DOT_IDX(gvec_sdot_idx_b, int32_t, int8_t, int8_t, H4)
862 DO_DOT_IDX(gvec_udot_idx_b, uint32_t, uint8_t, uint8_t, H4)
863 DO_DOT_IDX(gvec_sudot_idx_b, int32_t, int8_t, uint8_t, H4)
864 DO_DOT_IDX(gvec_usdot_idx_b, int32_t, uint8_t, int8_t, H4)
865 DO_DOT_IDX(gvec_sdot_idx_h, int64_t, int16_t, int16_t, H8)
866 DO_DOT_IDX(gvec_udot_idx_h, uint64_t, uint16_t, uint16_t, H8)
867
868 void HELPER(gvec_fcaddh)(void *vd, void *vn, void *vm,
869 void *vfpst, uint32_t desc)
870 {
871 uintptr_t opr_sz = simd_oprsz(desc);
872 float16 *d = vd;
873 float16 *n = vn;
874 float16 *m = vm;
875 float_status *fpst = vfpst;
876 uint32_t neg_real = extract32(desc, SIMD_DATA_SHIFT, 1);
877 uint32_t neg_imag = neg_real ^ 1;
878 uintptr_t i;
879
880 /* Shift boolean to the sign bit so we can xor to negate. */
881 neg_real <<= 15;
882 neg_imag <<= 15;
883
884 for (i = 0; i < opr_sz / 2; i += 2) {
885 float16 e0 = n[H2(i)];
886 float16 e1 = m[H2(i + 1)] ^ neg_imag;
887 float16 e2 = n[H2(i + 1)];
888 float16 e3 = m[H2(i)] ^ neg_real;
889
890 d[H2(i)] = float16_add(e0, e1, fpst);
891 d[H2(i + 1)] = float16_add(e2, e3, fpst);
892 }
893 clear_tail(d, opr_sz, simd_maxsz(desc));
894 }
895
HELPER(gvec_fcadds)896 void HELPER(gvec_fcadds)(void *vd, void *vn, void *vm,
897 void *vfpst, uint32_t desc)
898 {
899 uintptr_t opr_sz = simd_oprsz(desc);
900 float32 *d = vd;
901 float32 *n = vn;
902 float32 *m = vm;
903 float_status *fpst = vfpst;
904 uint32_t neg_real = extract32(desc, SIMD_DATA_SHIFT, 1);
905 uint32_t neg_imag = neg_real ^ 1;
906 uintptr_t i;
907
908 /* Shift boolean to the sign bit so we can xor to negate. */
909 neg_real <<= 31;
910 neg_imag <<= 31;
911
912 for (i = 0; i < opr_sz / 4; i += 2) {
913 float32 e0 = n[H4(i)];
914 float32 e1 = m[H4(i + 1)] ^ neg_imag;
915 float32 e2 = n[H4(i + 1)];
916 float32 e3 = m[H4(i)] ^ neg_real;
917
918 d[H4(i)] = float32_add(e0, e1, fpst);
919 d[H4(i + 1)] = float32_add(e2, e3, fpst);
920 }
921 clear_tail(d, opr_sz, simd_maxsz(desc));
922 }
923
HELPER(gvec_fcaddd)924 void HELPER(gvec_fcaddd)(void *vd, void *vn, void *vm,
925 void *vfpst, uint32_t desc)
926 {
927 uintptr_t opr_sz = simd_oprsz(desc);
928 float64 *d = vd;
929 float64 *n = vn;
930 float64 *m = vm;
931 float_status *fpst = vfpst;
932 uint64_t neg_real = extract64(desc, SIMD_DATA_SHIFT, 1);
933 uint64_t neg_imag = neg_real ^ 1;
934 uintptr_t i;
935
936 /* Shift boolean to the sign bit so we can xor to negate. */
937 neg_real <<= 63;
938 neg_imag <<= 63;
939
940 for (i = 0; i < opr_sz / 8; i += 2) {
941 float64 e0 = n[i];
942 float64 e1 = m[i + 1] ^ neg_imag;
943 float64 e2 = n[i + 1];
944 float64 e3 = m[i] ^ neg_real;
945
946 d[i] = float64_add(e0, e1, fpst);
947 d[i + 1] = float64_add(e2, e3, fpst);
948 }
949 clear_tail(d, opr_sz, simd_maxsz(desc));
950 }
951
HELPER(gvec_fcmlah)952 void HELPER(gvec_fcmlah)(void *vd, void *vn, void *vm, void *va,
953 void *vfpst, uint32_t desc)
954 {
955 uintptr_t opr_sz = simd_oprsz(desc);
956 float16 *d = vd, *n = vn, *m = vm, *a = va;
957 float_status *fpst = vfpst;
958 intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
959 uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
960 uint32_t neg_real = flip ^ neg_imag;
961 uintptr_t i;
962
963 /* Shift boolean to the sign bit so we can xor to negate. */
964 neg_real <<= 15;
965 neg_imag <<= 15;
966
967 for (i = 0; i < opr_sz / 2; i += 2) {
968 float16 e2 = n[H2(i + flip)];
969 float16 e1 = m[H2(i + flip)] ^ neg_real;
970 float16 e4 = e2;
971 float16 e3 = m[H2(i + 1 - flip)] ^ neg_imag;
972
973 d[H2(i)] = float16_muladd(e2, e1, a[H2(i)], 0, fpst);
974 d[H2(i + 1)] = float16_muladd(e4, e3, a[H2(i + 1)], 0, fpst);
975 }
976 clear_tail(d, opr_sz, simd_maxsz(desc));
977 }
978
HELPER(gvec_fcmlah_idx)979 void HELPER(gvec_fcmlah_idx)(void *vd, void *vn, void *vm, void *va,
980 void *vfpst, uint32_t desc)
981 {
982 uintptr_t opr_sz = simd_oprsz(desc);
983 float16 *d = vd, *n = vn, *m = vm, *a = va;
984 float_status *fpst = vfpst;
985 intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
986 uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
987 intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
988 uint32_t neg_real = flip ^ neg_imag;
989 intptr_t elements = opr_sz / sizeof(float16);
990 intptr_t eltspersegment = MIN(16 / sizeof(float16), elements);
991 intptr_t i, j;
992
993 /* Shift boolean to the sign bit so we can xor to negate. */
994 neg_real <<= 15;
995 neg_imag <<= 15;
996
997 for (i = 0; i < elements; i += eltspersegment) {
998 float16 mr = m[H2(i + 2 * index + 0)];
999 float16 mi = m[H2(i + 2 * index + 1)];
1000 float16 e1 = neg_real ^ (flip ? mi : mr);
1001 float16 e3 = neg_imag ^ (flip ? mr : mi);
1002
1003 for (j = i; j < i + eltspersegment; j += 2) {
1004 float16 e2 = n[H2(j + flip)];
1005 float16 e4 = e2;
1006
1007 d[H2(j)] = float16_muladd(e2, e1, a[H2(j)], 0, fpst);
1008 d[H2(j + 1)] = float16_muladd(e4, e3, a[H2(j + 1)], 0, fpst);
1009 }
1010 }
1011 clear_tail(d, opr_sz, simd_maxsz(desc));
1012 }
1013
HELPER(gvec_fcmlas)1014 void HELPER(gvec_fcmlas)(void *vd, void *vn, void *vm, void *va,
1015 void *vfpst, uint32_t desc)
1016 {
1017 uintptr_t opr_sz = simd_oprsz(desc);
1018 float32 *d = vd, *n = vn, *m = vm, *a = va;
1019 float_status *fpst = vfpst;
1020 intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1021 uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1022 uint32_t neg_real = flip ^ neg_imag;
1023 uintptr_t i;
1024
1025 /* Shift boolean to the sign bit so we can xor to negate. */
1026 neg_real <<= 31;
1027 neg_imag <<= 31;
1028
1029 for (i = 0; i < opr_sz / 4; i += 2) {
1030 float32 e2 = n[H4(i + flip)];
1031 float32 e1 = m[H4(i + flip)] ^ neg_real;
1032 float32 e4 = e2;
1033 float32 e3 = m[H4(i + 1 - flip)] ^ neg_imag;
1034
1035 d[H4(i)] = float32_muladd(e2, e1, a[H4(i)], 0, fpst);
1036 d[H4(i + 1)] = float32_muladd(e4, e3, a[H4(i + 1)], 0, fpst);
1037 }
1038 clear_tail(d, opr_sz, simd_maxsz(desc));
1039 }
1040
HELPER(gvec_fcmlas_idx)1041 void HELPER(gvec_fcmlas_idx)(void *vd, void *vn, void *vm, void *va,
1042 void *vfpst, uint32_t desc)
1043 {
1044 uintptr_t opr_sz = simd_oprsz(desc);
1045 float32 *d = vd, *n = vn, *m = vm, *a = va;
1046 float_status *fpst = vfpst;
1047 intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1048 uint32_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1049 intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
1050 uint32_t neg_real = flip ^ neg_imag;
1051 intptr_t elements = opr_sz / sizeof(float32);
1052 intptr_t eltspersegment = MIN(16 / sizeof(float32), elements);
1053 intptr_t i, j;
1054
1055 /* Shift boolean to the sign bit so we can xor to negate. */
1056 neg_real <<= 31;
1057 neg_imag <<= 31;
1058
1059 for (i = 0; i < elements; i += eltspersegment) {
1060 float32 mr = m[H4(i + 2 * index + 0)];
1061 float32 mi = m[H4(i + 2 * index + 1)];
1062 float32 e1 = neg_real ^ (flip ? mi : mr);
1063 float32 e3 = neg_imag ^ (flip ? mr : mi);
1064
1065 for (j = i; j < i + eltspersegment; j += 2) {
1066 float32 e2 = n[H4(j + flip)];
1067 float32 e4 = e2;
1068
1069 d[H4(j)] = float32_muladd(e2, e1, a[H4(j)], 0, fpst);
1070 d[H4(j + 1)] = float32_muladd(e4, e3, a[H4(j + 1)], 0, fpst);
1071 }
1072 }
1073 clear_tail(d, opr_sz, simd_maxsz(desc));
1074 }
1075
HELPER(gvec_fcmlad)1076 void HELPER(gvec_fcmlad)(void *vd, void *vn, void *vm, void *va,
1077 void *vfpst, uint32_t desc)
1078 {
1079 uintptr_t opr_sz = simd_oprsz(desc);
1080 float64 *d = vd, *n = vn, *m = vm, *a = va;
1081 float_status *fpst = vfpst;
1082 intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1083 uint64_t neg_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1084 uint64_t neg_real = flip ^ neg_imag;
1085 uintptr_t i;
1086
1087 /* Shift boolean to the sign bit so we can xor to negate. */
1088 neg_real <<= 63;
1089 neg_imag <<= 63;
1090
1091 for (i = 0; i < opr_sz / 8; i += 2) {
1092 float64 e2 = n[i + flip];
1093 float64 e1 = m[i + flip] ^ neg_real;
1094 float64 e4 = e2;
1095 float64 e3 = m[i + 1 - flip] ^ neg_imag;
1096
1097 d[i] = float64_muladd(e2, e1, a[i], 0, fpst);
1098 d[i + 1] = float64_muladd(e4, e3, a[i + 1], 0, fpst);
1099 }
1100 clear_tail(d, opr_sz, simd_maxsz(desc));
1101 }
1102
1103 /*
1104 * Floating point comparisons producing an integer result (all 1s or all 0s).
1105 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1106 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1107 */
float16_ceq(float16 op1,float16 op2,float_status * stat)1108 static uint16_t float16_ceq(float16 op1, float16 op2, float_status *stat)
1109 {
1110 return -float16_eq_quiet(op1, op2, stat);
1111 }
1112
float32_ceq(float32 op1,float32 op2,float_status * stat)1113 static uint32_t float32_ceq(float32 op1, float32 op2, float_status *stat)
1114 {
1115 return -float32_eq_quiet(op1, op2, stat);
1116 }
1117
float64_ceq(float64 op1,float64 op2,float_status * stat)1118 static uint64_t float64_ceq(float64 op1, float64 op2, float_status *stat)
1119 {
1120 return -float64_eq_quiet(op1, op2, stat);
1121 }
1122
float16_cge(float16 op1,float16 op2,float_status * stat)1123 static uint16_t float16_cge(float16 op1, float16 op2, float_status *stat)
1124 {
1125 return -float16_le(op2, op1, stat);
1126 }
1127
float32_cge(float32 op1,float32 op2,float_status * stat)1128 static uint32_t float32_cge(float32 op1, float32 op2, float_status *stat)
1129 {
1130 return -float32_le(op2, op1, stat);
1131 }
1132
float64_cge(float64 op1,float64 op2,float_status * stat)1133 static uint64_t float64_cge(float64 op1, float64 op2, float_status *stat)
1134 {
1135 return -float64_le(op2, op1, stat);
1136 }
1137
float16_cgt(float16 op1,float16 op2,float_status * stat)1138 static uint16_t float16_cgt(float16 op1, float16 op2, float_status *stat)
1139 {
1140 return -float16_lt(op2, op1, stat);
1141 }
1142
float32_cgt(float32 op1,float32 op2,float_status * stat)1143 static uint32_t float32_cgt(float32 op1, float32 op2, float_status *stat)
1144 {
1145 return -float32_lt(op2, op1, stat);
1146 }
1147
float64_cgt(float64 op1,float64 op2,float_status * stat)1148 static uint64_t float64_cgt(float64 op1, float64 op2, float_status *stat)
1149 {
1150 return -float64_lt(op2, op1, stat);
1151 }
1152
float16_acge(float16 op1,float16 op2,float_status * stat)1153 static uint16_t float16_acge(float16 op1, float16 op2, float_status *stat)
1154 {
1155 return -float16_le(float16_abs(op2), float16_abs(op1), stat);
1156 }
1157
float32_acge(float32 op1,float32 op2,float_status * stat)1158 static uint32_t float32_acge(float32 op1, float32 op2, float_status *stat)
1159 {
1160 return -float32_le(float32_abs(op2), float32_abs(op1), stat);
1161 }
1162
float64_acge(float64 op1,float64 op2,float_status * stat)1163 static uint64_t float64_acge(float64 op1, float64 op2, float_status *stat)
1164 {
1165 return -float64_le(float64_abs(op2), float64_abs(op1), stat);
1166 }
1167
float16_acgt(float16 op1,float16 op2,float_status * stat)1168 static uint16_t float16_acgt(float16 op1, float16 op2, float_status *stat)
1169 {
1170 return -float16_lt(float16_abs(op2), float16_abs(op1), stat);
1171 }
1172
float32_acgt(float32 op1,float32 op2,float_status * stat)1173 static uint32_t float32_acgt(float32 op1, float32 op2, float_status *stat)
1174 {
1175 return -float32_lt(float32_abs(op2), float32_abs(op1), stat);
1176 }
1177
float64_acgt(float64 op1,float64 op2,float_status * stat)1178 static uint64_t float64_acgt(float64 op1, float64 op2, float_status *stat)
1179 {
1180 return -float64_lt(float64_abs(op2), float64_abs(op1), stat);
1181 }
1182
vfp_tosszh(float16 x,void * fpstp)1183 static int16_t vfp_tosszh(float16 x, void *fpstp)
1184 {
1185 float_status *fpst = fpstp;
1186 if (float16_is_any_nan(x)) {
1187 float_raise(float_flag_invalid, fpst);
1188 return 0;
1189 }
1190 return float16_to_int16_round_to_zero(x, fpst);
1191 }
1192
vfp_touszh(float16 x,void * fpstp)1193 static uint16_t vfp_touszh(float16 x, void *fpstp)
1194 {
1195 float_status *fpst = fpstp;
1196 if (float16_is_any_nan(x)) {
1197 float_raise(float_flag_invalid, fpst);
1198 return 0;
1199 }
1200 return float16_to_uint16_round_to_zero(x, fpst);
1201 }
1202
1203 #define DO_2OP(NAME, FUNC, TYPE) \
1204 void HELPER(NAME)(void *vd, void *vn, void *stat, uint32_t desc) \
1205 { \
1206 intptr_t i, oprsz = simd_oprsz(desc); \
1207 TYPE *d = vd, *n = vn; \
1208 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1209 d[i] = FUNC(n[i], stat); \
1210 } \
1211 clear_tail(d, oprsz, simd_maxsz(desc)); \
1212 }
1213
DO_2OP(gvec_frecpe_h,helper_recpe_f16,float16)1214 DO_2OP(gvec_frecpe_h, helper_recpe_f16, float16)
1215 DO_2OP(gvec_frecpe_s, helper_recpe_f32, float32)
1216 DO_2OP(gvec_frecpe_d, helper_recpe_f64, float64)
1217
1218 DO_2OP(gvec_frsqrte_h, helper_rsqrte_f16, float16)
1219 DO_2OP(gvec_frsqrte_s, helper_rsqrte_f32, float32)
1220 DO_2OP(gvec_frsqrte_d, helper_rsqrte_f64, float64)
1221
1222 DO_2OP(gvec_vrintx_h, float16_round_to_int, float16)
1223 DO_2OP(gvec_vrintx_s, float32_round_to_int, float32)
1224
1225 DO_2OP(gvec_sitos, helper_vfp_sitos, int32_t)
1226 DO_2OP(gvec_uitos, helper_vfp_uitos, uint32_t)
1227 DO_2OP(gvec_tosizs, helper_vfp_tosizs, float32)
1228 DO_2OP(gvec_touizs, helper_vfp_touizs, float32)
1229 DO_2OP(gvec_sstoh, int16_to_float16, int16_t)
1230 DO_2OP(gvec_ustoh, uint16_to_float16, uint16_t)
1231 DO_2OP(gvec_tosszh, vfp_tosszh, float16)
1232 DO_2OP(gvec_touszh, vfp_touszh, float16)
1233
1234 #define WRAP_CMP0_FWD(FN, CMPOP, TYPE) \
1235 static TYPE TYPE##_##FN##0(TYPE op, float_status *stat) \
1236 { \
1237 return TYPE##_##CMPOP(op, TYPE##_zero, stat); \
1238 }
1239
1240 #define WRAP_CMP0_REV(FN, CMPOP, TYPE) \
1241 static TYPE TYPE##_##FN##0(TYPE op, float_status *stat) \
1242 { \
1243 return TYPE##_##CMPOP(TYPE##_zero, op, stat); \
1244 }
1245
1246 #define DO_2OP_CMP0(FN, CMPOP, DIRN) \
1247 WRAP_CMP0_##DIRN(FN, CMPOP, float16) \
1248 WRAP_CMP0_##DIRN(FN, CMPOP, float32) \
1249 DO_2OP(gvec_f##FN##0_h, float16_##FN##0, float16) \
1250 DO_2OP(gvec_f##FN##0_s, float32_##FN##0, float32)
1251
1252 DO_2OP_CMP0(cgt, cgt, FWD)
1253 DO_2OP_CMP0(cge, cge, FWD)
1254 DO_2OP_CMP0(ceq, ceq, FWD)
1255 DO_2OP_CMP0(clt, cgt, REV)
1256 DO_2OP_CMP0(cle, cge, REV)
1257
1258 #undef DO_2OP
1259 #undef DO_2OP_CMP0
1260
1261 /* Floating-point trigonometric starting value.
1262 * See the ARM ARM pseudocode function FPTrigSMul.
1263 */
1264 static float16 float16_ftsmul(float16 op1, uint16_t op2, float_status *stat)
1265 {
1266 float16 result = float16_mul(op1, op1, stat);
1267 if (!float16_is_any_nan(result)) {
1268 result = float16_set_sign(result, op2 & 1);
1269 }
1270 return result;
1271 }
1272
float32_ftsmul(float32 op1,uint32_t op2,float_status * stat)1273 static float32 float32_ftsmul(float32 op1, uint32_t op2, float_status *stat)
1274 {
1275 float32 result = float32_mul(op1, op1, stat);
1276 if (!float32_is_any_nan(result)) {
1277 result = float32_set_sign(result, op2 & 1);
1278 }
1279 return result;
1280 }
1281
float64_ftsmul(float64 op1,uint64_t op2,float_status * stat)1282 static float64 float64_ftsmul(float64 op1, uint64_t op2, float_status *stat)
1283 {
1284 float64 result = float64_mul(op1, op1, stat);
1285 if (!float64_is_any_nan(result)) {
1286 result = float64_set_sign(result, op2 & 1);
1287 }
1288 return result;
1289 }
1290
float16_abd(float16 op1,float16 op2,float_status * stat)1291 static float16 float16_abd(float16 op1, float16 op2, float_status *stat)
1292 {
1293 return float16_abs(float16_sub(op1, op2, stat));
1294 }
1295
float32_abd(float32 op1,float32 op2,float_status * stat)1296 static float32 float32_abd(float32 op1, float32 op2, float_status *stat)
1297 {
1298 return float32_abs(float32_sub(op1, op2, stat));
1299 }
1300
float64_abd(float64 op1,float64 op2,float_status * stat)1301 static float64 float64_abd(float64 op1, float64 op2, float_status *stat)
1302 {
1303 return float64_abs(float64_sub(op1, op2, stat));
1304 }
1305
1306 /*
1307 * Reciprocal step. These are the AArch32 version which uses a
1308 * non-fused multiply-and-subtract.
1309 */
float16_recps_nf(float16 op1,float16 op2,float_status * stat)1310 static float16 float16_recps_nf(float16 op1, float16 op2, float_status *stat)
1311 {
1312 op1 = float16_squash_input_denormal(op1, stat);
1313 op2 = float16_squash_input_denormal(op2, stat);
1314
1315 if ((float16_is_infinity(op1) && float16_is_zero(op2)) ||
1316 (float16_is_infinity(op2) && float16_is_zero(op1))) {
1317 return float16_two;
1318 }
1319 return float16_sub(float16_two, float16_mul(op1, op2, stat), stat);
1320 }
1321
float32_recps_nf(float32 op1,float32 op2,float_status * stat)1322 static float32 float32_recps_nf(float32 op1, float32 op2, float_status *stat)
1323 {
1324 op1 = float32_squash_input_denormal(op1, stat);
1325 op2 = float32_squash_input_denormal(op2, stat);
1326
1327 if ((float32_is_infinity(op1) && float32_is_zero(op2)) ||
1328 (float32_is_infinity(op2) && float32_is_zero(op1))) {
1329 return float32_two;
1330 }
1331 return float32_sub(float32_two, float32_mul(op1, op2, stat), stat);
1332 }
1333
1334 /* Reciprocal square-root step. AArch32 non-fused semantics. */
float16_rsqrts_nf(float16 op1,float16 op2,float_status * stat)1335 static float16 float16_rsqrts_nf(float16 op1, float16 op2, float_status *stat)
1336 {
1337 op1 = float16_squash_input_denormal(op1, stat);
1338 op2 = float16_squash_input_denormal(op2, stat);
1339
1340 if ((float16_is_infinity(op1) && float16_is_zero(op2)) ||
1341 (float16_is_infinity(op2) && float16_is_zero(op1))) {
1342 return float16_one_point_five;
1343 }
1344 op1 = float16_sub(float16_three, float16_mul(op1, op2, stat), stat);
1345 return float16_div(op1, float16_two, stat);
1346 }
1347
float32_rsqrts_nf(float32 op1,float32 op2,float_status * stat)1348 static float32 float32_rsqrts_nf(float32 op1, float32 op2, float_status *stat)
1349 {
1350 op1 = float32_squash_input_denormal(op1, stat);
1351 op2 = float32_squash_input_denormal(op2, stat);
1352
1353 if ((float32_is_infinity(op1) && float32_is_zero(op2)) ||
1354 (float32_is_infinity(op2) && float32_is_zero(op1))) {
1355 return float32_one_point_five;
1356 }
1357 op1 = float32_sub(float32_three, float32_mul(op1, op2, stat), stat);
1358 return float32_div(op1, float32_two, stat);
1359 }
1360
1361 #define DO_3OP(NAME, FUNC, TYPE) \
1362 void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
1363 { \
1364 intptr_t i, oprsz = simd_oprsz(desc); \
1365 TYPE *d = vd, *n = vn, *m = vm; \
1366 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1367 d[i] = FUNC(n[i], m[i], stat); \
1368 } \
1369 clear_tail(d, oprsz, simd_maxsz(desc)); \
1370 }
1371
DO_3OP(gvec_fadd_h,float16_add,float16)1372 DO_3OP(gvec_fadd_h, float16_add, float16)
1373 DO_3OP(gvec_fadd_s, float32_add, float32)
1374 DO_3OP(gvec_fadd_d, float64_add, float64)
1375
1376 DO_3OP(gvec_fsub_h, float16_sub, float16)
1377 DO_3OP(gvec_fsub_s, float32_sub, float32)
1378 DO_3OP(gvec_fsub_d, float64_sub, float64)
1379
1380 DO_3OP(gvec_fmul_h, float16_mul, float16)
1381 DO_3OP(gvec_fmul_s, float32_mul, float32)
1382 DO_3OP(gvec_fmul_d, float64_mul, float64)
1383
1384 DO_3OP(gvec_ftsmul_h, float16_ftsmul, float16)
1385 DO_3OP(gvec_ftsmul_s, float32_ftsmul, float32)
1386 DO_3OP(gvec_ftsmul_d, float64_ftsmul, float64)
1387
1388 DO_3OP(gvec_fabd_h, float16_abd, float16)
1389 DO_3OP(gvec_fabd_s, float32_abd, float32)
1390 DO_3OP(gvec_fabd_d, float64_abd, float64)
1391
1392 DO_3OP(gvec_fceq_h, float16_ceq, float16)
1393 DO_3OP(gvec_fceq_s, float32_ceq, float32)
1394 DO_3OP(gvec_fceq_d, float64_ceq, float64)
1395
1396 DO_3OP(gvec_fcge_h, float16_cge, float16)
1397 DO_3OP(gvec_fcge_s, float32_cge, float32)
1398 DO_3OP(gvec_fcge_d, float64_cge, float64)
1399
1400 DO_3OP(gvec_fcgt_h, float16_cgt, float16)
1401 DO_3OP(gvec_fcgt_s, float32_cgt, float32)
1402 DO_3OP(gvec_fcgt_d, float64_cgt, float64)
1403
1404 DO_3OP(gvec_facge_h, float16_acge, float16)
1405 DO_3OP(gvec_facge_s, float32_acge, float32)
1406 DO_3OP(gvec_facge_d, float64_acge, float64)
1407
1408 DO_3OP(gvec_facgt_h, float16_acgt, float16)
1409 DO_3OP(gvec_facgt_s, float32_acgt, float32)
1410 DO_3OP(gvec_facgt_d, float64_acgt, float64)
1411
1412 DO_3OP(gvec_fmax_h, float16_max, float16)
1413 DO_3OP(gvec_fmax_s, float32_max, float32)
1414 DO_3OP(gvec_fmax_d, float64_max, float64)
1415
1416 DO_3OP(gvec_fmin_h, float16_min, float16)
1417 DO_3OP(gvec_fmin_s, float32_min, float32)
1418 DO_3OP(gvec_fmin_d, float64_min, float64)
1419
1420 DO_3OP(gvec_fmaxnum_h, float16_maxnum, float16)
1421 DO_3OP(gvec_fmaxnum_s, float32_maxnum, float32)
1422 DO_3OP(gvec_fmaxnum_d, float64_maxnum, float64)
1423
1424 DO_3OP(gvec_fminnum_h, float16_minnum, float16)
1425 DO_3OP(gvec_fminnum_s, float32_minnum, float32)
1426 DO_3OP(gvec_fminnum_d, float64_minnum, float64)
1427
1428 DO_3OP(gvec_recps_nf_h, float16_recps_nf, float16)
1429 DO_3OP(gvec_recps_nf_s, float32_recps_nf, float32)
1430
1431 DO_3OP(gvec_rsqrts_nf_h, float16_rsqrts_nf, float16)
1432 DO_3OP(gvec_rsqrts_nf_s, float32_rsqrts_nf, float32)
1433
1434 #ifdef TARGET_AARCH64
1435 DO_3OP(gvec_fdiv_h, float16_div, float16)
1436 DO_3OP(gvec_fdiv_s, float32_div, float32)
1437 DO_3OP(gvec_fdiv_d, float64_div, float64)
1438
1439 DO_3OP(gvec_fmulx_h, helper_advsimd_mulxh, float16)
1440 DO_3OP(gvec_fmulx_s, helper_vfp_mulxs, float32)
1441 DO_3OP(gvec_fmulx_d, helper_vfp_mulxd, float64)
1442
1443 DO_3OP(gvec_recps_h, helper_recpsf_f16, float16)
1444 DO_3OP(gvec_recps_s, helper_recpsf_f32, float32)
1445 DO_3OP(gvec_recps_d, helper_recpsf_f64, float64)
1446
1447 DO_3OP(gvec_rsqrts_h, helper_rsqrtsf_f16, float16)
1448 DO_3OP(gvec_rsqrts_s, helper_rsqrtsf_f32, float32)
1449 DO_3OP(gvec_rsqrts_d, helper_rsqrtsf_f64, float64)
1450
1451 #endif
1452 #undef DO_3OP
1453
1454 /* Non-fused multiply-add (unlike float16_muladd etc, which are fused) */
1455 static float16 float16_muladd_nf(float16 dest, float16 op1, float16 op2,
1456 float_status *stat)
1457 {
1458 return float16_add(dest, float16_mul(op1, op2, stat), stat);
1459 }
1460
float32_muladd_nf(float32 dest,float32 op1,float32 op2,float_status * stat)1461 static float32 float32_muladd_nf(float32 dest, float32 op1, float32 op2,
1462 float_status *stat)
1463 {
1464 return float32_add(dest, float32_mul(op1, op2, stat), stat);
1465 }
1466
float16_mulsub_nf(float16 dest,float16 op1,float16 op2,float_status * stat)1467 static float16 float16_mulsub_nf(float16 dest, float16 op1, float16 op2,
1468 float_status *stat)
1469 {
1470 return float16_sub(dest, float16_mul(op1, op2, stat), stat);
1471 }
1472
float32_mulsub_nf(float32 dest,float32 op1,float32 op2,float_status * stat)1473 static float32 float32_mulsub_nf(float32 dest, float32 op1, float32 op2,
1474 float_status *stat)
1475 {
1476 return float32_sub(dest, float32_mul(op1, op2, stat), stat);
1477 }
1478
1479 /* Fused versions; these have the semantics Neon VFMA/VFMS want */
float16_muladd_f(float16 dest,float16 op1,float16 op2,float_status * stat)1480 static float16 float16_muladd_f(float16 dest, float16 op1, float16 op2,
1481 float_status *stat)
1482 {
1483 return float16_muladd(op1, op2, dest, 0, stat);
1484 }
1485
float32_muladd_f(float32 dest,float32 op1,float32 op2,float_status * stat)1486 static float32 float32_muladd_f(float32 dest, float32 op1, float32 op2,
1487 float_status *stat)
1488 {
1489 return float32_muladd(op1, op2, dest, 0, stat);
1490 }
1491
float64_muladd_f(float64 dest,float64 op1,float64 op2,float_status * stat)1492 static float64 float64_muladd_f(float64 dest, float64 op1, float64 op2,
1493 float_status *stat)
1494 {
1495 return float64_muladd(op1, op2, dest, 0, stat);
1496 }
1497
float16_mulsub_f(float16 dest,float16 op1,float16 op2,float_status * stat)1498 static float16 float16_mulsub_f(float16 dest, float16 op1, float16 op2,
1499 float_status *stat)
1500 {
1501 return float16_muladd(float16_chs(op1), op2, dest, 0, stat);
1502 }
1503
float32_mulsub_f(float32 dest,float32 op1,float32 op2,float_status * stat)1504 static float32 float32_mulsub_f(float32 dest, float32 op1, float32 op2,
1505 float_status *stat)
1506 {
1507 return float32_muladd(float32_chs(op1), op2, dest, 0, stat);
1508 }
1509
float64_mulsub_f(float64 dest,float64 op1,float64 op2,float_status * stat)1510 static float64 float64_mulsub_f(float64 dest, float64 op1, float64 op2,
1511 float_status *stat)
1512 {
1513 return float64_muladd(float64_chs(op1), op2, dest, 0, stat);
1514 }
1515
1516 #define DO_MULADD(NAME, FUNC, TYPE) \
1517 void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
1518 { \
1519 intptr_t i, oprsz = simd_oprsz(desc); \
1520 TYPE *d = vd, *n = vn, *m = vm; \
1521 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1522 d[i] = FUNC(d[i], n[i], m[i], stat); \
1523 } \
1524 clear_tail(d, oprsz, simd_maxsz(desc)); \
1525 }
1526
DO_MULADD(gvec_fmla_h,float16_muladd_nf,float16)1527 DO_MULADD(gvec_fmla_h, float16_muladd_nf, float16)
1528 DO_MULADD(gvec_fmla_s, float32_muladd_nf, float32)
1529
1530 DO_MULADD(gvec_fmls_h, float16_mulsub_nf, float16)
1531 DO_MULADD(gvec_fmls_s, float32_mulsub_nf, float32)
1532
1533 DO_MULADD(gvec_vfma_h, float16_muladd_f, float16)
1534 DO_MULADD(gvec_vfma_s, float32_muladd_f, float32)
1535 DO_MULADD(gvec_vfma_d, float64_muladd_f, float64)
1536
1537 DO_MULADD(gvec_vfms_h, float16_mulsub_f, float16)
1538 DO_MULADD(gvec_vfms_s, float32_mulsub_f, float32)
1539 DO_MULADD(gvec_vfms_d, float64_mulsub_f, float64)
1540
1541 /* For the indexed ops, SVE applies the index per 128-bit vector segment.
1542 * For AdvSIMD, there is of course only one such vector segment.
1543 */
1544
1545 #define DO_MUL_IDX(NAME, TYPE, H) \
1546 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1547 { \
1548 intptr_t i, j, oprsz = simd_oprsz(desc); \
1549 intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1550 intptr_t idx = simd_data(desc); \
1551 TYPE *d = vd, *n = vn, *m = vm; \
1552 for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1553 TYPE mm = m[H(i + idx)]; \
1554 for (j = 0; j < segment; j++) { \
1555 d[i + j] = n[i + j] * mm; \
1556 } \
1557 } \
1558 clear_tail(d, oprsz, simd_maxsz(desc)); \
1559 }
1560
1561 DO_MUL_IDX(gvec_mul_idx_h, uint16_t, H2)
1562 DO_MUL_IDX(gvec_mul_idx_s, uint32_t, H4)
1563 DO_MUL_IDX(gvec_mul_idx_d, uint64_t, H8)
1564
1565 #undef DO_MUL_IDX
1566
1567 #define DO_MLA_IDX(NAME, TYPE, OP, H) \
1568 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1569 { \
1570 intptr_t i, j, oprsz = simd_oprsz(desc); \
1571 intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1572 intptr_t idx = simd_data(desc); \
1573 TYPE *d = vd, *n = vn, *m = vm, *a = va; \
1574 for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1575 TYPE mm = m[H(i + idx)]; \
1576 for (j = 0; j < segment; j++) { \
1577 d[i + j] = a[i + j] OP n[i + j] * mm; \
1578 } \
1579 } \
1580 clear_tail(d, oprsz, simd_maxsz(desc)); \
1581 }
1582
1583 DO_MLA_IDX(gvec_mla_idx_h, uint16_t, +, H2)
1584 DO_MLA_IDX(gvec_mla_idx_s, uint32_t, +, H4)
1585 DO_MLA_IDX(gvec_mla_idx_d, uint64_t, +, H8)
1586
1587 DO_MLA_IDX(gvec_mls_idx_h, uint16_t, -, H2)
1588 DO_MLA_IDX(gvec_mls_idx_s, uint32_t, -, H4)
1589 DO_MLA_IDX(gvec_mls_idx_d, uint64_t, -, H8)
1590
1591 #undef DO_MLA_IDX
1592
1593 #define DO_FMUL_IDX(NAME, ADD, MUL, TYPE, H) \
1594 void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
1595 { \
1596 intptr_t i, j, oprsz = simd_oprsz(desc); \
1597 intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1598 intptr_t idx = simd_data(desc); \
1599 TYPE *d = vd, *n = vn, *m = vm; \
1600 for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1601 TYPE mm = m[H(i + idx)]; \
1602 for (j = 0; j < segment; j++) { \
1603 d[i + j] = ADD(d[i + j], MUL(n[i + j], mm, stat), stat); \
1604 } \
1605 } \
1606 clear_tail(d, oprsz, simd_maxsz(desc)); \
1607 }
1608
1609 #define nop(N, M, S) (M)
1610
1611 DO_FMUL_IDX(gvec_fmul_idx_h, nop, float16_mul, float16, H2)
1612 DO_FMUL_IDX(gvec_fmul_idx_s, nop, float32_mul, float32, H4)
1613 DO_FMUL_IDX(gvec_fmul_idx_d, nop, float64_mul, float64, H8)
1614
1615 #ifdef TARGET_AARCH64
1616
1617 DO_FMUL_IDX(gvec_fmulx_idx_h, nop, helper_advsimd_mulxh, float16, H2)
1618 DO_FMUL_IDX(gvec_fmulx_idx_s, nop, helper_vfp_mulxs, float32, H4)
1619 DO_FMUL_IDX(gvec_fmulx_idx_d, nop, helper_vfp_mulxd, float64, H8)
1620
1621 #endif
1622
1623 #undef nop
1624
1625 /*
1626 * Non-fused multiply-accumulate operations, for Neon. NB that unlike
1627 * the fused ops below they assume accumulate both from and into Vd.
1628 */
1629 DO_FMUL_IDX(gvec_fmla_nf_idx_h, float16_add, float16_mul, float16, H2)
1630 DO_FMUL_IDX(gvec_fmla_nf_idx_s, float32_add, float32_mul, float32, H4)
1631 DO_FMUL_IDX(gvec_fmls_nf_idx_h, float16_sub, float16_mul, float16, H2)
1632 DO_FMUL_IDX(gvec_fmls_nf_idx_s, float32_sub, float32_mul, float32, H4)
1633
1634 #undef DO_FMUL_IDX
1635
1636 #define DO_FMLA_IDX(NAME, TYPE, H) \
1637 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, \
1638 void *stat, uint32_t desc) \
1639 { \
1640 intptr_t i, j, oprsz = simd_oprsz(desc); \
1641 intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1642 TYPE op1_neg = extract32(desc, SIMD_DATA_SHIFT, 1); \
1643 intptr_t idx = desc >> (SIMD_DATA_SHIFT + 1); \
1644 TYPE *d = vd, *n = vn, *m = vm, *a = va; \
1645 op1_neg <<= (8 * sizeof(TYPE) - 1); \
1646 for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1647 TYPE mm = m[H(i + idx)]; \
1648 for (j = 0; j < segment; j++) { \
1649 d[i + j] = TYPE##_muladd(n[i + j] ^ op1_neg, \
1650 mm, a[i + j], 0, stat); \
1651 } \
1652 } \
1653 clear_tail(d, oprsz, simd_maxsz(desc)); \
1654 }
1655
1656 DO_FMLA_IDX(gvec_fmla_idx_h, float16, H2)
1657 DO_FMLA_IDX(gvec_fmla_idx_s, float32, H4)
1658 DO_FMLA_IDX(gvec_fmla_idx_d, float64, H8)
1659
1660 #undef DO_FMLA_IDX
1661
1662 #define DO_SAT(NAME, WTYPE, TYPEN, TYPEM, OP, MIN, MAX) \
1663 void HELPER(NAME)(void *vd, void *vq, void *vn, void *vm, uint32_t desc) \
1664 { \
1665 intptr_t i, oprsz = simd_oprsz(desc); \
1666 TYPEN *d = vd, *n = vn; TYPEM *m = vm; \
1667 bool q = false; \
1668 for (i = 0; i < oprsz / sizeof(TYPEN); i++) { \
1669 WTYPE dd = (WTYPE)n[i] OP m[i]; \
1670 if (dd < MIN) { \
1671 dd = MIN; \
1672 q = true; \
1673 } else if (dd > MAX) { \
1674 dd = MAX; \
1675 q = true; \
1676 } \
1677 d[i] = dd; \
1678 } \
1679 if (q) { \
1680 uint32_t *qc = vq; \
1681 qc[0] = 1; \
1682 } \
1683 clear_tail(d, oprsz, simd_maxsz(desc)); \
1684 }
1685
1686 DO_SAT(gvec_uqadd_b, int, uint8_t, uint8_t, +, 0, UINT8_MAX)
1687 DO_SAT(gvec_uqadd_h, int, uint16_t, uint16_t, +, 0, UINT16_MAX)
1688 DO_SAT(gvec_uqadd_s, int64_t, uint32_t, uint32_t, +, 0, UINT32_MAX)
1689
1690 DO_SAT(gvec_sqadd_b, int, int8_t, int8_t, +, INT8_MIN, INT8_MAX)
1691 DO_SAT(gvec_sqadd_h, int, int16_t, int16_t, +, INT16_MIN, INT16_MAX)
1692 DO_SAT(gvec_sqadd_s, int64_t, int32_t, int32_t, +, INT32_MIN, INT32_MAX)
1693
1694 DO_SAT(gvec_uqsub_b, int, uint8_t, uint8_t, -, 0, UINT8_MAX)
1695 DO_SAT(gvec_uqsub_h, int, uint16_t, uint16_t, -, 0, UINT16_MAX)
1696 DO_SAT(gvec_uqsub_s, int64_t, uint32_t, uint32_t, -, 0, UINT32_MAX)
1697
1698 DO_SAT(gvec_sqsub_b, int, int8_t, int8_t, -, INT8_MIN, INT8_MAX)
1699 DO_SAT(gvec_sqsub_h, int, int16_t, int16_t, -, INT16_MIN, INT16_MAX)
1700 DO_SAT(gvec_sqsub_s, int64_t, int32_t, int32_t, -, INT32_MIN, INT32_MAX)
1701
1702 DO_SAT(gvec_usqadd_b, int, uint8_t, int8_t, +, 0, UINT8_MAX)
1703 DO_SAT(gvec_usqadd_h, int, uint16_t, int16_t, +, 0, UINT16_MAX)
1704 DO_SAT(gvec_usqadd_s, int64_t, uint32_t, int32_t, +, 0, UINT32_MAX)
1705
1706 DO_SAT(gvec_suqadd_b, int, int8_t, uint8_t, +, INT8_MIN, INT8_MAX)
1707 DO_SAT(gvec_suqadd_h, int, int16_t, uint16_t, +, INT16_MIN, INT16_MAX)
1708 DO_SAT(gvec_suqadd_s, int64_t, int32_t, uint32_t, +, INT32_MIN, INT32_MAX)
1709
1710 #undef DO_SAT
1711
1712 void HELPER(gvec_uqadd_d)(void *vd, void *vq, void *vn,
1713 void *vm, uint32_t desc)
1714 {
1715 intptr_t i, oprsz = simd_oprsz(desc);
1716 uint64_t *d = vd, *n = vn, *m = vm;
1717 bool q = false;
1718
1719 for (i = 0; i < oprsz / 8; i++) {
1720 uint64_t nn = n[i], mm = m[i], dd = nn + mm;
1721 if (dd < nn) {
1722 dd = UINT64_MAX;
1723 q = true;
1724 }
1725 d[i] = dd;
1726 }
1727 if (q) {
1728 uint32_t *qc = vq;
1729 qc[0] = 1;
1730 }
1731 clear_tail(d, oprsz, simd_maxsz(desc));
1732 }
1733
HELPER(gvec_uqsub_d)1734 void HELPER(gvec_uqsub_d)(void *vd, void *vq, void *vn,
1735 void *vm, uint32_t desc)
1736 {
1737 intptr_t i, oprsz = simd_oprsz(desc);
1738 uint64_t *d = vd, *n = vn, *m = vm;
1739 bool q = false;
1740
1741 for (i = 0; i < oprsz / 8; i++) {
1742 uint64_t nn = n[i], mm = m[i], dd = nn - mm;
1743 if (nn < mm) {
1744 dd = 0;
1745 q = true;
1746 }
1747 d[i] = dd;
1748 }
1749 if (q) {
1750 uint32_t *qc = vq;
1751 qc[0] = 1;
1752 }
1753 clear_tail(d, oprsz, simd_maxsz(desc));
1754 }
1755
HELPER(gvec_sqadd_d)1756 void HELPER(gvec_sqadd_d)(void *vd, void *vq, void *vn,
1757 void *vm, uint32_t desc)
1758 {
1759 intptr_t i, oprsz = simd_oprsz(desc);
1760 int64_t *d = vd, *n = vn, *m = vm;
1761 bool q = false;
1762
1763 for (i = 0; i < oprsz / 8; i++) {
1764 int64_t nn = n[i], mm = m[i], dd = nn + mm;
1765 if (((dd ^ nn) & ~(nn ^ mm)) & INT64_MIN) {
1766 dd = (nn >> 63) ^ ~INT64_MIN;
1767 q = true;
1768 }
1769 d[i] = dd;
1770 }
1771 if (q) {
1772 uint32_t *qc = vq;
1773 qc[0] = 1;
1774 }
1775 clear_tail(d, oprsz, simd_maxsz(desc));
1776 }
1777
HELPER(gvec_sqsub_d)1778 void HELPER(gvec_sqsub_d)(void *vd, void *vq, void *vn,
1779 void *vm, uint32_t desc)
1780 {
1781 intptr_t i, oprsz = simd_oprsz(desc);
1782 int64_t *d = vd, *n = vn, *m = vm;
1783 bool q = false;
1784
1785 for (i = 0; i < oprsz / 8; i++) {
1786 int64_t nn = n[i], mm = m[i], dd = nn - mm;
1787 if (((dd ^ nn) & (nn ^ mm)) & INT64_MIN) {
1788 dd = (nn >> 63) ^ ~INT64_MIN;
1789 q = true;
1790 }
1791 d[i] = dd;
1792 }
1793 if (q) {
1794 uint32_t *qc = vq;
1795 qc[0] = 1;
1796 }
1797 clear_tail(d, oprsz, simd_maxsz(desc));
1798 }
1799
HELPER(gvec_usqadd_d)1800 void HELPER(gvec_usqadd_d)(void *vd, void *vq, void *vn,
1801 void *vm, uint32_t desc)
1802 {
1803 intptr_t i, oprsz = simd_oprsz(desc);
1804 uint64_t *d = vd, *n = vn, *m = vm;
1805 bool q = false;
1806
1807 for (i = 0; i < oprsz / 8; i++) {
1808 uint64_t nn = n[i];
1809 int64_t mm = m[i];
1810 uint64_t dd = nn + mm;
1811
1812 if (mm < 0) {
1813 if (nn < (uint64_t)-mm) {
1814 dd = 0;
1815 q = true;
1816 }
1817 } else {
1818 if (dd < nn) {
1819 dd = UINT64_MAX;
1820 q = true;
1821 }
1822 }
1823 d[i] = dd;
1824 }
1825 if (q) {
1826 uint32_t *qc = vq;
1827 qc[0] = 1;
1828 }
1829 clear_tail(d, oprsz, simd_maxsz(desc));
1830 }
1831
HELPER(gvec_suqadd_d)1832 void HELPER(gvec_suqadd_d)(void *vd, void *vq, void *vn,
1833 void *vm, uint32_t desc)
1834 {
1835 intptr_t i, oprsz = simd_oprsz(desc);
1836 uint64_t *d = vd, *n = vn, *m = vm;
1837 bool q = false;
1838
1839 for (i = 0; i < oprsz / 8; i++) {
1840 int64_t nn = n[i];
1841 uint64_t mm = m[i];
1842 int64_t dd = nn + mm;
1843
1844 if (mm > (uint64_t)(INT64_MAX - nn)) {
1845 dd = INT64_MAX;
1846 q = true;
1847 }
1848 d[i] = dd;
1849 }
1850 if (q) {
1851 uint32_t *qc = vq;
1852 qc[0] = 1;
1853 }
1854 clear_tail(d, oprsz, simd_maxsz(desc));
1855 }
1856
1857 #define DO_SRA(NAME, TYPE) \
1858 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1859 { \
1860 intptr_t i, oprsz = simd_oprsz(desc); \
1861 int shift = simd_data(desc); \
1862 TYPE *d = vd, *n = vn; \
1863 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1864 d[i] += n[i] >> shift; \
1865 } \
1866 clear_tail(d, oprsz, simd_maxsz(desc)); \
1867 }
1868
DO_SRA(gvec_ssra_b,int8_t)1869 DO_SRA(gvec_ssra_b, int8_t)
1870 DO_SRA(gvec_ssra_h, int16_t)
1871 DO_SRA(gvec_ssra_s, int32_t)
1872 DO_SRA(gvec_ssra_d, int64_t)
1873
1874 DO_SRA(gvec_usra_b, uint8_t)
1875 DO_SRA(gvec_usra_h, uint16_t)
1876 DO_SRA(gvec_usra_s, uint32_t)
1877 DO_SRA(gvec_usra_d, uint64_t)
1878
1879 #undef DO_SRA
1880
1881 #define DO_RSHR(NAME, TYPE) \
1882 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1883 { \
1884 intptr_t i, oprsz = simd_oprsz(desc); \
1885 int shift = simd_data(desc); \
1886 TYPE *d = vd, *n = vn; \
1887 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1888 TYPE tmp = n[i] >> (shift - 1); \
1889 d[i] = (tmp >> 1) + (tmp & 1); \
1890 } \
1891 clear_tail(d, oprsz, simd_maxsz(desc)); \
1892 }
1893
1894 DO_RSHR(gvec_srshr_b, int8_t)
1895 DO_RSHR(gvec_srshr_h, int16_t)
1896 DO_RSHR(gvec_srshr_s, int32_t)
1897 DO_RSHR(gvec_srshr_d, int64_t)
1898
1899 DO_RSHR(gvec_urshr_b, uint8_t)
1900 DO_RSHR(gvec_urshr_h, uint16_t)
1901 DO_RSHR(gvec_urshr_s, uint32_t)
1902 DO_RSHR(gvec_urshr_d, uint64_t)
1903
1904 #undef DO_RSHR
1905
1906 #define DO_RSRA(NAME, TYPE) \
1907 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1908 { \
1909 intptr_t i, oprsz = simd_oprsz(desc); \
1910 int shift = simd_data(desc); \
1911 TYPE *d = vd, *n = vn; \
1912 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1913 TYPE tmp = n[i] >> (shift - 1); \
1914 d[i] += (tmp >> 1) + (tmp & 1); \
1915 } \
1916 clear_tail(d, oprsz, simd_maxsz(desc)); \
1917 }
1918
1919 DO_RSRA(gvec_srsra_b, int8_t)
1920 DO_RSRA(gvec_srsra_h, int16_t)
1921 DO_RSRA(gvec_srsra_s, int32_t)
1922 DO_RSRA(gvec_srsra_d, int64_t)
1923
1924 DO_RSRA(gvec_ursra_b, uint8_t)
1925 DO_RSRA(gvec_ursra_h, uint16_t)
1926 DO_RSRA(gvec_ursra_s, uint32_t)
1927 DO_RSRA(gvec_ursra_d, uint64_t)
1928
1929 #undef DO_RSRA
1930
1931 #define DO_SRI(NAME, TYPE) \
1932 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1933 { \
1934 intptr_t i, oprsz = simd_oprsz(desc); \
1935 int shift = simd_data(desc); \
1936 TYPE *d = vd, *n = vn; \
1937 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1938 d[i] = deposit64(d[i], 0, sizeof(TYPE) * 8 - shift, n[i] >> shift); \
1939 } \
1940 clear_tail(d, oprsz, simd_maxsz(desc)); \
1941 }
1942
1943 DO_SRI(gvec_sri_b, uint8_t)
1944 DO_SRI(gvec_sri_h, uint16_t)
1945 DO_SRI(gvec_sri_s, uint32_t)
1946 DO_SRI(gvec_sri_d, uint64_t)
1947
1948 #undef DO_SRI
1949
1950 #define DO_SLI(NAME, TYPE) \
1951 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1952 { \
1953 intptr_t i, oprsz = simd_oprsz(desc); \
1954 int shift = simd_data(desc); \
1955 TYPE *d = vd, *n = vn; \
1956 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1957 d[i] = deposit64(d[i], shift, sizeof(TYPE) * 8 - shift, n[i]); \
1958 } \
1959 clear_tail(d, oprsz, simd_maxsz(desc)); \
1960 }
1961
1962 DO_SLI(gvec_sli_b, uint8_t)
1963 DO_SLI(gvec_sli_h, uint16_t)
1964 DO_SLI(gvec_sli_s, uint32_t)
1965 DO_SLI(gvec_sli_d, uint64_t)
1966
1967 #undef DO_SLI
1968
1969 /*
1970 * Convert float16 to float32, raising no exceptions and
1971 * preserving exceptional values, including SNaN.
1972 * This is effectively an unpack+repack operation.
1973 */
1974 static float32 float16_to_float32_by_bits(uint32_t f16, bool fz16)
1975 {
1976 const int f16_bias = 15;
1977 const int f32_bias = 127;
1978 uint32_t sign = extract32(f16, 15, 1);
1979 uint32_t exp = extract32(f16, 10, 5);
1980 uint32_t frac = extract32(f16, 0, 10);
1981
1982 if (exp == 0x1f) {
1983 /* Inf or NaN */
1984 exp = 0xff;
1985 } else if (exp == 0) {
1986 /* Zero or denormal. */
1987 if (frac != 0) {
1988 if (fz16) {
1989 frac = 0;
1990 } else {
1991 /*
1992 * Denormal; these are all normal float32.
1993 * Shift the fraction so that the msb is at bit 11,
1994 * then remove bit 11 as the implicit bit of the
1995 * normalized float32. Note that we still go through
1996 * the shift for normal numbers below, to put the
1997 * float32 fraction at the right place.
1998 */
1999 int shift = clz32(frac) - 21;
2000 frac = (frac << shift) & 0x3ff;
2001 exp = f32_bias - f16_bias - shift + 1;
2002 }
2003 }
2004 } else {
2005 /* Normal number; adjust the bias. */
2006 exp += f32_bias - f16_bias;
2007 }
2008 sign <<= 31;
2009 exp <<= 23;
2010 frac <<= 23 - 10;
2011
2012 return sign | exp | frac;
2013 }
2014
load4_f16(uint64_t * ptr,int is_q,int is_2)2015 static uint64_t load4_f16(uint64_t *ptr, int is_q, int is_2)
2016 {
2017 /*
2018 * Branchless load of u32[0], u64[0], u32[1], or u64[1].
2019 * Load the 2nd qword iff is_q & is_2.
2020 * Shift to the 2nd dword iff !is_q & is_2.
2021 * For !is_q & !is_2, the upper bits of the result are garbage.
2022 */
2023 return ptr[is_q & is_2] >> ((is_2 & ~is_q) << 5);
2024 }
2025
2026 /*
2027 * Note that FMLAL requires oprsz == 8 or oprsz == 16,
2028 * as there is not yet SVE versions that might use blocking.
2029 */
2030
do_fmlal(float32 * d,void * vn,void * vm,float_status * fpst,uint32_t desc,bool fz16)2031 static void do_fmlal(float32 *d, void *vn, void *vm, float_status *fpst,
2032 uint32_t desc, bool fz16)
2033 {
2034 intptr_t i, oprsz = simd_oprsz(desc);
2035 int is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2036 int is_2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
2037 int is_q = oprsz == 16;
2038 uint64_t n_4, m_4;
2039
2040 /* Pre-load all of the f16 data, avoiding overlap issues. */
2041 n_4 = load4_f16(vn, is_q, is_2);
2042 m_4 = load4_f16(vm, is_q, is_2);
2043
2044 /* Negate all inputs for FMLSL at once. */
2045 if (is_s) {
2046 n_4 ^= 0x8000800080008000ull;
2047 }
2048
2049 for (i = 0; i < oprsz / 4; i++) {
2050 float32 n_1 = float16_to_float32_by_bits(n_4 >> (i * 16), fz16);
2051 float32 m_1 = float16_to_float32_by_bits(m_4 >> (i * 16), fz16);
2052 d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], 0, fpst);
2053 }
2054 clear_tail(d, oprsz, simd_maxsz(desc));
2055 }
2056
HELPER(gvec_fmlal_a32)2057 void HELPER(gvec_fmlal_a32)(void *vd, void *vn, void *vm,
2058 void *venv, uint32_t desc)
2059 {
2060 CPUARMState *env = venv;
2061 do_fmlal(vd, vn, vm, &env->vfp.standard_fp_status, desc,
2062 get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
2063 }
2064
HELPER(gvec_fmlal_a64)2065 void HELPER(gvec_fmlal_a64)(void *vd, void *vn, void *vm,
2066 void *venv, uint32_t desc)
2067 {
2068 CPUARMState *env = venv;
2069 do_fmlal(vd, vn, vm, &env->vfp.fp_status, desc,
2070 get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
2071 }
2072
HELPER(sve2_fmlal_zzzw_s)2073 void HELPER(sve2_fmlal_zzzw_s)(void *vd, void *vn, void *vm, void *va,
2074 void *venv, uint32_t desc)
2075 {
2076 intptr_t i, oprsz = simd_oprsz(desc);
2077 uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
2078 intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
2079 CPUARMState *env = venv;
2080 float_status *status = &env->vfp.fp_status;
2081 bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
2082
2083 for (i = 0; i < oprsz; i += sizeof(float32)) {
2084 float16 nn_16 = *(float16 *)(vn + H1_2(i + sel)) ^ negn;
2085 float16 mm_16 = *(float16 *)(vm + H1_2(i + sel));
2086 float32 nn = float16_to_float32_by_bits(nn_16, fz16);
2087 float32 mm = float16_to_float32_by_bits(mm_16, fz16);
2088 float32 aa = *(float32 *)(va + H1_4(i));
2089
2090 *(float32 *)(vd + H1_4(i)) = float32_muladd(nn, mm, aa, 0, status);
2091 }
2092 }
2093
do_fmlal_idx(float32 * d,void * vn,void * vm,float_status * fpst,uint32_t desc,bool fz16)2094 static void do_fmlal_idx(float32 *d, void *vn, void *vm, float_status *fpst,
2095 uint32_t desc, bool fz16)
2096 {
2097 intptr_t i, oprsz = simd_oprsz(desc);
2098 int is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2099 int is_2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
2100 int index = extract32(desc, SIMD_DATA_SHIFT + 2, 3);
2101 int is_q = oprsz == 16;
2102 uint64_t n_4;
2103 float32 m_1;
2104
2105 /* Pre-load all of the f16 data, avoiding overlap issues. */
2106 n_4 = load4_f16(vn, is_q, is_2);
2107
2108 /* Negate all inputs for FMLSL at once. */
2109 if (is_s) {
2110 n_4 ^= 0x8000800080008000ull;
2111 }
2112
2113 m_1 = float16_to_float32_by_bits(((float16 *)vm)[H2(index)], fz16);
2114
2115 for (i = 0; i < oprsz / 4; i++) {
2116 float32 n_1 = float16_to_float32_by_bits(n_4 >> (i * 16), fz16);
2117 d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], 0, fpst);
2118 }
2119 clear_tail(d, oprsz, simd_maxsz(desc));
2120 }
2121
HELPER(gvec_fmlal_idx_a32)2122 void HELPER(gvec_fmlal_idx_a32)(void *vd, void *vn, void *vm,
2123 void *venv, uint32_t desc)
2124 {
2125 CPUARMState *env = venv;
2126 do_fmlal_idx(vd, vn, vm, &env->vfp.standard_fp_status, desc,
2127 get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
2128 }
2129
HELPER(gvec_fmlal_idx_a64)2130 void HELPER(gvec_fmlal_idx_a64)(void *vd, void *vn, void *vm,
2131 void *venv, uint32_t desc)
2132 {
2133 CPUARMState *env = venv;
2134 do_fmlal_idx(vd, vn, vm, &env->vfp.fp_status, desc,
2135 get_flush_inputs_to_zero(&env->vfp.fp_status_f16));
2136 }
2137
HELPER(sve2_fmlal_zzxw_s)2138 void HELPER(sve2_fmlal_zzxw_s)(void *vd, void *vn, void *vm, void *va,
2139 void *venv, uint32_t desc)
2140 {
2141 intptr_t i, j, oprsz = simd_oprsz(desc);
2142 uint16_t negn = extract32(desc, SIMD_DATA_SHIFT, 1) << 15;
2143 intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
2144 intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 2, 3) * sizeof(float16);
2145 CPUARMState *env = venv;
2146 float_status *status = &env->vfp.fp_status;
2147 bool fz16 = get_flush_inputs_to_zero(&env->vfp.fp_status_f16);
2148
2149 for (i = 0; i < oprsz; i += 16) {
2150 float16 mm_16 = *(float16 *)(vm + i + idx);
2151 float32 mm = float16_to_float32_by_bits(mm_16, fz16);
2152
2153 for (j = 0; j < 16; j += sizeof(float32)) {
2154 float16 nn_16 = *(float16 *)(vn + H1_2(i + j + sel)) ^ negn;
2155 float32 nn = float16_to_float32_by_bits(nn_16, fz16);
2156 float32 aa = *(float32 *)(va + H1_4(i + j));
2157
2158 *(float32 *)(vd + H1_4(i + j)) =
2159 float32_muladd(nn, mm, aa, 0, status);
2160 }
2161 }
2162 }
2163
HELPER(gvec_sshl_b)2164 void HELPER(gvec_sshl_b)(void *vd, void *vn, void *vm, uint32_t desc)
2165 {
2166 intptr_t i, opr_sz = simd_oprsz(desc);
2167 int8_t *d = vd, *n = vn, *m = vm;
2168
2169 for (i = 0; i < opr_sz; ++i) {
2170 int8_t mm = m[i];
2171 int8_t nn = n[i];
2172 int8_t res = 0;
2173 if (mm >= 0) {
2174 if (mm < 8) {
2175 res = nn << mm;
2176 }
2177 } else {
2178 res = nn >> (mm > -8 ? -mm : 7);
2179 }
2180 d[i] = res;
2181 }
2182 clear_tail(d, opr_sz, simd_maxsz(desc));
2183 }
2184
HELPER(gvec_sshl_h)2185 void HELPER(gvec_sshl_h)(void *vd, void *vn, void *vm, uint32_t desc)
2186 {
2187 intptr_t i, opr_sz = simd_oprsz(desc);
2188 int16_t *d = vd, *n = vn, *m = vm;
2189
2190 for (i = 0; i < opr_sz / 2; ++i) {
2191 int8_t mm = m[i]; /* only 8 bits of shift are significant */
2192 int16_t nn = n[i];
2193 int16_t res = 0;
2194 if (mm >= 0) {
2195 if (mm < 16) {
2196 res = nn << mm;
2197 }
2198 } else {
2199 res = nn >> (mm > -16 ? -mm : 15);
2200 }
2201 d[i] = res;
2202 }
2203 clear_tail(d, opr_sz, simd_maxsz(desc));
2204 }
2205
HELPER(gvec_ushl_b)2206 void HELPER(gvec_ushl_b)(void *vd, void *vn, void *vm, uint32_t desc)
2207 {
2208 intptr_t i, opr_sz = simd_oprsz(desc);
2209 uint8_t *d = vd, *n = vn, *m = vm;
2210
2211 for (i = 0; i < opr_sz; ++i) {
2212 int8_t mm = m[i];
2213 uint8_t nn = n[i];
2214 uint8_t res = 0;
2215 if (mm >= 0) {
2216 if (mm < 8) {
2217 res = nn << mm;
2218 }
2219 } else {
2220 if (mm > -8) {
2221 res = nn >> -mm;
2222 }
2223 }
2224 d[i] = res;
2225 }
2226 clear_tail(d, opr_sz, simd_maxsz(desc));
2227 }
2228
HELPER(gvec_ushl_h)2229 void HELPER(gvec_ushl_h)(void *vd, void *vn, void *vm, uint32_t desc)
2230 {
2231 intptr_t i, opr_sz = simd_oprsz(desc);
2232 uint16_t *d = vd, *n = vn, *m = vm;
2233
2234 for (i = 0; i < opr_sz / 2; ++i) {
2235 int8_t mm = m[i]; /* only 8 bits of shift are significant */
2236 uint16_t nn = n[i];
2237 uint16_t res = 0;
2238 if (mm >= 0) {
2239 if (mm < 16) {
2240 res = nn << mm;
2241 }
2242 } else {
2243 if (mm > -16) {
2244 res = nn >> -mm;
2245 }
2246 }
2247 d[i] = res;
2248 }
2249 clear_tail(d, opr_sz, simd_maxsz(desc));
2250 }
2251
2252 /*
2253 * 8x8->8 polynomial multiply.
2254 *
2255 * Polynomial multiplication is like integer multiplication except the
2256 * partial products are XORed, not added.
2257 *
2258 * TODO: expose this as a generic vector operation, as it is a common
2259 * crypto building block.
2260 */
HELPER(gvec_pmul_b)2261 void HELPER(gvec_pmul_b)(void *vd, void *vn, void *vm, uint32_t desc)
2262 {
2263 intptr_t i, opr_sz = simd_oprsz(desc);
2264 uint64_t *d = vd, *n = vn, *m = vm;
2265
2266 for (i = 0; i < opr_sz / 8; ++i) {
2267 d[i] = clmul_8x8_low(n[i], m[i]);
2268 }
2269 clear_tail(d, opr_sz, simd_maxsz(desc));
2270 }
2271
2272 /*
2273 * 64x64->128 polynomial multiply.
2274 * Because of the lanes are not accessed in strict columns,
2275 * this probably cannot be turned into a generic helper.
2276 */
HELPER(gvec_pmull_q)2277 void HELPER(gvec_pmull_q)(void *vd, void *vn, void *vm, uint32_t desc)
2278 {
2279 intptr_t i, opr_sz = simd_oprsz(desc);
2280 intptr_t hi = simd_data(desc);
2281 uint64_t *d = vd, *n = vn, *m = vm;
2282
2283 for (i = 0; i < opr_sz / 8; i += 2) {
2284 Int128 r = clmul_64(n[i + hi], m[i + hi]);
2285 d[i] = int128_getlo(r);
2286 d[i + 1] = int128_gethi(r);
2287 }
2288 clear_tail(d, opr_sz, simd_maxsz(desc));
2289 }
2290
HELPER(neon_pmull_h)2291 void HELPER(neon_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
2292 {
2293 int hi = simd_data(desc);
2294 uint64_t *d = vd, *n = vn, *m = vm;
2295 uint64_t nn = n[hi], mm = m[hi];
2296
2297 d[0] = clmul_8x4_packed(nn, mm);
2298 nn >>= 32;
2299 mm >>= 32;
2300 d[1] = clmul_8x4_packed(nn, mm);
2301
2302 clear_tail(d, 16, simd_maxsz(desc));
2303 }
2304
2305 #ifdef TARGET_AARCH64
HELPER(sve2_pmull_h)2306 void HELPER(sve2_pmull_h)(void *vd, void *vn, void *vm, uint32_t desc)
2307 {
2308 int shift = simd_data(desc) * 8;
2309 intptr_t i, opr_sz = simd_oprsz(desc);
2310 uint64_t *d = vd, *n = vn, *m = vm;
2311
2312 for (i = 0; i < opr_sz / 8; ++i) {
2313 d[i] = clmul_8x4_even(n[i] >> shift, m[i] >> shift);
2314 }
2315 }
2316
HELPER(sve2_pmull_d)2317 void HELPER(sve2_pmull_d)(void *vd, void *vn, void *vm, uint32_t desc)
2318 {
2319 intptr_t sel = H4(simd_data(desc));
2320 intptr_t i, opr_sz = simd_oprsz(desc);
2321 uint32_t *n = vn, *m = vm;
2322 uint64_t *d = vd;
2323
2324 for (i = 0; i < opr_sz / 8; ++i) {
2325 d[i] = clmul_32(n[2 * i + sel], m[2 * i + sel]);
2326 }
2327 }
2328 #endif
2329
2330 #define DO_CMP0(NAME, TYPE, OP) \
2331 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
2332 { \
2333 intptr_t i, opr_sz = simd_oprsz(desc); \
2334 for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
2335 TYPE nn = *(TYPE *)(vn + i); \
2336 *(TYPE *)(vd + i) = -(nn OP 0); \
2337 } \
2338 clear_tail(vd, opr_sz, simd_maxsz(desc)); \
2339 }
2340
2341 DO_CMP0(gvec_ceq0_b, int8_t, ==)
2342 DO_CMP0(gvec_clt0_b, int8_t, <)
2343 DO_CMP0(gvec_cle0_b, int8_t, <=)
2344 DO_CMP0(gvec_cgt0_b, int8_t, >)
2345 DO_CMP0(gvec_cge0_b, int8_t, >=)
2346
2347 DO_CMP0(gvec_ceq0_h, int16_t, ==)
2348 DO_CMP0(gvec_clt0_h, int16_t, <)
2349 DO_CMP0(gvec_cle0_h, int16_t, <=)
2350 DO_CMP0(gvec_cgt0_h, int16_t, >)
2351 DO_CMP0(gvec_cge0_h, int16_t, >=)
2352
2353 #undef DO_CMP0
2354
2355 #define DO_ABD(NAME, TYPE) \
2356 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2357 { \
2358 intptr_t i, opr_sz = simd_oprsz(desc); \
2359 TYPE *d = vd, *n = vn, *m = vm; \
2360 \
2361 for (i = 0; i < opr_sz / sizeof(TYPE); ++i) { \
2362 d[i] = n[i] < m[i] ? m[i] - n[i] : n[i] - m[i]; \
2363 } \
2364 clear_tail(d, opr_sz, simd_maxsz(desc)); \
2365 }
2366
DO_ABD(gvec_sabd_b,int8_t)2367 DO_ABD(gvec_sabd_b, int8_t)
2368 DO_ABD(gvec_sabd_h, int16_t)
2369 DO_ABD(gvec_sabd_s, int32_t)
2370 DO_ABD(gvec_sabd_d, int64_t)
2371
2372 DO_ABD(gvec_uabd_b, uint8_t)
2373 DO_ABD(gvec_uabd_h, uint16_t)
2374 DO_ABD(gvec_uabd_s, uint32_t)
2375 DO_ABD(gvec_uabd_d, uint64_t)
2376
2377 #undef DO_ABD
2378
2379 #define DO_ABA(NAME, TYPE) \
2380 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2381 { \
2382 intptr_t i, opr_sz = simd_oprsz(desc); \
2383 TYPE *d = vd, *n = vn, *m = vm; \
2384 \
2385 for (i = 0; i < opr_sz / sizeof(TYPE); ++i) { \
2386 d[i] += n[i] < m[i] ? m[i] - n[i] : n[i] - m[i]; \
2387 } \
2388 clear_tail(d, opr_sz, simd_maxsz(desc)); \
2389 }
2390
2391 DO_ABA(gvec_saba_b, int8_t)
2392 DO_ABA(gvec_saba_h, int16_t)
2393 DO_ABA(gvec_saba_s, int32_t)
2394 DO_ABA(gvec_saba_d, int64_t)
2395
2396 DO_ABA(gvec_uaba_b, uint8_t)
2397 DO_ABA(gvec_uaba_h, uint16_t)
2398 DO_ABA(gvec_uaba_s, uint32_t)
2399 DO_ABA(gvec_uaba_d, uint64_t)
2400
2401 #undef DO_ABA
2402
2403 #define DO_3OP_PAIR(NAME, FUNC, TYPE, H) \
2404 void HELPER(NAME)(void *vd, void *vn, void *vm, void *stat, uint32_t desc) \
2405 { \
2406 ARMVectorReg scratch; \
2407 intptr_t oprsz = simd_oprsz(desc); \
2408 intptr_t half = oprsz / sizeof(TYPE) / 2; \
2409 TYPE *d = vd, *n = vn, *m = vm; \
2410 if (unlikely(d == m)) { \
2411 m = memcpy(&scratch, m, oprsz); \
2412 } \
2413 for (intptr_t i = 0; i < half; ++i) { \
2414 d[H(i)] = FUNC(n[H(i * 2)], n[H(i * 2 + 1)], stat); \
2415 } \
2416 for (intptr_t i = 0; i < half; ++i) { \
2417 d[H(i + half)] = FUNC(m[H(i * 2)], m[H(i * 2 + 1)], stat); \
2418 } \
2419 clear_tail(d, oprsz, simd_maxsz(desc)); \
2420 }
2421
2422 DO_3OP_PAIR(gvec_faddp_h, float16_add, float16, H2)
2423 DO_3OP_PAIR(gvec_faddp_s, float32_add, float32, H4)
2424 DO_3OP_PAIR(gvec_faddp_d, float64_add, float64, )
2425
2426 DO_3OP_PAIR(gvec_fmaxp_h, float16_max, float16, H2)
2427 DO_3OP_PAIR(gvec_fmaxp_s, float32_max, float32, H4)
2428 DO_3OP_PAIR(gvec_fmaxp_d, float64_max, float64, )
2429
2430 DO_3OP_PAIR(gvec_fminp_h, float16_min, float16, H2)
2431 DO_3OP_PAIR(gvec_fminp_s, float32_min, float32, H4)
2432 DO_3OP_PAIR(gvec_fminp_d, float64_min, float64, )
2433
2434 DO_3OP_PAIR(gvec_fmaxnump_h, float16_maxnum, float16, H2)
2435 DO_3OP_PAIR(gvec_fmaxnump_s, float32_maxnum, float32, H4)
2436 DO_3OP_PAIR(gvec_fmaxnump_d, float64_maxnum, float64, )
2437
2438 DO_3OP_PAIR(gvec_fminnump_h, float16_minnum, float16, H2)
2439 DO_3OP_PAIR(gvec_fminnump_s, float32_minnum, float32, H4)
2440 DO_3OP_PAIR(gvec_fminnump_d, float64_minnum, float64, )
2441
2442 #undef DO_3OP_PAIR
2443
2444 #define DO_3OP_PAIR(NAME, FUNC, TYPE, H) \
2445 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2446 { \
2447 ARMVectorReg scratch; \
2448 intptr_t oprsz = simd_oprsz(desc); \
2449 intptr_t half = oprsz / sizeof(TYPE) / 2; \
2450 TYPE *d = vd, *n = vn, *m = vm; \
2451 if (unlikely(d == m)) { \
2452 m = memcpy(&scratch, m, oprsz); \
2453 } \
2454 for (intptr_t i = 0; i < half; ++i) { \
2455 d[H(i)] = FUNC(n[H(i * 2)], n[H(i * 2 + 1)]); \
2456 } \
2457 for (intptr_t i = 0; i < half; ++i) { \
2458 d[H(i + half)] = FUNC(m[H(i * 2)], m[H(i * 2 + 1)]); \
2459 } \
2460 clear_tail(d, oprsz, simd_maxsz(desc)); \
2461 }
2462
2463 #define ADD(A, B) (A + B)
2464 DO_3OP_PAIR(gvec_addp_b, ADD, uint8_t, H1)
2465 DO_3OP_PAIR(gvec_addp_h, ADD, uint16_t, H2)
2466 DO_3OP_PAIR(gvec_addp_s, ADD, uint32_t, H4)
2467 DO_3OP_PAIR(gvec_addp_d, ADD, uint64_t, )
2468 #undef ADD
2469
2470 DO_3OP_PAIR(gvec_smaxp_b, MAX, int8_t, H1)
2471 DO_3OP_PAIR(gvec_smaxp_h, MAX, int16_t, H2)
2472 DO_3OP_PAIR(gvec_smaxp_s, MAX, int32_t, H4)
2473
2474 DO_3OP_PAIR(gvec_umaxp_b, MAX, uint8_t, H1)
2475 DO_3OP_PAIR(gvec_umaxp_h, MAX, uint16_t, H2)
2476 DO_3OP_PAIR(gvec_umaxp_s, MAX, uint32_t, H4)
2477
2478 DO_3OP_PAIR(gvec_sminp_b, MIN, int8_t, H1)
2479 DO_3OP_PAIR(gvec_sminp_h, MIN, int16_t, H2)
2480 DO_3OP_PAIR(gvec_sminp_s, MIN, int32_t, H4)
2481
2482 DO_3OP_PAIR(gvec_uminp_b, MIN, uint8_t, H1)
2483 DO_3OP_PAIR(gvec_uminp_h, MIN, uint16_t, H2)
2484 DO_3OP_PAIR(gvec_uminp_s, MIN, uint32_t, H4)
2485
2486 #undef DO_3OP_PAIR
2487
2488 #define DO_VCVT_FIXED(NAME, FUNC, TYPE) \
2489 void HELPER(NAME)(void *vd, void *vn, void *stat, uint32_t desc) \
2490 { \
2491 intptr_t i, oprsz = simd_oprsz(desc); \
2492 int shift = simd_data(desc); \
2493 TYPE *d = vd, *n = vn; \
2494 float_status *fpst = stat; \
2495 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2496 d[i] = FUNC(n[i], shift, fpst); \
2497 } \
2498 clear_tail(d, oprsz, simd_maxsz(desc)); \
2499 }
2500
2501 DO_VCVT_FIXED(gvec_vcvt_sf, helper_vfp_sltos, uint32_t)
2502 DO_VCVT_FIXED(gvec_vcvt_uf, helper_vfp_ultos, uint32_t)
2503 DO_VCVT_FIXED(gvec_vcvt_fs, helper_vfp_tosls_round_to_zero, uint32_t)
2504 DO_VCVT_FIXED(gvec_vcvt_fu, helper_vfp_touls_round_to_zero, uint32_t)
2505 DO_VCVT_FIXED(gvec_vcvt_sh, helper_vfp_shtoh, uint16_t)
2506 DO_VCVT_FIXED(gvec_vcvt_uh, helper_vfp_uhtoh, uint16_t)
2507 DO_VCVT_FIXED(gvec_vcvt_hs, helper_vfp_toshh_round_to_zero, uint16_t)
2508 DO_VCVT_FIXED(gvec_vcvt_hu, helper_vfp_touhh_round_to_zero, uint16_t)
2509
2510 #undef DO_VCVT_FIXED
2511
2512 #define DO_VCVT_RMODE(NAME, FUNC, TYPE) \
2513 void HELPER(NAME)(void *vd, void *vn, void *stat, uint32_t desc) \
2514 { \
2515 float_status *fpst = stat; \
2516 intptr_t i, oprsz = simd_oprsz(desc); \
2517 uint32_t rmode = simd_data(desc); \
2518 uint32_t prev_rmode = get_float_rounding_mode(fpst); \
2519 TYPE *d = vd, *n = vn; \
2520 set_float_rounding_mode(rmode, fpst); \
2521 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2522 d[i] = FUNC(n[i], 0, fpst); \
2523 } \
2524 set_float_rounding_mode(prev_rmode, fpst); \
2525 clear_tail(d, oprsz, simd_maxsz(desc)); \
2526 }
2527
2528 DO_VCVT_RMODE(gvec_vcvt_rm_ss, helper_vfp_tosls, uint32_t)
2529 DO_VCVT_RMODE(gvec_vcvt_rm_us, helper_vfp_touls, uint32_t)
2530 DO_VCVT_RMODE(gvec_vcvt_rm_sh, helper_vfp_toshh, uint16_t)
2531 DO_VCVT_RMODE(gvec_vcvt_rm_uh, helper_vfp_touhh, uint16_t)
2532
2533 #undef DO_VCVT_RMODE
2534
2535 #define DO_VRINT_RMODE(NAME, FUNC, TYPE) \
2536 void HELPER(NAME)(void *vd, void *vn, void *stat, uint32_t desc) \
2537 { \
2538 float_status *fpst = stat; \
2539 intptr_t i, oprsz = simd_oprsz(desc); \
2540 uint32_t rmode = simd_data(desc); \
2541 uint32_t prev_rmode = get_float_rounding_mode(fpst); \
2542 TYPE *d = vd, *n = vn; \
2543 set_float_rounding_mode(rmode, fpst); \
2544 for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2545 d[i] = FUNC(n[i], fpst); \
2546 } \
2547 set_float_rounding_mode(prev_rmode, fpst); \
2548 clear_tail(d, oprsz, simd_maxsz(desc)); \
2549 }
2550
2551 DO_VRINT_RMODE(gvec_vrint_rm_h, helper_rinth, uint16_t)
2552 DO_VRINT_RMODE(gvec_vrint_rm_s, helper_rints, uint32_t)
2553
2554 #undef DO_VRINT_RMODE
2555
2556 #ifdef TARGET_AARCH64
2557 void HELPER(simd_tblx)(void *vd, void *vm, void *venv, uint32_t desc)
2558 {
2559 const uint8_t *indices = vm;
2560 CPUARMState *env = venv;
2561 size_t oprsz = simd_oprsz(desc);
2562 uint32_t rn = extract32(desc, SIMD_DATA_SHIFT, 5);
2563 bool is_tbx = extract32(desc, SIMD_DATA_SHIFT + 5, 1);
2564 uint32_t table_len = desc >> (SIMD_DATA_SHIFT + 6);
2565 union {
2566 uint8_t b[16];
2567 uint64_t d[2];
2568 } result;
2569
2570 /*
2571 * We must construct the final result in a temp, lest the output
2572 * overlaps the input table. For TBL, begin with zero; for TBX,
2573 * begin with the original register contents. Note that we always
2574 * copy 16 bytes here to avoid an extra branch; clearing the high
2575 * bits of the register for oprsz == 8 is handled below.
2576 */
2577 if (is_tbx) {
2578 memcpy(&result, vd, 16);
2579 } else {
2580 memset(&result, 0, 16);
2581 }
2582
2583 for (size_t i = 0; i < oprsz; ++i) {
2584 uint32_t index = indices[H1(i)];
2585
2586 if (index < table_len) {
2587 /*
2588 * Convert index (a byte offset into the virtual table
2589 * which is a series of 128-bit vectors concatenated)
2590 * into the correct register element, bearing in mind
2591 * that the table can wrap around from V31 to V0.
2592 */
2593 const uint8_t *table = (const uint8_t *)
2594 aa64_vfp_qreg(env, (rn + (index >> 4)) % 32);
2595 result.b[H1(i)] = table[H1(index % 16)];
2596 }
2597 }
2598
2599 memcpy(vd, &result, 16);
2600 clear_tail(vd, oprsz, simd_maxsz(desc));
2601 }
2602 #endif
2603
2604 /*
2605 * NxN -> N highpart multiply
2606 *
2607 * TODO: expose this as a generic vector operation.
2608 */
2609
HELPER(gvec_smulh_b)2610 void HELPER(gvec_smulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
2611 {
2612 intptr_t i, opr_sz = simd_oprsz(desc);
2613 int8_t *d = vd, *n = vn, *m = vm;
2614
2615 for (i = 0; i < opr_sz; ++i) {
2616 d[i] = ((int32_t)n[i] * m[i]) >> 8;
2617 }
2618 clear_tail(d, opr_sz, simd_maxsz(desc));
2619 }
2620
HELPER(gvec_smulh_h)2621 void HELPER(gvec_smulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
2622 {
2623 intptr_t i, opr_sz = simd_oprsz(desc);
2624 int16_t *d = vd, *n = vn, *m = vm;
2625
2626 for (i = 0; i < opr_sz / 2; ++i) {
2627 d[i] = ((int32_t)n[i] * m[i]) >> 16;
2628 }
2629 clear_tail(d, opr_sz, simd_maxsz(desc));
2630 }
2631
HELPER(gvec_smulh_s)2632 void HELPER(gvec_smulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
2633 {
2634 intptr_t i, opr_sz = simd_oprsz(desc);
2635 int32_t *d = vd, *n = vn, *m = vm;
2636
2637 for (i = 0; i < opr_sz / 4; ++i) {
2638 d[i] = ((int64_t)n[i] * m[i]) >> 32;
2639 }
2640 clear_tail(d, opr_sz, simd_maxsz(desc));
2641 }
2642
HELPER(gvec_smulh_d)2643 void HELPER(gvec_smulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
2644 {
2645 intptr_t i, opr_sz = simd_oprsz(desc);
2646 uint64_t *d = vd, *n = vn, *m = vm;
2647 uint64_t discard;
2648
2649 for (i = 0; i < opr_sz / 8; ++i) {
2650 muls64(&discard, &d[i], n[i], m[i]);
2651 }
2652 clear_tail(d, opr_sz, simd_maxsz(desc));
2653 }
2654
HELPER(gvec_umulh_b)2655 void HELPER(gvec_umulh_b)(void *vd, void *vn, void *vm, uint32_t desc)
2656 {
2657 intptr_t i, opr_sz = simd_oprsz(desc);
2658 uint8_t *d = vd, *n = vn, *m = vm;
2659
2660 for (i = 0; i < opr_sz; ++i) {
2661 d[i] = ((uint32_t)n[i] * m[i]) >> 8;
2662 }
2663 clear_tail(d, opr_sz, simd_maxsz(desc));
2664 }
2665
HELPER(gvec_umulh_h)2666 void HELPER(gvec_umulh_h)(void *vd, void *vn, void *vm, uint32_t desc)
2667 {
2668 intptr_t i, opr_sz = simd_oprsz(desc);
2669 uint16_t *d = vd, *n = vn, *m = vm;
2670
2671 for (i = 0; i < opr_sz / 2; ++i) {
2672 d[i] = ((uint32_t)n[i] * m[i]) >> 16;
2673 }
2674 clear_tail(d, opr_sz, simd_maxsz(desc));
2675 }
2676
HELPER(gvec_umulh_s)2677 void HELPER(gvec_umulh_s)(void *vd, void *vn, void *vm, uint32_t desc)
2678 {
2679 intptr_t i, opr_sz = simd_oprsz(desc);
2680 uint32_t *d = vd, *n = vn, *m = vm;
2681
2682 for (i = 0; i < opr_sz / 4; ++i) {
2683 d[i] = ((uint64_t)n[i] * m[i]) >> 32;
2684 }
2685 clear_tail(d, opr_sz, simd_maxsz(desc));
2686 }
2687
HELPER(gvec_umulh_d)2688 void HELPER(gvec_umulh_d)(void *vd, void *vn, void *vm, uint32_t desc)
2689 {
2690 intptr_t i, opr_sz = simd_oprsz(desc);
2691 uint64_t *d = vd, *n = vn, *m = vm;
2692 uint64_t discard;
2693
2694 for (i = 0; i < opr_sz / 8; ++i) {
2695 mulu64(&discard, &d[i], n[i], m[i]);
2696 }
2697 clear_tail(d, opr_sz, simd_maxsz(desc));
2698 }
2699
HELPER(gvec_xar_d)2700 void HELPER(gvec_xar_d)(void *vd, void *vn, void *vm, uint32_t desc)
2701 {
2702 intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2703 int shr = simd_data(desc);
2704 uint64_t *d = vd, *n = vn, *m = vm;
2705
2706 for (i = 0; i < opr_sz; ++i) {
2707 d[i] = ror64(n[i] ^ m[i], shr);
2708 }
2709 clear_tail(d, opr_sz * 8, simd_maxsz(desc));
2710 }
2711
2712 /*
2713 * Integer matrix-multiply accumulate
2714 */
2715
do_smmla_b(uint32_t sum,void * vn,void * vm)2716 static uint32_t do_smmla_b(uint32_t sum, void *vn, void *vm)
2717 {
2718 int8_t *n = vn, *m = vm;
2719
2720 for (intptr_t k = 0; k < 8; ++k) {
2721 sum += n[H1(k)] * m[H1(k)];
2722 }
2723 return sum;
2724 }
2725
do_ummla_b(uint32_t sum,void * vn,void * vm)2726 static uint32_t do_ummla_b(uint32_t sum, void *vn, void *vm)
2727 {
2728 uint8_t *n = vn, *m = vm;
2729
2730 for (intptr_t k = 0; k < 8; ++k) {
2731 sum += n[H1(k)] * m[H1(k)];
2732 }
2733 return sum;
2734 }
2735
do_usmmla_b(uint32_t sum,void * vn,void * vm)2736 static uint32_t do_usmmla_b(uint32_t sum, void *vn, void *vm)
2737 {
2738 uint8_t *n = vn;
2739 int8_t *m = vm;
2740
2741 for (intptr_t k = 0; k < 8; ++k) {
2742 sum += n[H1(k)] * m[H1(k)];
2743 }
2744 return sum;
2745 }
2746
do_mmla_b(void * vd,void * vn,void * vm,void * va,uint32_t desc,uint32_t (* inner_loop)(uint32_t,void *,void *))2747 static void do_mmla_b(void *vd, void *vn, void *vm, void *va, uint32_t desc,
2748 uint32_t (*inner_loop)(uint32_t, void *, void *))
2749 {
2750 intptr_t seg, opr_sz = simd_oprsz(desc);
2751
2752 for (seg = 0; seg < opr_sz; seg += 16) {
2753 uint32_t *d = vd + seg;
2754 uint32_t *a = va + seg;
2755 uint32_t sum0, sum1, sum2, sum3;
2756
2757 /*
2758 * Process the entire segment at once, writing back the
2759 * results only after we've consumed all of the inputs.
2760 *
2761 * Key to indices by column:
2762 * i j i j
2763 */
2764 sum0 = a[H4(0 + 0)];
2765 sum0 = inner_loop(sum0, vn + seg + 0, vm + seg + 0);
2766 sum1 = a[H4(0 + 1)];
2767 sum1 = inner_loop(sum1, vn + seg + 0, vm + seg + 8);
2768 sum2 = a[H4(2 + 0)];
2769 sum2 = inner_loop(sum2, vn + seg + 8, vm + seg + 0);
2770 sum3 = a[H4(2 + 1)];
2771 sum3 = inner_loop(sum3, vn + seg + 8, vm + seg + 8);
2772
2773 d[H4(0)] = sum0;
2774 d[H4(1)] = sum1;
2775 d[H4(2)] = sum2;
2776 d[H4(3)] = sum3;
2777 }
2778 clear_tail(vd, opr_sz, simd_maxsz(desc));
2779 }
2780
2781 #define DO_MMLA_B(NAME, INNER) \
2782 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
2783 { do_mmla_b(vd, vn, vm, va, desc, INNER); }
2784
DO_MMLA_B(gvec_smmla_b,do_smmla_b)2785 DO_MMLA_B(gvec_smmla_b, do_smmla_b)
2786 DO_MMLA_B(gvec_ummla_b, do_ummla_b)
2787 DO_MMLA_B(gvec_usmmla_b, do_usmmla_b)
2788
2789 /*
2790 * BFloat16 Dot Product
2791 */
2792
2793 float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2)
2794 {
2795 /* FPCR is ignored for BFDOT and BFMMLA. */
2796 float_status bf_status = {
2797 .tininess_before_rounding = float_tininess_before_rounding,
2798 .float_rounding_mode = float_round_to_odd_inf,
2799 .flush_to_zero = true,
2800 .flush_inputs_to_zero = true,
2801 .default_nan_mode = true,
2802 };
2803 float32 t1, t2;
2804
2805 /*
2806 * Extract each BFloat16 from the element pair, and shift
2807 * them such that they become float32.
2808 */
2809 t1 = float32_mul(e1 << 16, e2 << 16, &bf_status);
2810 t2 = float32_mul(e1 & 0xffff0000u, e2 & 0xffff0000u, &bf_status);
2811 t1 = float32_add(t1, t2, &bf_status);
2812 t1 = float32_add(sum, t1, &bf_status);
2813
2814 return t1;
2815 }
2816
HELPER(gvec_bfdot)2817 void HELPER(gvec_bfdot)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
2818 {
2819 intptr_t i, opr_sz = simd_oprsz(desc);
2820 float32 *d = vd, *a = va;
2821 uint32_t *n = vn, *m = vm;
2822
2823 for (i = 0; i < opr_sz / 4; ++i) {
2824 d[i] = bfdotadd(a[i], n[i], m[i]);
2825 }
2826 clear_tail(d, opr_sz, simd_maxsz(desc));
2827 }
2828
HELPER(gvec_bfdot_idx)2829 void HELPER(gvec_bfdot_idx)(void *vd, void *vn, void *vm,
2830 void *va, uint32_t desc)
2831 {
2832 intptr_t i, j, opr_sz = simd_oprsz(desc);
2833 intptr_t index = simd_data(desc);
2834 intptr_t elements = opr_sz / 4;
2835 intptr_t eltspersegment = MIN(16 / 4, elements);
2836 float32 *d = vd, *a = va;
2837 uint32_t *n = vn, *m = vm;
2838
2839 for (i = 0; i < elements; i += eltspersegment) {
2840 uint32_t m_idx = m[i + H4(index)];
2841
2842 for (j = i; j < i + eltspersegment; j++) {
2843 d[j] = bfdotadd(a[j], n[j], m_idx);
2844 }
2845 }
2846 clear_tail(d, opr_sz, simd_maxsz(desc));
2847 }
2848
HELPER(gvec_bfmmla)2849 void HELPER(gvec_bfmmla)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
2850 {
2851 intptr_t s, opr_sz = simd_oprsz(desc);
2852 float32 *d = vd, *a = va;
2853 uint32_t *n = vn, *m = vm;
2854
2855 for (s = 0; s < opr_sz / 4; s += 4) {
2856 float32 sum00, sum01, sum10, sum11;
2857
2858 /*
2859 * Process the entire segment at once, writing back the
2860 * results only after we've consumed all of the inputs.
2861 *
2862 * Key to indices by column:
2863 * i j i k j k
2864 */
2865 sum00 = a[s + H4(0 + 0)];
2866 sum00 = bfdotadd(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)]);
2867 sum00 = bfdotadd(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)]);
2868
2869 sum01 = a[s + H4(0 + 1)];
2870 sum01 = bfdotadd(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)]);
2871 sum01 = bfdotadd(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)]);
2872
2873 sum10 = a[s + H4(2 + 0)];
2874 sum10 = bfdotadd(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)]);
2875 sum10 = bfdotadd(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)]);
2876
2877 sum11 = a[s + H4(2 + 1)];
2878 sum11 = bfdotadd(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)]);
2879 sum11 = bfdotadd(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)]);
2880
2881 d[s + H4(0 + 0)] = sum00;
2882 d[s + H4(0 + 1)] = sum01;
2883 d[s + H4(2 + 0)] = sum10;
2884 d[s + H4(2 + 1)] = sum11;
2885 }
2886 clear_tail(d, opr_sz, simd_maxsz(desc));
2887 }
2888
HELPER(gvec_bfmlal)2889 void HELPER(gvec_bfmlal)(void *vd, void *vn, void *vm, void *va,
2890 void *stat, uint32_t desc)
2891 {
2892 intptr_t i, opr_sz = simd_oprsz(desc);
2893 intptr_t sel = simd_data(desc);
2894 float32 *d = vd, *a = va;
2895 bfloat16 *n = vn, *m = vm;
2896
2897 for (i = 0; i < opr_sz / 4; ++i) {
2898 float32 nn = n[H2(i * 2 + sel)] << 16;
2899 float32 mm = m[H2(i * 2 + sel)] << 16;
2900 d[H4(i)] = float32_muladd(nn, mm, a[H4(i)], 0, stat);
2901 }
2902 clear_tail(d, opr_sz, simd_maxsz(desc));
2903 }
2904
HELPER(gvec_bfmlal_idx)2905 void HELPER(gvec_bfmlal_idx)(void *vd, void *vn, void *vm,
2906 void *va, void *stat, uint32_t desc)
2907 {
2908 intptr_t i, j, opr_sz = simd_oprsz(desc);
2909 intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1);
2910 intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 1, 3);
2911 intptr_t elements = opr_sz / 4;
2912 intptr_t eltspersegment = MIN(16 / 4, elements);
2913 float32 *d = vd, *a = va;
2914 bfloat16 *n = vn, *m = vm;
2915
2916 for (i = 0; i < elements; i += eltspersegment) {
2917 float32 m_idx = m[H2(2 * i + index)] << 16;
2918
2919 for (j = i; j < i + eltspersegment; j++) {
2920 float32 n_j = n[H2(2 * j + sel)] << 16;
2921 d[H4(j)] = float32_muladd(n_j, m_idx, a[H4(j)], 0, stat);
2922 }
2923 }
2924 clear_tail(d, opr_sz, simd_maxsz(desc));
2925 }
2926
2927 #define DO_CLAMP(NAME, TYPE) \
2928 void HELPER(NAME)(void *d, void *n, void *m, void *a, uint32_t desc) \
2929 { \
2930 intptr_t i, opr_sz = simd_oprsz(desc); \
2931 for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
2932 TYPE aa = *(TYPE *)(a + i); \
2933 TYPE nn = *(TYPE *)(n + i); \
2934 TYPE mm = *(TYPE *)(m + i); \
2935 TYPE dd = MIN(MAX(aa, nn), mm); \
2936 *(TYPE *)(d + i) = dd; \
2937 } \
2938 clear_tail(d, opr_sz, simd_maxsz(desc)); \
2939 }
2940
2941 DO_CLAMP(gvec_sclamp_b, int8_t)
2942 DO_CLAMP(gvec_sclamp_h, int16_t)
2943 DO_CLAMP(gvec_sclamp_s, int32_t)
2944 DO_CLAMP(gvec_sclamp_d, int64_t)
2945
2946 DO_CLAMP(gvec_uclamp_b, uint8_t)
2947 DO_CLAMP(gvec_uclamp_h, uint16_t)
2948 DO_CLAMP(gvec_uclamp_s, uint32_t)
2949 DO_CLAMP(gvec_uclamp_d, uint64_t)
2950