1 /*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
7 * This code is licensed under the GNU GPL v2.
8 */
9 #include "qemu/osdep.h"
10
11 #include "cpu.h"
12 #include "exec/helper-proto.h"
13 #include "fpu/softfloat.h"
14
15 #define SIGNBIT (uint32_t)0x80000000
16 #define SIGNBIT64 ((uint64_t)1 << 63)
17
18 #define SET_QC() env->vfp.qc[0] = 1
19
20 #define NEON_TYPE1(name, type) \
21 typedef struct \
22 { \
23 type v1; \
24 } neon_##name;
25 #ifdef HOST_WORDS_BIGENDIAN
26 #define NEON_TYPE2(name, type) \
27 typedef struct \
28 { \
29 type v2; \
30 type v1; \
31 } neon_##name;
32 #define NEON_TYPE4(name, type) \
33 typedef struct \
34 { \
35 type v4; \
36 type v3; \
37 type v2; \
38 type v1; \
39 } neon_##name;
40 #else
41 #define NEON_TYPE2(name, type) \
42 typedef struct \
43 { \
44 type v1; \
45 type v2; \
46 } neon_##name;
47 #define NEON_TYPE4(name, type) \
48 typedef struct \
49 { \
50 type v1; \
51 type v2; \
52 type v3; \
53 type v4; \
54 } neon_##name;
55 #endif
56
NEON_TYPE4(s8,int8_t)57 NEON_TYPE4(s8, int8_t)
58 NEON_TYPE4(u8, uint8_t)
59 NEON_TYPE2(s16, int16_t)
60 NEON_TYPE2(u16, uint16_t)
61 NEON_TYPE1(s32, int32_t)
62 NEON_TYPE1(u32, uint32_t)
63 #undef NEON_TYPE4
64 #undef NEON_TYPE2
65 #undef NEON_TYPE1
66
67 /* Copy from a uint32_t to a vector structure type. */
68 #define NEON_UNPACK(vtype, dest, val) do { \
69 union { \
70 vtype v; \
71 uint32_t i; \
72 } conv_u; \
73 conv_u.i = (val); \
74 dest = conv_u.v; \
75 } while(0)
76
77 /* Copy from a vector structure type to a uint32_t. */
78 #define NEON_PACK(vtype, dest, val) do { \
79 union { \
80 vtype v; \
81 uint32_t i; \
82 } conv_u; \
83 conv_u.v = (val); \
84 dest = conv_u.i; \
85 } while(0)
86
87 #define NEON_DO1 \
88 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
89 #define NEON_DO2 \
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
91 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
92 #define NEON_DO4 \
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
95 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
96 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
97
98 #define NEON_VOP_BODY(vtype, n) \
99 { \
100 uint32_t res; \
101 vtype vsrc1; \
102 vtype vsrc2; \
103 vtype vdest; \
104 NEON_UNPACK(vtype, vsrc1, arg1); \
105 NEON_UNPACK(vtype, vsrc2, arg2); \
106 NEON_DO##n; \
107 NEON_PACK(vtype, res, vdest); \
108 return res; \
109 }
110
111 #define NEON_VOP(name, vtype, n) \
112 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
113 NEON_VOP_BODY(vtype, n)
114
115 #define NEON_VOP_ENV(name, vtype, n) \
116 uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
117 NEON_VOP_BODY(vtype, n)
118
119 /* Pairwise operations. */
120 /* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
122 #define NEON_PDO2 \
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
125 #define NEON_PDO4 \
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
130
131 #define NEON_POP(name, vtype, n) \
132 uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
133 { \
134 uint32_t res; \
135 vtype vsrc1; \
136 vtype vsrc2; \
137 vtype vdest; \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
140 NEON_PDO##n; \
141 NEON_PACK(vtype, res, vdest); \
142 return res; \
143 }
144
145 /* Unary operators. */
146 #define NEON_VOP1(name, vtype, n) \
147 uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
148 { \
149 vtype vsrc1; \
150 vtype vdest; \
151 NEON_UNPACK(vtype, vsrc1, arg); \
152 NEON_DO##n; \
153 NEON_PACK(vtype, arg, vdest); \
154 return arg; \
155 }
156
157
158 #define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
161 SET_QC(); \
162 dest = ~0; \
163 } else { \
164 dest = tmp; \
165 }} while(0)
166 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
167 NEON_VOP_ENV(qadd_u8, neon_u8, 4)
168 #undef NEON_FN
169 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
170 NEON_VOP_ENV(qadd_u16, neon_u16, 2)
171 #undef NEON_FN
172 #undef NEON_USAT
173
174 uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
175 {
176 uint32_t res = a + b;
177 if (res < a) {
178 SET_QC();
179 res = ~0;
180 }
181 return res;
182 }
183
HELPER(neon_qadd_u64)184 uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
185 {
186 uint64_t res;
187
188 res = src1 + src2;
189 if (res < src1) {
190 SET_QC();
191 res = ~(uint64_t)0;
192 }
193 return res;
194 }
195
196 #define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
199 SET_QC(); \
200 if (src2 > 0) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
202 } else { \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
204 } \
205 } \
206 dest = tmp; \
207 } while(0)
208 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
209 NEON_VOP_ENV(qadd_s8, neon_s8, 4)
210 #undef NEON_FN
211 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
212 NEON_VOP_ENV(qadd_s16, neon_s16, 2)
213 #undef NEON_FN
214 #undef NEON_SSAT
215
HELPER(neon_qadd_s32)216 uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
217 {
218 uint32_t res = a + b;
219 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
220 SET_QC();
221 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
222 }
223 return res;
224 }
225
HELPER(neon_qadd_s64)226 uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
227 {
228 uint64_t res;
229
230 res = src1 + src2;
231 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
232 SET_QC();
233 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
234 }
235 return res;
236 }
237
238 /* Unsigned saturating accumulate of signed value
239 *
240 * Op1/Rn is treated as signed
241 * Op2/Rd is treated as unsigned
242 *
243 * Explicit casting is used to ensure the correct sign extension of
244 * inputs. The result is treated as a unsigned value and saturated as such.
245 *
246 * We use a macro for the 8/16 bit cases which expects signed integers of va,
247 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
248 */
249
250 #define USATACC(bits, shift) \
251 do { \
252 va = sextract32(a, shift, bits); \
253 vb = extract32(b, shift, bits); \
254 vr = va + vb; \
255 if (vr > UINT##bits##_MAX) { \
256 SET_QC(); \
257 vr = UINT##bits##_MAX; \
258 } else if (vr < 0) { \
259 SET_QC(); \
260 vr = 0; \
261 } \
262 r = deposit32(r, shift, bits, vr); \
263 } while (0)
264
HELPER(neon_uqadd_s8)265 uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
266 {
267 int16_t va, vb, vr;
268 uint32_t r = 0;
269
270 USATACC(8, 0);
271 USATACC(8, 8);
272 USATACC(8, 16);
273 USATACC(8, 24);
274 return r;
275 }
276
HELPER(neon_uqadd_s16)277 uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
278 {
279 int32_t va, vb, vr;
280 uint64_t r = 0;
281
282 USATACC(16, 0);
283 USATACC(16, 16);
284 return r;
285 }
286
287 #undef USATACC
288
HELPER(neon_uqadd_s32)289 uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
290 {
291 int64_t va = (int32_t)a;
292 int64_t vb = (uint32_t)b;
293 int64_t vr = va + vb;
294 if (vr > UINT32_MAX) {
295 SET_QC();
296 vr = UINT32_MAX;
297 } else if (vr < 0) {
298 SET_QC();
299 vr = 0;
300 }
301 return vr;
302 }
303
HELPER(neon_uqadd_s64)304 uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
305 {
306 uint64_t res;
307 res = a + b;
308 /* We only need to look at the pattern of SIGN bits to detect
309 * +ve/-ve saturation
310 */
311 if (~a & b & ~res & SIGNBIT64) {
312 SET_QC();
313 res = UINT64_MAX;
314 } else if (a & ~b & res & SIGNBIT64) {
315 SET_QC();
316 res = 0;
317 }
318 return res;
319 }
320
321 /* Signed saturating accumulate of unsigned value
322 *
323 * Op1/Rn is treated as unsigned
324 * Op2/Rd is treated as signed
325 *
326 * The result is treated as a signed value and saturated as such
327 *
328 * We use a macro for the 8/16 bit cases which expects signed integers of va,
329 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
330 */
331
332 #define SSATACC(bits, shift) \
333 do { \
334 va = extract32(a, shift, bits); \
335 vb = sextract32(b, shift, bits); \
336 vr = va + vb; \
337 if (vr > INT##bits##_MAX) { \
338 SET_QC(); \
339 vr = INT##bits##_MAX; \
340 } else if (vr < INT##bits##_MIN) { \
341 SET_QC(); \
342 vr = INT##bits##_MIN; \
343 } \
344 r = deposit32(r, shift, bits, vr); \
345 } while (0)
346
HELPER(neon_sqadd_u8)347 uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
348 {
349 int16_t va, vb, vr;
350 uint32_t r = 0;
351
352 SSATACC(8, 0);
353 SSATACC(8, 8);
354 SSATACC(8, 16);
355 SSATACC(8, 24);
356 return r;
357 }
358
HELPER(neon_sqadd_u16)359 uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
360 {
361 int32_t va, vb, vr;
362 uint32_t r = 0;
363
364 SSATACC(16, 0);
365 SSATACC(16, 16);
366
367 return r;
368 }
369
370 #undef SSATACC
371
HELPER(neon_sqadd_u32)372 uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
373 {
374 int64_t res;
375 int64_t op1 = (uint32_t)a;
376 int64_t op2 = (int32_t)b;
377 res = op1 + op2;
378 if (res > INT32_MAX) {
379 SET_QC();
380 res = INT32_MAX;
381 } else if (res < INT32_MIN) {
382 SET_QC();
383 res = INT32_MIN;
384 }
385 return res;
386 }
387
HELPER(neon_sqadd_u64)388 uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
389 {
390 uint64_t res;
391 res = a + b;
392 /* We only need to look at the pattern of SIGN bits to detect an overflow */
393 if (((a & res)
394 | (~b & res)
395 | (a & ~b)) & SIGNBIT64) {
396 SET_QC();
397 res = INT64_MAX;
398 }
399 return res;
400 }
401
402
403 #define NEON_USAT(dest, src1, src2, type) do { \
404 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
405 if (tmp != (type)tmp) { \
406 SET_QC(); \
407 dest = 0; \
408 } else { \
409 dest = tmp; \
410 }} while(0)
411 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
412 NEON_VOP_ENV(qsub_u8, neon_u8, 4)
413 #undef NEON_FN
414 #define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
415 NEON_VOP_ENV(qsub_u16, neon_u16, 2)
416 #undef NEON_FN
417 #undef NEON_USAT
418
HELPER(neon_qsub_u32)419 uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
420 {
421 uint32_t res = a - b;
422 if (res > a) {
423 SET_QC();
424 res = 0;
425 }
426 return res;
427 }
428
HELPER(neon_qsub_u64)429 uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
430 {
431 uint64_t res;
432
433 if (src1 < src2) {
434 SET_QC();
435 res = 0;
436 } else {
437 res = src1 - src2;
438 }
439 return res;
440 }
441
442 #define NEON_SSAT(dest, src1, src2, type) do { \
443 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
444 if (tmp != (type)tmp) { \
445 SET_QC(); \
446 if (src2 < 0) { \
447 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
448 } else { \
449 tmp = 1 << (sizeof(type) * 8 - 1); \
450 } \
451 } \
452 dest = tmp; \
453 } while(0)
454 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
455 NEON_VOP_ENV(qsub_s8, neon_s8, 4)
456 #undef NEON_FN
457 #define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
458 NEON_VOP_ENV(qsub_s16, neon_s16, 2)
459 #undef NEON_FN
460 #undef NEON_SSAT
461
HELPER(neon_qsub_s32)462 uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
463 {
464 uint32_t res = a - b;
465 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
466 SET_QC();
467 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
468 }
469 return res;
470 }
471
HELPER(neon_qsub_s64)472 uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
473 {
474 uint64_t res;
475
476 res = src1 - src2;
477 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
478 SET_QC();
479 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
480 }
481 return res;
482 }
483
484 #define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
485 NEON_VOP(hadd_s8, neon_s8, 4)
486 NEON_VOP(hadd_u8, neon_u8, 4)
487 NEON_VOP(hadd_s16, neon_s16, 2)
488 NEON_VOP(hadd_u16, neon_u16, 2)
489 #undef NEON_FN
490
HELPER(neon_hadd_s32)491 int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
492 {
493 int32_t dest;
494
495 dest = (src1 >> 1) + (src2 >> 1);
496 if (src1 & src2 & 1)
497 dest++;
498 return dest;
499 }
500
HELPER(neon_hadd_u32)501 uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
502 {
503 uint32_t dest;
504
505 dest = (src1 >> 1) + (src2 >> 1);
506 if (src1 & src2 & 1)
507 dest++;
508 return dest;
509 }
510
511 #define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
512 NEON_VOP(rhadd_s8, neon_s8, 4)
513 NEON_VOP(rhadd_u8, neon_u8, 4)
514 NEON_VOP(rhadd_s16, neon_s16, 2)
515 NEON_VOP(rhadd_u16, neon_u16, 2)
516 #undef NEON_FN
517
HELPER(neon_rhadd_s32)518 int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
519 {
520 int32_t dest;
521
522 dest = (src1 >> 1) + (src2 >> 1);
523 if ((src1 | src2) & 1)
524 dest++;
525 return dest;
526 }
527
HELPER(neon_rhadd_u32)528 uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
529 {
530 uint32_t dest;
531
532 dest = (src1 >> 1) + (src2 >> 1);
533 if ((src1 | src2) & 1)
534 dest++;
535 return dest;
536 }
537
538 #define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
539 NEON_VOP(hsub_s8, neon_s8, 4)
540 NEON_VOP(hsub_u8, neon_u8, 4)
541 NEON_VOP(hsub_s16, neon_s16, 2)
542 NEON_VOP(hsub_u16, neon_u16, 2)
543 #undef NEON_FN
544
HELPER(neon_hsub_s32)545 int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
546 {
547 int32_t dest;
548
549 dest = (src1 >> 1) - (src2 >> 1);
550 if ((~src1) & src2 & 1)
551 dest--;
552 return dest;
553 }
554
HELPER(neon_hsub_u32)555 uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
556 {
557 uint32_t dest;
558
559 dest = (src1 >> 1) - (src2 >> 1);
560 if ((~src1) & src2 & 1)
561 dest--;
562 return dest;
563 }
564
565 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
566 NEON_VOP(cgt_s8, neon_s8, 4)
567 NEON_VOP(cgt_u8, neon_u8, 4)
568 NEON_VOP(cgt_s16, neon_s16, 2)
569 NEON_VOP(cgt_u16, neon_u16, 2)
570 NEON_VOP(cgt_s32, neon_s32, 1)
571 NEON_VOP(cgt_u32, neon_u32, 1)
572 #undef NEON_FN
573
574 #define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
575 NEON_VOP(cge_s8, neon_s8, 4)
576 NEON_VOP(cge_u8, neon_u8, 4)
577 NEON_VOP(cge_s16, neon_s16, 2)
578 NEON_VOP(cge_u16, neon_u16, 2)
579 NEON_VOP(cge_s32, neon_s32, 1)
580 NEON_VOP(cge_u32, neon_u32, 1)
581 #undef NEON_FN
582
583 #define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
584 NEON_POP(pmin_s8, neon_s8, 4)
585 NEON_POP(pmin_u8, neon_u8, 4)
586 NEON_POP(pmin_s16, neon_s16, 2)
587 NEON_POP(pmin_u16, neon_u16, 2)
588 #undef NEON_FN
589
590 #define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
591 NEON_POP(pmax_s8, neon_s8, 4)
592 NEON_POP(pmax_u8, neon_u8, 4)
593 NEON_POP(pmax_s16, neon_s16, 2)
594 NEON_POP(pmax_u16, neon_u16, 2)
595 #undef NEON_FN
596
597 #define NEON_FN(dest, src1, src2) \
598 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
599 NEON_VOP(abd_s8, neon_s8, 4)
600 NEON_VOP(abd_u8, neon_u8, 4)
601 NEON_VOP(abd_s16, neon_s16, 2)
602 NEON_VOP(abd_u16, neon_u16, 2)
603 NEON_VOP(abd_s32, neon_s32, 1)
604 NEON_VOP(abd_u32, neon_u32, 1)
605 #undef NEON_FN
606
607 #define NEON_FN(dest, src1, src2) do { \
608 int8_t tmp; \
609 tmp = (int8_t)src2; \
610 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
611 tmp <= -(ssize_t)sizeof(src1) * 8) { \
612 dest = 0; \
613 } else if (tmp < 0) { \
614 dest = src1 >> -tmp; \
615 } else { \
616 dest = src1 << tmp; \
617 }} while (0)
618 NEON_VOP(shl_u8, neon_u8, 4)
619 NEON_VOP(shl_u16, neon_u16, 2)
620 NEON_VOP(shl_u32, neon_u32, 1)
621 #undef NEON_FN
622
HELPER(neon_shl_u64)623 uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
624 {
625 int8_t shift = (int8_t)shiftop;
626 if (shift >= 64 || shift <= -64) {
627 val = 0;
628 } else if (shift < 0) {
629 val >>= -shift;
630 } else {
631 val <<= shift;
632 }
633 return val;
634 }
635
636 #define NEON_FN(dest, src1, src2) do { \
637 int8_t tmp; \
638 tmp = (int8_t)src2; \
639 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
640 dest = 0; \
641 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
642 dest = src1 >> (sizeof(src1) * 8 - 1); \
643 } else if (tmp < 0) { \
644 dest = src1 >> -tmp; \
645 } else { \
646 dest = src1 << tmp; \
647 }} while (0)
648 NEON_VOP(shl_s8, neon_s8, 4)
649 NEON_VOP(shl_s16, neon_s16, 2)
650 NEON_VOP(shl_s32, neon_s32, 1)
651 #undef NEON_FN
652
HELPER(neon_shl_s64)653 uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
654 {
655 int8_t shift = (int8_t)shiftop;
656 int64_t val = valop;
657 if (shift >= 64) {
658 val = 0;
659 } else if (shift <= -64) {
660 val >>= 63;
661 } else if (shift < 0) {
662 val >>= -shift;
663 } else {
664 val <<= shift;
665 }
666 return val;
667 }
668
669 #define NEON_FN(dest, src1, src2) do { \
670 int8_t tmp; \
671 tmp = (int8_t)src2; \
672 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
673 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
674 dest = 0; \
675 } else if (tmp < 0) { \
676 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
677 } else { \
678 dest = src1 << tmp; \
679 }} while (0)
680 NEON_VOP(rshl_s8, neon_s8, 4)
681 NEON_VOP(rshl_s16, neon_s16, 2)
682 #undef NEON_FN
683
684 /* The addition of the rounding constant may overflow, so we use an
685 * intermediate 64 bit accumulator. */
HELPER(neon_rshl_s32)686 uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
687 {
688 int32_t dest;
689 int32_t val = (int32_t)valop;
690 int8_t shift = (int8_t)shiftop;
691 if ((shift >= 32) || (shift <= -32)) {
692 dest = 0;
693 } else if (shift < 0) {
694 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
695 dest = big_dest >> -shift;
696 } else {
697 dest = val << shift;
698 }
699 return dest;
700 }
701
702 /* Handling addition overflow with 64 bit input values is more
703 * tricky than with 32 bit values. */
HELPER(neon_rshl_s64)704 uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
705 {
706 int8_t shift = (int8_t)shiftop;
707 int64_t val = valop;
708 if ((shift >= 64) || (shift <= -64)) {
709 val = 0;
710 } else if (shift < 0) {
711 val >>= (-shift - 1);
712 if (val == INT64_MAX) {
713 /* In this case, it means that the rounding constant is 1,
714 * and the addition would overflow. Return the actual
715 * result directly. */
716 val = 0x4000000000000000LL;
717 } else {
718 val++;
719 val >>= 1;
720 }
721 } else {
722 val <<= shift;
723 }
724 return val;
725 }
726
727 #define NEON_FN(dest, src1, src2) do { \
728 int8_t tmp; \
729 tmp = (int8_t)src2; \
730 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
731 tmp < -(ssize_t)sizeof(src1) * 8) { \
732 dest = 0; \
733 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
734 dest = src1 >> (-tmp - 1); \
735 } else if (tmp < 0) { \
736 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
737 } else { \
738 dest = src1 << tmp; \
739 }} while (0)
740 NEON_VOP(rshl_u8, neon_u8, 4)
741 NEON_VOP(rshl_u16, neon_u16, 2)
742 #undef NEON_FN
743
744 /* The addition of the rounding constant may overflow, so we use an
745 * intermediate 64 bit accumulator. */
HELPER(neon_rshl_u32)746 uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
747 {
748 uint32_t dest;
749 int8_t shift = (int8_t)shiftop;
750 if (shift >= 32 || shift < -32) {
751 dest = 0;
752 } else if (shift == -32) {
753 dest = val >> 31;
754 } else if (shift < 0) {
755 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
756 dest = big_dest >> -shift;
757 } else {
758 dest = val << shift;
759 }
760 return dest;
761 }
762
763 /* Handling addition overflow with 64 bit input values is more
764 * tricky than with 32 bit values. */
HELPER(neon_rshl_u64)765 uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
766 {
767 int8_t shift = (uint8_t)shiftop;
768 if (shift >= 64 || shift < -64) {
769 val = 0;
770 } else if (shift == -64) {
771 /* Rounding a 1-bit result just preserves that bit. */
772 val >>= 63;
773 } else if (shift < 0) {
774 val >>= (-shift - 1);
775 if (val == UINT64_MAX) {
776 /* In this case, it means that the rounding constant is 1,
777 * and the addition would overflow. Return the actual
778 * result directly. */
779 val = 0x8000000000000000ULL;
780 } else {
781 val++;
782 val >>= 1;
783 }
784 } else {
785 val <<= shift;
786 }
787 return val;
788 }
789
790 #define NEON_FN(dest, src1, src2) do { \
791 int8_t tmp; \
792 tmp = (int8_t)src2; \
793 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
794 if (src1) { \
795 SET_QC(); \
796 dest = ~0; \
797 } else { \
798 dest = 0; \
799 } \
800 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
801 dest = 0; \
802 } else if (tmp < 0) { \
803 dest = src1 >> -tmp; \
804 } else { \
805 dest = src1 << tmp; \
806 if ((dest >> tmp) != src1) { \
807 SET_QC(); \
808 dest = ~0; \
809 } \
810 }} while (0)
811 NEON_VOP_ENV(qshl_u8, neon_u8, 4)
812 NEON_VOP_ENV(qshl_u16, neon_u16, 2)
813 NEON_VOP_ENV(qshl_u32, neon_u32, 1)
814 #undef NEON_FN
815
HELPER(neon_qshl_u64)816 uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
817 {
818 int8_t shift = (int8_t)shiftop;
819 if (shift >= 64) {
820 if (val) {
821 val = ~(uint64_t)0;
822 SET_QC();
823 }
824 } else if (shift <= -64) {
825 val = 0;
826 } else if (shift < 0) {
827 val >>= -shift;
828 } else {
829 uint64_t tmp = val;
830 val <<= shift;
831 if ((val >> shift) != tmp) {
832 SET_QC();
833 val = ~(uint64_t)0;
834 }
835 }
836 return val;
837 }
838
839 #define NEON_FN(dest, src1, src2) do { \
840 int8_t tmp; \
841 tmp = (int8_t)src2; \
842 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
843 if (src1) { \
844 SET_QC(); \
845 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
846 if (src1 > 0) { \
847 dest--; \
848 } \
849 } else { \
850 dest = src1; \
851 } \
852 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
853 dest = src1 >> 31; \
854 } else if (tmp < 0) { \
855 dest = src1 >> -tmp; \
856 } else { \
857 dest = src1 << tmp; \
858 if ((dest >> tmp) != src1) { \
859 SET_QC(); \
860 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
861 if (src1 > 0) { \
862 dest--; \
863 } \
864 } \
865 }} while (0)
866 NEON_VOP_ENV(qshl_s8, neon_s8, 4)
867 NEON_VOP_ENV(qshl_s16, neon_s16, 2)
868 NEON_VOP_ENV(qshl_s32, neon_s32, 1)
869 #undef NEON_FN
870
HELPER(neon_qshl_s64)871 uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
872 {
873 int8_t shift = (uint8_t)shiftop;
874 int64_t val = valop;
875 if (shift >= 64) {
876 if (val) {
877 SET_QC();
878 val = (val >> 63) ^ ~SIGNBIT64;
879 }
880 } else if (shift <= -64) {
881 val >>= 63;
882 } else if (shift < 0) {
883 val >>= -shift;
884 } else {
885 int64_t tmp = val;
886 val <<= shift;
887 if ((val >> shift) != tmp) {
888 SET_QC();
889 val = (tmp >> 63) ^ ~SIGNBIT64;
890 }
891 }
892 return val;
893 }
894
895 #define NEON_FN(dest, src1, src2) do { \
896 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
897 SET_QC(); \
898 dest = 0; \
899 } else { \
900 int8_t tmp; \
901 tmp = (int8_t)src2; \
902 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
903 if (src1) { \
904 SET_QC(); \
905 dest = ~0; \
906 } else { \
907 dest = 0; \
908 } \
909 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
910 dest = 0; \
911 } else if (tmp < 0) { \
912 dest = src1 >> -tmp; \
913 } else { \
914 dest = src1 << tmp; \
915 if ((dest >> tmp) != src1) { \
916 SET_QC(); \
917 dest = ~0; \
918 } \
919 } \
920 }} while (0)
921 NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
922 NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
923 #undef NEON_FN
924
HELPER(neon_qshlu_s32)925 uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
926 {
927 if ((int32_t)valop < 0) {
928 SET_QC();
929 return 0;
930 }
931 return helper_neon_qshl_u32(env, valop, shiftop);
932 }
933
HELPER(neon_qshlu_s64)934 uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
935 {
936 if ((int64_t)valop < 0) {
937 SET_QC();
938 return 0;
939 }
940 return helper_neon_qshl_u64(env, valop, shiftop);
941 }
942
943 #define NEON_FN(dest, src1, src2) do { \
944 int8_t tmp; \
945 tmp = (int8_t)src2; \
946 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
947 if (src1) { \
948 SET_QC(); \
949 dest = ~0; \
950 } else { \
951 dest = 0; \
952 } \
953 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
954 dest = 0; \
955 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
956 dest = src1 >> (sizeof(src1) * 8 - 1); \
957 } else if (tmp < 0) { \
958 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
959 } else { \
960 dest = src1 << tmp; \
961 if ((dest >> tmp) != src1) { \
962 SET_QC(); \
963 dest = ~0; \
964 } \
965 }} while (0)
966 NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
967 NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
968 #undef NEON_FN
969
970 /* The addition of the rounding constant may overflow, so we use an
971 * intermediate 64 bit accumulator. */
HELPER(neon_qrshl_u32)972 uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
973 {
974 uint32_t dest;
975 int8_t shift = (int8_t)shiftop;
976 if (shift >= 32) {
977 if (val) {
978 SET_QC();
979 dest = ~0;
980 } else {
981 dest = 0;
982 }
983 } else if (shift < -32) {
984 dest = 0;
985 } else if (shift == -32) {
986 dest = val >> 31;
987 } else if (shift < 0) {
988 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
989 dest = big_dest >> -shift;
990 } else {
991 dest = val << shift;
992 if ((dest >> shift) != val) {
993 SET_QC();
994 dest = ~0;
995 }
996 }
997 return dest;
998 }
999
1000 /* Handling addition overflow with 64 bit input values is more
1001 * tricky than with 32 bit values. */
HELPER(neon_qrshl_u64)1002 uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
1003 {
1004 int8_t shift = (int8_t)shiftop;
1005 if (shift >= 64) {
1006 if (val) {
1007 SET_QC();
1008 val = ~0;
1009 }
1010 } else if (shift < -64) {
1011 val = 0;
1012 } else if (shift == -64) {
1013 val >>= 63;
1014 } else if (shift < 0) {
1015 val >>= (-shift - 1);
1016 if (val == UINT64_MAX) {
1017 /* In this case, it means that the rounding constant is 1,
1018 * and the addition would overflow. Return the actual
1019 * result directly. */
1020 val = 0x8000000000000000ULL;
1021 } else {
1022 val++;
1023 val >>= 1;
1024 }
1025 } else { \
1026 uint64_t tmp = val;
1027 val <<= shift;
1028 if ((val >> shift) != tmp) {
1029 SET_QC();
1030 val = ~0;
1031 }
1032 }
1033 return val;
1034 }
1035
1036 #define NEON_FN(dest, src1, src2) do { \
1037 int8_t tmp; \
1038 tmp = (int8_t)src2; \
1039 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1040 if (src1) { \
1041 SET_QC(); \
1042 dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
1043 if (src1 > 0) { \
1044 dest--; \
1045 } \
1046 } else { \
1047 dest = 0; \
1048 } \
1049 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1050 dest = 0; \
1051 } else if (tmp < 0) { \
1052 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1053 } else { \
1054 dest = src1 << tmp; \
1055 if ((dest >> tmp) != src1) { \
1056 SET_QC(); \
1057 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1058 if (src1 > 0) { \
1059 dest--; \
1060 } \
1061 } \
1062 }} while (0)
1063 NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1064 NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
1065 #undef NEON_FN
1066
1067 /* The addition of the rounding constant may overflow, so we use an
1068 * intermediate 64 bit accumulator. */
HELPER(neon_qrshl_s32)1069 uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
1070 {
1071 int32_t dest;
1072 int32_t val = (int32_t)valop;
1073 int8_t shift = (int8_t)shiftop;
1074 if (shift >= 32) {
1075 if (val) {
1076 SET_QC();
1077 dest = (val >> 31) ^ ~SIGNBIT;
1078 } else {
1079 dest = 0;
1080 }
1081 } else if (shift <= -32) {
1082 dest = 0;
1083 } else if (shift < 0) {
1084 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1085 dest = big_dest >> -shift;
1086 } else {
1087 dest = val << shift;
1088 if ((dest >> shift) != val) {
1089 SET_QC();
1090 dest = (val >> 31) ^ ~SIGNBIT;
1091 }
1092 }
1093 return dest;
1094 }
1095
1096 /* Handling addition overflow with 64 bit input values is more
1097 * tricky than with 32 bit values. */
HELPER(neon_qrshl_s64)1098 uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
1099 {
1100 int8_t shift = (uint8_t)shiftop;
1101 int64_t val = valop;
1102
1103 if (shift >= 64) {
1104 if (val) {
1105 SET_QC();
1106 val = (val >> 63) ^ ~SIGNBIT64;
1107 }
1108 } else if (shift <= -64) {
1109 val = 0;
1110 } else if (shift < 0) {
1111 val >>= (-shift - 1);
1112 if (val == INT64_MAX) {
1113 /* In this case, it means that the rounding constant is 1,
1114 * and the addition would overflow. Return the actual
1115 * result directly. */
1116 val = 0x4000000000000000ULL;
1117 } else {
1118 val++;
1119 val >>= 1;
1120 }
1121 } else {
1122 int64_t tmp = val;
1123 val <<= shift;
1124 if ((val >> shift) != tmp) {
1125 SET_QC();
1126 val = (tmp >> 63) ^ ~SIGNBIT64;
1127 }
1128 }
1129 return val;
1130 }
1131
HELPER(neon_add_u8)1132 uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1133 {
1134 uint32_t mask;
1135 mask = (a ^ b) & 0x80808080u;
1136 a &= ~0x80808080u;
1137 b &= ~0x80808080u;
1138 return (a + b) ^ mask;
1139 }
1140
HELPER(neon_add_u16)1141 uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1142 {
1143 uint32_t mask;
1144 mask = (a ^ b) & 0x80008000u;
1145 a &= ~0x80008000u;
1146 b &= ~0x80008000u;
1147 return (a + b) ^ mask;
1148 }
1149
1150 #define NEON_FN(dest, src1, src2) dest = src1 + src2
1151 NEON_POP(padd_u8, neon_u8, 4)
1152 NEON_POP(padd_u16, neon_u16, 2)
1153 #undef NEON_FN
1154
1155 #define NEON_FN(dest, src1, src2) dest = src1 - src2
1156 NEON_VOP(sub_u8, neon_u8, 4)
1157 NEON_VOP(sub_u16, neon_u16, 2)
1158 #undef NEON_FN
1159
1160 #define NEON_FN(dest, src1, src2) dest = src1 * src2
1161 NEON_VOP(mul_u8, neon_u8, 4)
1162 NEON_VOP(mul_u16, neon_u16, 2)
1163 #undef NEON_FN
1164
1165 /* Polynomial multiplication is like integer multiplication except the
1166 partial products are XORed, not added. */
HELPER(neon_mul_p8)1167 uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1168 {
1169 uint32_t mask;
1170 uint32_t result;
1171 result = 0;
1172 while (op1) {
1173 mask = 0;
1174 if (op1 & 1)
1175 mask |= 0xff;
1176 if (op1 & (1 << 8))
1177 mask |= (0xff << 8);
1178 if (op1 & (1 << 16))
1179 mask |= (0xff << 16);
1180 if (op1 & (1 << 24))
1181 mask |= (0xff << 24);
1182 result ^= op2 & mask;
1183 op1 = (op1 >> 1) & 0x7f7f7f7f;
1184 op2 = (op2 << 1) & 0xfefefefe;
1185 }
1186 return result;
1187 }
1188
HELPER(neon_mull_p8)1189 uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1190 {
1191 uint64_t result = 0;
1192 uint64_t mask;
1193 uint64_t op2ex = op2;
1194 op2ex = (op2ex & 0xff) |
1195 ((op2ex & 0xff00) << 8) |
1196 ((op2ex & 0xff0000) << 16) |
1197 ((op2ex & 0xff000000) << 24);
1198 while (op1) {
1199 mask = 0;
1200 if (op1 & 1) {
1201 mask |= 0xffff;
1202 }
1203 if (op1 & (1 << 8)) {
1204 mask |= (0xffffU << 16);
1205 }
1206 if (op1 & (1 << 16)) {
1207 mask |= (0xffffULL << 32);
1208 }
1209 if (op1 & (1 << 24)) {
1210 mask |= (0xffffULL << 48);
1211 }
1212 result ^= op2ex & mask;
1213 op1 = (op1 >> 1) & 0x7f7f7f7f;
1214 op2ex <<= 1;
1215 }
1216 return result;
1217 }
1218
1219 #define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1220 NEON_VOP(tst_u8, neon_u8, 4)
1221 NEON_VOP(tst_u16, neon_u16, 2)
1222 NEON_VOP(tst_u32, neon_u32, 1)
1223 #undef NEON_FN
1224
1225 #define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1226 NEON_VOP(ceq_u8, neon_u8, 4)
1227 NEON_VOP(ceq_u16, neon_u16, 2)
1228 NEON_VOP(ceq_u32, neon_u32, 1)
1229 #undef NEON_FN
1230
1231 /* Count Leading Sign/Zero Bits. */
do_clz8(uint8_t x)1232 static inline int do_clz8(uint8_t x)
1233 {
1234 int n;
1235 for (n = 8; x; n--)
1236 x >>= 1;
1237 return n;
1238 }
1239
do_clz16(uint16_t x)1240 static inline int do_clz16(uint16_t x)
1241 {
1242 int n;
1243 for (n = 16; x; n--)
1244 x >>= 1;
1245 return n;
1246 }
1247
1248 #define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1249 NEON_VOP1(clz_u8, neon_u8, 4)
1250 #undef NEON_FN
1251
1252 #define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1253 NEON_VOP1(clz_u16, neon_u16, 2)
1254 #undef NEON_FN
1255
1256 #define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1257 NEON_VOP1(cls_s8, neon_s8, 4)
1258 #undef NEON_FN
1259
1260 #define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1261 NEON_VOP1(cls_s16, neon_s16, 2)
1262 #undef NEON_FN
1263
HELPER(neon_cls_s32)1264 uint32_t HELPER(neon_cls_s32)(uint32_t x)
1265 {
1266 int count;
1267 if ((int32_t)x < 0)
1268 x = ~x;
1269 for (count = 32; x; count--)
1270 x = x >> 1;
1271 return count - 1;
1272 }
1273
1274 /* Bit count. */
HELPER(neon_cnt_u8)1275 uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1276 {
1277 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1278 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1279 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1280 return x;
1281 }
1282
1283 /* Reverse bits in each 8 bit word */
HELPER(neon_rbit_u8)1284 uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1285 {
1286 x = ((x & 0xf0f0f0f0) >> 4)
1287 | ((x & 0x0f0f0f0f) << 4);
1288 x = ((x & 0x88888888) >> 3)
1289 | ((x & 0x44444444) >> 1)
1290 | ((x & 0x22222222) << 1)
1291 | ((x & 0x11111111) << 3);
1292 return x;
1293 }
1294
1295 #define NEON_QDMULH16(dest, src1, src2, round) do { \
1296 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1297 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1298 SET_QC(); \
1299 tmp = (tmp >> 31) ^ ~SIGNBIT; \
1300 } else { \
1301 tmp <<= 1; \
1302 } \
1303 if (round) { \
1304 int32_t old = tmp; \
1305 tmp += 1 << 15; \
1306 if ((int32_t)tmp < old) { \
1307 SET_QC(); \
1308 tmp = SIGNBIT - 1; \
1309 } \
1310 } \
1311 dest = tmp >> 16; \
1312 } while(0)
1313 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
1314 NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
1315 #undef NEON_FN
1316 #define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
1317 NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
1318 #undef NEON_FN
1319 #undef NEON_QDMULH16
1320
1321 #define NEON_QDMULH32(dest, src1, src2, round) do { \
1322 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1323 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1324 SET_QC(); \
1325 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1326 } else { \
1327 tmp <<= 1; \
1328 } \
1329 if (round) { \
1330 int64_t old = tmp; \
1331 tmp += (int64_t)1 << 31; \
1332 if ((int64_t)tmp < old) { \
1333 SET_QC(); \
1334 tmp = SIGNBIT64 - 1; \
1335 } \
1336 } \
1337 dest = tmp >> 32; \
1338 } while(0)
1339 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
1340 NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
1341 #undef NEON_FN
1342 #define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
1343 NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
1344 #undef NEON_FN
1345 #undef NEON_QDMULH32
1346
HELPER(neon_narrow_u8)1347 uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1348 {
1349 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1350 | ((x >> 24) & 0xff000000u);
1351 }
1352
HELPER(neon_narrow_u16)1353 uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1354 {
1355 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1356 }
1357
HELPER(neon_narrow_high_u8)1358 uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1359 {
1360 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1361 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1362 }
1363
HELPER(neon_narrow_high_u16)1364 uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1365 {
1366 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1367 }
1368
HELPER(neon_narrow_round_high_u8)1369 uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1370 {
1371 x &= 0xff80ff80ff80ff80ull;
1372 x += 0x0080008000800080ull;
1373 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1374 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1375 }
1376
HELPER(neon_narrow_round_high_u16)1377 uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1378 {
1379 x &= 0xffff8000ffff8000ull;
1380 x += 0x0000800000008000ull;
1381 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1382 }
1383
HELPER(neon_unarrow_sat8)1384 uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
1385 {
1386 uint16_t s;
1387 uint8_t d;
1388 uint32_t res = 0;
1389 #define SAT8(n) \
1390 s = x >> n; \
1391 if (s & 0x8000) { \
1392 SET_QC(); \
1393 } else { \
1394 if (s > 0xff) { \
1395 d = 0xff; \
1396 SET_QC(); \
1397 } else { \
1398 d = s; \
1399 } \
1400 res |= (uint32_t)d << (n / 2); \
1401 }
1402
1403 SAT8(0);
1404 SAT8(16);
1405 SAT8(32);
1406 SAT8(48);
1407 #undef SAT8
1408 return res;
1409 }
1410
HELPER(neon_narrow_sat_u8)1411 uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
1412 {
1413 uint16_t s;
1414 uint8_t d;
1415 uint32_t res = 0;
1416 #define SAT8(n) \
1417 s = x >> n; \
1418 if (s > 0xff) { \
1419 d = 0xff; \
1420 SET_QC(); \
1421 } else { \
1422 d = s; \
1423 } \
1424 res |= (uint32_t)d << (n / 2);
1425
1426 SAT8(0);
1427 SAT8(16);
1428 SAT8(32);
1429 SAT8(48);
1430 #undef SAT8
1431 return res;
1432 }
1433
HELPER(neon_narrow_sat_s8)1434 uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
1435 {
1436 int16_t s;
1437 uint8_t d;
1438 uint32_t res = 0;
1439 #define SAT8(n) \
1440 s = x >> n; \
1441 if (s != (int8_t)s) { \
1442 d = (s >> 15) ^ 0x7f; \
1443 SET_QC(); \
1444 } else { \
1445 d = s; \
1446 } \
1447 res |= (uint32_t)d << (n / 2);
1448
1449 SAT8(0);
1450 SAT8(16);
1451 SAT8(32);
1452 SAT8(48);
1453 #undef SAT8
1454 return res;
1455 }
1456
HELPER(neon_unarrow_sat16)1457 uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
1458 {
1459 uint32_t high;
1460 uint32_t low;
1461 low = x;
1462 if (low & 0x80000000) {
1463 low = 0;
1464 SET_QC();
1465 } else if (low > 0xffff) {
1466 low = 0xffff;
1467 SET_QC();
1468 }
1469 high = x >> 32;
1470 if (high & 0x80000000) {
1471 high = 0;
1472 SET_QC();
1473 } else if (high > 0xffff) {
1474 high = 0xffff;
1475 SET_QC();
1476 }
1477 return low | (high << 16);
1478 }
1479
HELPER(neon_narrow_sat_u16)1480 uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
1481 {
1482 uint32_t high;
1483 uint32_t low;
1484 low = x;
1485 if (low > 0xffff) {
1486 low = 0xffff;
1487 SET_QC();
1488 }
1489 high = x >> 32;
1490 if (high > 0xffff) {
1491 high = 0xffff;
1492 SET_QC();
1493 }
1494 return low | (high << 16);
1495 }
1496
HELPER(neon_narrow_sat_s16)1497 uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
1498 {
1499 int32_t low;
1500 int32_t high;
1501 low = x;
1502 if (low != (int16_t)low) {
1503 low = (low >> 31) ^ 0x7fff;
1504 SET_QC();
1505 }
1506 high = x >> 32;
1507 if (high != (int16_t)high) {
1508 high = (high >> 31) ^ 0x7fff;
1509 SET_QC();
1510 }
1511 return (uint16_t)low | (high << 16);
1512 }
1513
HELPER(neon_unarrow_sat32)1514 uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
1515 {
1516 if (x & 0x8000000000000000ull) {
1517 SET_QC();
1518 return 0;
1519 }
1520 if (x > 0xffffffffu) {
1521 SET_QC();
1522 return 0xffffffffu;
1523 }
1524 return x;
1525 }
1526
HELPER(neon_narrow_sat_u32)1527 uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
1528 {
1529 if (x > 0xffffffffu) {
1530 SET_QC();
1531 return 0xffffffffu;
1532 }
1533 return x;
1534 }
1535
HELPER(neon_narrow_sat_s32)1536 uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
1537 {
1538 if ((int64_t)x != (int32_t)x) {
1539 SET_QC();
1540 return ((int64_t)x >> 63) ^ 0x7fffffff;
1541 }
1542 return x;
1543 }
1544
HELPER(neon_widen_u8)1545 uint64_t HELPER(neon_widen_u8)(uint32_t x)
1546 {
1547 uint64_t tmp;
1548 uint64_t ret;
1549 ret = (uint8_t)x;
1550 tmp = (uint8_t)(x >> 8);
1551 ret |= tmp << 16;
1552 tmp = (uint8_t)(x >> 16);
1553 ret |= tmp << 32;
1554 tmp = (uint8_t)(x >> 24);
1555 ret |= tmp << 48;
1556 return ret;
1557 }
1558
HELPER(neon_widen_s8)1559 uint64_t HELPER(neon_widen_s8)(uint32_t x)
1560 {
1561 uint64_t tmp;
1562 uint64_t ret;
1563 ret = (uint16_t)(int8_t)x;
1564 tmp = (uint16_t)(int8_t)(x >> 8);
1565 ret |= tmp << 16;
1566 tmp = (uint16_t)(int8_t)(x >> 16);
1567 ret |= tmp << 32;
1568 tmp = (uint16_t)(int8_t)(x >> 24);
1569 ret |= tmp << 48;
1570 return ret;
1571 }
1572
HELPER(neon_widen_u16)1573 uint64_t HELPER(neon_widen_u16)(uint32_t x)
1574 {
1575 uint64_t high = (uint16_t)(x >> 16);
1576 return ((uint16_t)x) | (high << 32);
1577 }
1578
HELPER(neon_widen_s16)1579 uint64_t HELPER(neon_widen_s16)(uint32_t x)
1580 {
1581 uint64_t high = (int16_t)(x >> 16);
1582 return ((uint32_t)(int16_t)x) | (high << 32);
1583 }
1584
HELPER(neon_addl_u16)1585 uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1586 {
1587 uint64_t mask;
1588 mask = (a ^ b) & 0x8000800080008000ull;
1589 a &= ~0x8000800080008000ull;
1590 b &= ~0x8000800080008000ull;
1591 return (a + b) ^ mask;
1592 }
1593
HELPER(neon_addl_u32)1594 uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1595 {
1596 uint64_t mask;
1597 mask = (a ^ b) & 0x8000000080000000ull;
1598 a &= ~0x8000000080000000ull;
1599 b &= ~0x8000000080000000ull;
1600 return (a + b) ^ mask;
1601 }
1602
HELPER(neon_paddl_u16)1603 uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1604 {
1605 uint64_t tmp;
1606 uint64_t tmp2;
1607
1608 tmp = a & 0x0000ffff0000ffffull;
1609 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1610 tmp2 = b & 0xffff0000ffff0000ull;
1611 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1612 return ( tmp & 0xffff)
1613 | ((tmp >> 16) & 0xffff0000ull)
1614 | ((tmp2 << 16) & 0xffff00000000ull)
1615 | ( tmp2 & 0xffff000000000000ull);
1616 }
1617
HELPER(neon_paddl_u32)1618 uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1619 {
1620 uint32_t low = a + (a >> 32);
1621 uint32_t high = b + (b >> 32);
1622 return low + ((uint64_t)high << 32);
1623 }
1624
HELPER(neon_subl_u16)1625 uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1626 {
1627 uint64_t mask;
1628 mask = (a ^ ~b) & 0x8000800080008000ull;
1629 a |= 0x8000800080008000ull;
1630 b &= ~0x8000800080008000ull;
1631 return (a - b) ^ mask;
1632 }
1633
HELPER(neon_subl_u32)1634 uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1635 {
1636 uint64_t mask;
1637 mask = (a ^ ~b) & 0x8000000080000000ull;
1638 a |= 0x8000000080000000ull;
1639 b &= ~0x8000000080000000ull;
1640 return (a - b) ^ mask;
1641 }
1642
HELPER(neon_addl_saturate_s32)1643 uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
1644 {
1645 uint32_t x, y;
1646 uint32_t low, high;
1647
1648 x = a;
1649 y = b;
1650 low = x + y;
1651 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1652 SET_QC();
1653 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1654 }
1655 x = a >> 32;
1656 y = b >> 32;
1657 high = x + y;
1658 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1659 SET_QC();
1660 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1661 }
1662 return low | ((uint64_t)high << 32);
1663 }
1664
HELPER(neon_addl_saturate_s64)1665 uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
1666 {
1667 uint64_t result;
1668
1669 result = a + b;
1670 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1671 SET_QC();
1672 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1673 }
1674 return result;
1675 }
1676
1677 /* We have to do the arithmetic in a larger type than
1678 * the input type, because for example with a signed 32 bit
1679 * op the absolute difference can overflow a signed 32 bit value.
1680 */
1681 #define DO_ABD(dest, x, y, intype, arithtype) do { \
1682 arithtype tmp_x = (intype)(x); \
1683 arithtype tmp_y = (intype)(y); \
1684 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1685 } while(0)
1686
HELPER(neon_abdl_u16)1687 uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1688 {
1689 uint64_t tmp;
1690 uint64_t result;
1691 DO_ABD(result, a, b, uint8_t, uint32_t);
1692 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
1693 result |= tmp << 16;
1694 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
1695 result |= tmp << 32;
1696 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
1697 result |= tmp << 48;
1698 return result;
1699 }
1700
HELPER(neon_abdl_s16)1701 uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1702 {
1703 uint64_t tmp;
1704 uint64_t result;
1705 DO_ABD(result, a, b, int8_t, int32_t);
1706 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
1707 result |= tmp << 16;
1708 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
1709 result |= tmp << 32;
1710 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
1711 result |= tmp << 48;
1712 return result;
1713 }
1714
HELPER(neon_abdl_u32)1715 uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1716 {
1717 uint64_t tmp;
1718 uint64_t result;
1719 DO_ABD(result, a, b, uint16_t, uint32_t);
1720 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1721 return result | (tmp << 32);
1722 }
1723
HELPER(neon_abdl_s32)1724 uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1725 {
1726 uint64_t tmp;
1727 uint64_t result;
1728 DO_ABD(result, a, b, int16_t, int32_t);
1729 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
1730 return result | (tmp << 32);
1731 }
1732
HELPER(neon_abdl_u64)1733 uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1734 {
1735 uint64_t result;
1736 DO_ABD(result, a, b, uint32_t, uint64_t);
1737 return result;
1738 }
1739
HELPER(neon_abdl_s64)1740 uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1741 {
1742 uint64_t result;
1743 DO_ABD(result, a, b, int32_t, int64_t);
1744 return result;
1745 }
1746 #undef DO_ABD
1747
1748 /* Widening multiply. Named type is the source type. */
1749 #define DO_MULL(dest, x, y, type1, type2) do { \
1750 type1 tmp_x = x; \
1751 type1 tmp_y = y; \
1752 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1753 } while(0)
1754
HELPER(neon_mull_u8)1755 uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1756 {
1757 uint64_t tmp;
1758 uint64_t result;
1759
1760 DO_MULL(result, a, b, uint8_t, uint16_t);
1761 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1762 result |= tmp << 16;
1763 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1764 result |= tmp << 32;
1765 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1766 result |= tmp << 48;
1767 return result;
1768 }
1769
HELPER(neon_mull_s8)1770 uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1771 {
1772 uint64_t tmp;
1773 uint64_t result;
1774
1775 DO_MULL(result, a, b, int8_t, uint16_t);
1776 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1777 result |= tmp << 16;
1778 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1779 result |= tmp << 32;
1780 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1781 result |= tmp << 48;
1782 return result;
1783 }
1784
HELPER(neon_mull_u16)1785 uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1786 {
1787 uint64_t tmp;
1788 uint64_t result;
1789
1790 DO_MULL(result, a, b, uint16_t, uint32_t);
1791 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1792 return result | (tmp << 32);
1793 }
1794
HELPER(neon_mull_s16)1795 uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1796 {
1797 uint64_t tmp;
1798 uint64_t result;
1799
1800 DO_MULL(result, a, b, int16_t, uint32_t);
1801 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1802 return result | (tmp << 32);
1803 }
1804
HELPER(neon_negl_u16)1805 uint64_t HELPER(neon_negl_u16)(uint64_t x)
1806 {
1807 uint16_t tmp;
1808 uint64_t result;
1809 result = (uint16_t)-x;
1810 tmp = -(x >> 16);
1811 result |= (uint64_t)tmp << 16;
1812 tmp = -(x >> 32);
1813 result |= (uint64_t)tmp << 32;
1814 tmp = -(x >> 48);
1815 result |= (uint64_t)tmp << 48;
1816 return result;
1817 }
1818
HELPER(neon_negl_u32)1819 uint64_t HELPER(neon_negl_u32)(uint64_t x)
1820 {
1821 uint32_t low = -x;
1822 uint32_t high = -(x >> 32);
1823 return low | ((uint64_t)high << 32);
1824 }
1825
1826 /* Saturating sign manipulation. */
1827 /* ??? Make these use NEON_VOP1 */
1828 #define DO_QABS8(x) do { \
1829 if (x == (int8_t)0x80) { \
1830 x = 0x7f; \
1831 SET_QC(); \
1832 } else if (x < 0) { \
1833 x = -x; \
1834 }} while (0)
HELPER(neon_qabs_s8)1835 uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
1836 {
1837 neon_s8 vec;
1838 NEON_UNPACK(neon_s8, vec, x);
1839 DO_QABS8(vec.v1);
1840 DO_QABS8(vec.v2);
1841 DO_QABS8(vec.v3);
1842 DO_QABS8(vec.v4);
1843 NEON_PACK(neon_s8, x, vec);
1844 return x;
1845 }
1846 #undef DO_QABS8
1847
1848 #define DO_QNEG8(x) do { \
1849 if (x == (int8_t)0x80) { \
1850 x = 0x7f; \
1851 SET_QC(); \
1852 } else { \
1853 x = -x; \
1854 }} while (0)
HELPER(neon_qneg_s8)1855 uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
1856 {
1857 neon_s8 vec;
1858 NEON_UNPACK(neon_s8, vec, x);
1859 DO_QNEG8(vec.v1);
1860 DO_QNEG8(vec.v2);
1861 DO_QNEG8(vec.v3);
1862 DO_QNEG8(vec.v4);
1863 NEON_PACK(neon_s8, x, vec);
1864 return x;
1865 }
1866 #undef DO_QNEG8
1867
1868 #define DO_QABS16(x) do { \
1869 if (x == (int16_t)0x8000) { \
1870 x = 0x7fff; \
1871 SET_QC(); \
1872 } else if (x < 0) { \
1873 x = -x; \
1874 }} while (0)
HELPER(neon_qabs_s16)1875 uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
1876 {
1877 neon_s16 vec;
1878 NEON_UNPACK(neon_s16, vec, x);
1879 DO_QABS16(vec.v1);
1880 DO_QABS16(vec.v2);
1881 NEON_PACK(neon_s16, x, vec);
1882 return x;
1883 }
1884 #undef DO_QABS16
1885
1886 #define DO_QNEG16(x) do { \
1887 if (x == (int16_t)0x8000) { \
1888 x = 0x7fff; \
1889 SET_QC(); \
1890 } else { \
1891 x = -x; \
1892 }} while (0)
HELPER(neon_qneg_s16)1893 uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
1894 {
1895 neon_s16 vec;
1896 NEON_UNPACK(neon_s16, vec, x);
1897 DO_QNEG16(vec.v1);
1898 DO_QNEG16(vec.v2);
1899 NEON_PACK(neon_s16, x, vec);
1900 return x;
1901 }
1902 #undef DO_QNEG16
1903
HELPER(neon_qabs_s32)1904 uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
1905 {
1906 if (x == SIGNBIT) {
1907 SET_QC();
1908 x = ~SIGNBIT;
1909 } else if ((int32_t)x < 0) {
1910 x = -x;
1911 }
1912 return x;
1913 }
1914
HELPER(neon_qneg_s32)1915 uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
1916 {
1917 if (x == SIGNBIT) {
1918 SET_QC();
1919 x = ~SIGNBIT;
1920 } else {
1921 x = -x;
1922 }
1923 return x;
1924 }
1925
HELPER(neon_qabs_s64)1926 uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1927 {
1928 if (x == SIGNBIT64) {
1929 SET_QC();
1930 x = ~SIGNBIT64;
1931 } else if ((int64_t)x < 0) {
1932 x = -x;
1933 }
1934 return x;
1935 }
1936
HELPER(neon_qneg_s64)1937 uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1938 {
1939 if (x == SIGNBIT64) {
1940 SET_QC();
1941 x = ~SIGNBIT64;
1942 } else {
1943 x = -x;
1944 }
1945 return x;
1946 }
1947
1948 /* NEON Float helpers. */
HELPER(neon_abd_f32)1949 uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
1950 {
1951 float_status *fpst = fpstp;
1952 float32 f0 = make_float32(a);
1953 float32 f1 = make_float32(b);
1954 return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
1955 }
1956
1957 /* Floating point comparisons produce an integer result.
1958 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1959 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1960 */
HELPER(neon_ceq_f32)1961 uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
1962 {
1963 float_status *fpst = fpstp;
1964 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
1965 }
1966
HELPER(neon_cge_f32)1967 uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
1968 {
1969 float_status *fpst = fpstp;
1970 return -float32_le(make_float32(b), make_float32(a), fpst);
1971 }
1972
HELPER(neon_cgt_f32)1973 uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1974 {
1975 float_status *fpst = fpstp;
1976 return -float32_lt(make_float32(b), make_float32(a), fpst);
1977 }
1978
HELPER(neon_acge_f32)1979 uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
1980 {
1981 float_status *fpst = fpstp;
1982 float32 f0 = float32_abs(make_float32(a));
1983 float32 f1 = float32_abs(make_float32(b));
1984 return -float32_le(f1, f0, fpst);
1985 }
1986
HELPER(neon_acgt_f32)1987 uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
1988 {
1989 float_status *fpst = fpstp;
1990 float32 f0 = float32_abs(make_float32(a));
1991 float32 f1 = float32_abs(make_float32(b));
1992 return -float32_lt(f1, f0, fpst);
1993 }
1994
HELPER(neon_acge_f64)1995 uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1996 {
1997 float_status *fpst = fpstp;
1998 float64 f0 = float64_abs(make_float64(a));
1999 float64 f1 = float64_abs(make_float64(b));
2000 return -float64_le(f1, f0, fpst);
2001 }
2002
HELPER(neon_acgt_f64)2003 uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
2004 {
2005 float_status *fpst = fpstp;
2006 float64 f0 = float64_abs(make_float64(a));
2007 float64 f1 = float64_abs(make_float64(b));
2008 return -float64_lt(f1, f0, fpst);
2009 }
2010
2011 #define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
2012
HELPER(neon_qunzip8)2013 void HELPER(neon_qunzip8)(void *vd, void *vm)
2014 {
2015 uint64_t *rd = vd, *rm = vm;
2016 uint64_t zd0 = rd[0], zd1 = rd[1];
2017 uint64_t zm0 = rm[0], zm1 = rm[1];
2018
2019 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
2020 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
2021 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
2022 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
2023 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
2024 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
2025 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2026 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
2027 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
2028 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
2029 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
2030 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
2031 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
2032 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
2033 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
2034 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2035
2036 rm[0] = m0;
2037 rm[1] = m1;
2038 rd[0] = d0;
2039 rd[1] = d1;
2040 }
2041
HELPER(neon_qunzip16)2042 void HELPER(neon_qunzip16)(void *vd, void *vm)
2043 {
2044 uint64_t *rd = vd, *rm = vm;
2045 uint64_t zd0 = rd[0], zd1 = rd[1];
2046 uint64_t zm0 = rm[0], zm1 = rm[1];
2047
2048 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
2049 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
2050 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
2051 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
2052 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
2053 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
2054 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
2055 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2056
2057 rm[0] = m0;
2058 rm[1] = m1;
2059 rd[0] = d0;
2060 rd[1] = d1;
2061 }
2062
HELPER(neon_qunzip32)2063 void HELPER(neon_qunzip32)(void *vd, void *vm)
2064 {
2065 uint64_t *rd = vd, *rm = vm;
2066 uint64_t zd0 = rd[0], zd1 = rd[1];
2067 uint64_t zm0 = rm[0], zm1 = rm[1];
2068
2069 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
2070 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2071 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
2072 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2073
2074 rm[0] = m0;
2075 rm[1] = m1;
2076 rd[0] = d0;
2077 rd[1] = d1;
2078 }
2079
HELPER(neon_unzip8)2080 void HELPER(neon_unzip8)(void *vd, void *vm)
2081 {
2082 uint64_t *rd = vd, *rm = vm;
2083 uint64_t zd = rd[0], zm = rm[0];
2084
2085 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
2086 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2087 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2088 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2089 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2090 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2091 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2092 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2093
2094 rm[0] = m0;
2095 rd[0] = d0;
2096 }
2097
HELPER(neon_unzip16)2098 void HELPER(neon_unzip16)(void *vd, void *vm)
2099 {
2100 uint64_t *rd = vd, *rm = vm;
2101 uint64_t zd = rd[0], zm = rm[0];
2102
2103 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2104 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2105 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2106 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2107
2108 rm[0] = m0;
2109 rd[0] = d0;
2110 }
2111
HELPER(neon_qzip8)2112 void HELPER(neon_qzip8)(void *vd, void *vm)
2113 {
2114 uint64_t *rd = vd, *rm = vm;
2115 uint64_t zd0 = rd[0], zd1 = rd[1];
2116 uint64_t zm0 = rm[0], zm1 = rm[1];
2117
2118 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2119 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2120 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2121 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2122 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2123 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2124 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2125 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2126 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2127 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2128 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2129 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2130 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2131 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2132 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2133 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
2134
2135 rm[0] = m0;
2136 rm[1] = m1;
2137 rd[0] = d0;
2138 rd[1] = d1;
2139 }
2140
HELPER(neon_qzip16)2141 void HELPER(neon_qzip16)(void *vd, void *vm)
2142 {
2143 uint64_t *rd = vd, *rm = vm;
2144 uint64_t zd0 = rd[0], zd1 = rd[1];
2145 uint64_t zm0 = rm[0], zm1 = rm[1];
2146
2147 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2148 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2149 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2150 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2151 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2152 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2153 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2154 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
2155
2156 rm[0] = m0;
2157 rm[1] = m1;
2158 rd[0] = d0;
2159 rd[1] = d1;
2160 }
2161
HELPER(neon_qzip32)2162 void HELPER(neon_qzip32)(void *vd, void *vm)
2163 {
2164 uint64_t *rd = vd, *rm = vm;
2165 uint64_t zd0 = rd[0], zd1 = rd[1];
2166 uint64_t zm0 = rm[0], zm1 = rm[1];
2167
2168 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2169 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2170 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2171 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
2172
2173 rm[0] = m0;
2174 rm[1] = m1;
2175 rd[0] = d0;
2176 rd[1] = d1;
2177 }
2178
HELPER(neon_zip8)2179 void HELPER(neon_zip8)(void *vd, void *vm)
2180 {
2181 uint64_t *rd = vd, *rm = vm;
2182 uint64_t zd = rd[0], zm = rm[0];
2183
2184 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2185 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2186 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2187 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2188 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2189 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2190 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2191 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
2192
2193 rm[0] = m0;
2194 rd[0] = d0;
2195 }
2196
HELPER(neon_zip16)2197 void HELPER(neon_zip16)(void *vd, void *vm)
2198 {
2199 uint64_t *rd = vd, *rm = vm;
2200 uint64_t zd = rd[0], zm = rm[0];
2201
2202 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2203 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2204 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2205 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
2206
2207 rm[0] = m0;
2208 rd[0] = d0;
2209 }
2210
2211 /* Helper function for 64 bit polynomial multiply case:
2212 * perform PolynomialMult(op1, op2) and return either the top or
2213 * bottom half of the 128 bit result.
2214 */
HELPER(neon_pmull_64_lo)2215 uint64_t HELPER(neon_pmull_64_lo)(uint64_t op1, uint64_t op2)
2216 {
2217 int bitnum;
2218 uint64_t res = 0;
2219
2220 for (bitnum = 0; bitnum < 64; bitnum++) {
2221 if (op1 & (1ULL << bitnum)) {
2222 res ^= op2 << bitnum;
2223 }
2224 }
2225 return res;
2226 }
HELPER(neon_pmull_64_hi)2227 uint64_t HELPER(neon_pmull_64_hi)(uint64_t op1, uint64_t op2)
2228 {
2229 int bitnum;
2230 uint64_t res = 0;
2231
2232 /* bit 0 of op1 can't influence the high 64 bits at all */
2233 for (bitnum = 1; bitnum < 64; bitnum++) {
2234 if (op1 & (1ULL << bitnum)) {
2235 res ^= op2 >> (64 - bitnum);
2236 }
2237 }
2238 return res;
2239 }
2240