1 /*===---- emmintrin.h - Implementation of SSE2 intrinsics on PowerPC -------=== 2 * 3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 * See https://llvm.org/LICENSE.txt for license information. 5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 * 7 *===-----------------------------------------------------------------------=== 8 */ 9 10 /* Implemented from the specification included in the Intel C++ Compiler 11 User Guide and Reference, version 9.0. */ 12 13 #ifndef NO_WARN_X86_INTRINSICS 14 /* This header file is to help porting code using Intel intrinsics 15 explicitly from x86_64 to powerpc64/powerpc64le. 16 17 Since X86 SSE2 intrinsics mainly handles __m128i and __m128d type, 18 PowerPC VMX/VSX ISA is a good match for vector float SIMD operations. 19 However scalar float operations in vector (XMM) registers require 20 the POWER8 VSX ISA (2.07) level. There are differences for data 21 format and placement of float scalars in the vector register, which 22 require extra steps to match SSE2 scalar float semantics on POWER. 23 24 It should be noted that there's much difference between X86_64's 25 MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use 26 portable <fenv.h> instead of access MXSCR directly. 27 28 Most SSE2 scalar float intrinsic operations can be performed more 29 efficiently as C language float scalar operations or optimized to 30 use vector SIMD operations. We recommend this for new applications. 31 */ 32 #error \ 33 "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error." 34 #endif 35 36 #ifndef EMMINTRIN_H_ 37 #define EMMINTRIN_H_ 38 39 #if defined(__powerpc64__) && \ 40 (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) 41 42 #include <altivec.h> 43 44 /* We need definitions from the SSE header files. */ 45 #include <xmmintrin.h> 46 47 /* SSE2 */ 48 typedef __vector double __v2df; 49 typedef __vector float __v4f; 50 typedef __vector long long __v2di; 51 typedef __vector unsigned long long __v2du; 52 typedef __vector int __v4si; 53 typedef __vector unsigned int __v4su; 54 typedef __vector short __v8hi; 55 typedef __vector unsigned short __v8hu; 56 typedef __vector signed char __v16qi; 57 typedef __vector unsigned char __v16qu; 58 59 /* The Intel API is flexible enough that we must allow aliasing with other 60 vector types, and their scalar components. */ 61 typedef long long __m128i __attribute__((__vector_size__(16), __may_alias__)); 62 typedef double __m128d __attribute__((__vector_size__(16), __may_alias__)); 63 64 /* Unaligned version of the same types. */ 65 typedef long long __m128i_u 66 __attribute__((__vector_size__(16), __may_alias__, __aligned__(1))); 67 typedef double __m128d_u 68 __attribute__((__vector_size__(16), __may_alias__, __aligned__(1))); 69 70 /* Define two value permute mask. */ 71 #define _MM_SHUFFLE2(x, y) (((x) << 1) | (y)) 72 73 /* Create a vector with element 0 as F and the rest zero. */ 74 extern __inline __m128d 75 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 76 _mm_set_sd(double __F) { 77 return __extension__(__m128d){__F, 0.0}; 78 } 79 80 /* Create a vector with both elements equal to F. */ 81 extern __inline __m128d 82 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 83 _mm_set1_pd(double __F) { 84 return __extension__(__m128d){__F, __F}; 85 } 86 87 extern __inline __m128d 88 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 89 _mm_set_pd1(double __F) { 90 return _mm_set1_pd(__F); 91 } 92 93 /* Create a vector with the lower value X and upper value W. */ 94 extern __inline __m128d 95 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 96 _mm_set_pd(double __W, double __X) { 97 return __extension__(__m128d){__X, __W}; 98 } 99 100 /* Create a vector with the lower value W and upper value X. */ 101 extern __inline __m128d 102 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 103 _mm_setr_pd(double __W, double __X) { 104 return __extension__(__m128d){__W, __X}; 105 } 106 107 /* Create an undefined vector. */ 108 extern __inline __m128d 109 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 110 _mm_undefined_pd(void) { 111 __m128d __Y = __Y; 112 return __Y; 113 } 114 115 /* Create a vector of zeros. */ 116 extern __inline __m128d 117 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 118 _mm_setzero_pd(void) { 119 return (__m128d)vec_splats(0); 120 } 121 122 /* Sets the low DPFP value of A from the low value of B. */ 123 extern __inline __m128d 124 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 125 _mm_move_sd(__m128d __A, __m128d __B) { 126 __v2df __result = (__v2df)__A; 127 __result[0] = ((__v2df)__B)[0]; 128 return (__m128d)__result; 129 } 130 131 /* Load two DPFP values from P. The address must be 16-byte aligned. */ 132 extern __inline __m128d 133 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 134 _mm_load_pd(double const *__P) { 135 return ((__m128d)vec_ld(0, (__v16qu *)__P)); 136 } 137 138 /* Load two DPFP values from P. The address need not be 16-byte aligned. */ 139 extern __inline __m128d 140 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 141 _mm_loadu_pd(double const *__P) { 142 return (vec_vsx_ld(0, __P)); 143 } 144 145 /* Create a vector with all two elements equal to *P. */ 146 extern __inline __m128d 147 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 148 _mm_load1_pd(double const *__P) { 149 return (vec_splats(*__P)); 150 } 151 152 /* Create a vector with element 0 as *P and the rest zero. */ 153 extern __inline __m128d 154 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 155 _mm_load_sd(double const *__P) { 156 return _mm_set_sd(*__P); 157 } 158 159 extern __inline __m128d 160 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 161 _mm_load_pd1(double const *__P) { 162 return _mm_load1_pd(__P); 163 } 164 165 /* Load two DPFP values in reverse order. The address must be aligned. */ 166 extern __inline __m128d 167 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 168 _mm_loadr_pd(double const *__P) { 169 __v2df __tmp = _mm_load_pd(__P); 170 return (__m128d)vec_xxpermdi(__tmp, __tmp, 2); 171 } 172 173 /* Store two DPFP values. The address must be 16-byte aligned. */ 174 extern __inline void 175 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 176 _mm_store_pd(double *__P, __m128d __A) { 177 vec_st((__v16qu)__A, 0, (__v16qu *)__P); 178 } 179 180 /* Store two DPFP values. The address need not be 16-byte aligned. */ 181 extern __inline void 182 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 183 _mm_storeu_pd(double *__P, __m128d __A) { 184 *(__m128d_u *)__P = __A; 185 } 186 187 /* Stores the lower DPFP value. */ 188 extern __inline void 189 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 190 _mm_store_sd(double *__P, __m128d __A) { 191 *__P = ((__v2df)__A)[0]; 192 } 193 194 extern __inline double 195 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 196 _mm_cvtsd_f64(__m128d __A) { 197 return ((__v2df)__A)[0]; 198 } 199 200 extern __inline void 201 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 202 _mm_storel_pd(double *__P, __m128d __A) { 203 _mm_store_sd(__P, __A); 204 } 205 206 /* Stores the upper DPFP value. */ 207 extern __inline void 208 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 209 _mm_storeh_pd(double *__P, __m128d __A) { 210 *__P = ((__v2df)__A)[1]; 211 } 212 /* Store the lower DPFP value across two words. 213 The address must be 16-byte aligned. */ 214 extern __inline void 215 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 216 _mm_store1_pd(double *__P, __m128d __A) { 217 _mm_store_pd(__P, vec_splat(__A, 0)); 218 } 219 220 extern __inline void 221 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 222 _mm_store_pd1(double *__P, __m128d __A) { 223 _mm_store1_pd(__P, __A); 224 } 225 226 /* Store two DPFP values in reverse order. The address must be aligned. */ 227 extern __inline void 228 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 229 _mm_storer_pd(double *__P, __m128d __A) { 230 _mm_store_pd(__P, vec_xxpermdi(__A, __A, 2)); 231 } 232 233 /* Intel intrinsic. */ 234 extern __inline long long 235 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 236 _mm_cvtsi128_si64(__m128i __A) { 237 return ((__v2di)__A)[0]; 238 } 239 240 /* Microsoft intrinsic. */ 241 extern __inline long long 242 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 243 _mm_cvtsi128_si64x(__m128i __A) { 244 return ((__v2di)__A)[0]; 245 } 246 247 extern __inline __m128d 248 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 249 _mm_add_pd(__m128d __A, __m128d __B) { 250 return (__m128d)((__v2df)__A + (__v2df)__B); 251 } 252 253 /* Add the lower double-precision (64-bit) floating-point element in 254 a and b, store the result in the lower element of dst, and copy 255 the upper element from a to the upper element of dst. */ 256 extern __inline __m128d 257 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 258 _mm_add_sd(__m128d __A, __m128d __B) { 259 __A[0] = __A[0] + __B[0]; 260 return (__A); 261 } 262 263 extern __inline __m128d 264 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 265 _mm_sub_pd(__m128d __A, __m128d __B) { 266 return (__m128d)((__v2df)__A - (__v2df)__B); 267 } 268 269 extern __inline __m128d 270 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 271 _mm_sub_sd(__m128d __A, __m128d __B) { 272 __A[0] = __A[0] - __B[0]; 273 return (__A); 274 } 275 276 extern __inline __m128d 277 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 278 _mm_mul_pd(__m128d __A, __m128d __B) { 279 return (__m128d)((__v2df)__A * (__v2df)__B); 280 } 281 282 extern __inline __m128d 283 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 284 _mm_mul_sd(__m128d __A, __m128d __B) { 285 __A[0] = __A[0] * __B[0]; 286 return (__A); 287 } 288 289 extern __inline __m128d 290 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 291 _mm_div_pd(__m128d __A, __m128d __B) { 292 return (__m128d)((__v2df)__A / (__v2df)__B); 293 } 294 295 extern __inline __m128d 296 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 297 _mm_div_sd(__m128d __A, __m128d __B) { 298 __A[0] = __A[0] / __B[0]; 299 return (__A); 300 } 301 302 extern __inline __m128d 303 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 304 _mm_sqrt_pd(__m128d __A) { 305 return (vec_sqrt(__A)); 306 } 307 308 /* Return pair {sqrt (B[0]), A[1]}. */ 309 extern __inline __m128d 310 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 311 _mm_sqrt_sd(__m128d __A, __m128d __B) { 312 __v2df __c; 313 __c = vec_sqrt((__v2df)_mm_set1_pd(__B[0])); 314 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 315 } 316 317 extern __inline __m128d 318 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 319 _mm_min_pd(__m128d __A, __m128d __B) { 320 return (vec_min(__A, __B)); 321 } 322 323 extern __inline __m128d 324 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 325 _mm_min_sd(__m128d __A, __m128d __B) { 326 __v2df __a, __b, __c; 327 __a = vec_splats(__A[0]); 328 __b = vec_splats(__B[0]); 329 __c = vec_min(__a, __b); 330 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 331 } 332 333 extern __inline __m128d 334 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 335 _mm_max_pd(__m128d __A, __m128d __B) { 336 return (vec_max(__A, __B)); 337 } 338 339 extern __inline __m128d 340 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 341 _mm_max_sd(__m128d __A, __m128d __B) { 342 __v2df __a, __b, __c; 343 __a = vec_splats(__A[0]); 344 __b = vec_splats(__B[0]); 345 __c = vec_max(__a, __b); 346 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 347 } 348 349 extern __inline __m128d 350 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 351 _mm_cmpeq_pd(__m128d __A, __m128d __B) { 352 return ((__m128d)vec_cmpeq((__v2df)__A, (__v2df)__B)); 353 } 354 355 extern __inline __m128d 356 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 357 _mm_cmplt_pd(__m128d __A, __m128d __B) { 358 return ((__m128d)vec_cmplt((__v2df)__A, (__v2df)__B)); 359 } 360 361 extern __inline __m128d 362 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 363 _mm_cmple_pd(__m128d __A, __m128d __B) { 364 return ((__m128d)vec_cmple((__v2df)__A, (__v2df)__B)); 365 } 366 367 extern __inline __m128d 368 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 369 _mm_cmpgt_pd(__m128d __A, __m128d __B) { 370 return ((__m128d)vec_cmpgt((__v2df)__A, (__v2df)__B)); 371 } 372 373 extern __inline __m128d 374 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 375 _mm_cmpge_pd(__m128d __A, __m128d __B) { 376 return ((__m128d)vec_cmpge((__v2df)__A, (__v2df)__B)); 377 } 378 379 extern __inline __m128d 380 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 381 _mm_cmpneq_pd(__m128d __A, __m128d __B) { 382 __v2df __temp = (__v2df)vec_cmpeq((__v2df)__A, (__v2df)__B); 383 return ((__m128d)vec_nor(__temp, __temp)); 384 } 385 386 extern __inline __m128d 387 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 388 _mm_cmpnlt_pd(__m128d __A, __m128d __B) { 389 return ((__m128d)vec_cmpge((__v2df)__A, (__v2df)__B)); 390 } 391 392 extern __inline __m128d 393 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 394 _mm_cmpnle_pd(__m128d __A, __m128d __B) { 395 return ((__m128d)vec_cmpgt((__v2df)__A, (__v2df)__B)); 396 } 397 398 extern __inline __m128d 399 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 400 _mm_cmpngt_pd(__m128d __A, __m128d __B) { 401 return ((__m128d)vec_cmple((__v2df)__A, (__v2df)__B)); 402 } 403 404 extern __inline __m128d 405 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 406 _mm_cmpnge_pd(__m128d __A, __m128d __B) { 407 return ((__m128d)vec_cmplt((__v2df)__A, (__v2df)__B)); 408 } 409 410 extern __inline __m128d 411 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 412 _mm_cmpord_pd(__m128d __A, __m128d __B) { 413 __v2du __c, __d; 414 /* Compare against self will return false (0's) if NAN. */ 415 __c = (__v2du)vec_cmpeq(__A, __A); 416 __d = (__v2du)vec_cmpeq(__B, __B); 417 /* A != NAN and B != NAN. */ 418 return ((__m128d)vec_and(__c, __d)); 419 } 420 421 extern __inline __m128d 422 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 423 _mm_cmpunord_pd(__m128d __A, __m128d __B) { 424 #if _ARCH_PWR8 425 __v2du __c, __d; 426 /* Compare against self will return false (0's) if NAN. */ 427 __c = (__v2du)vec_cmpeq((__v2df)__A, (__v2df)__A); 428 __d = (__v2du)vec_cmpeq((__v2df)__B, (__v2df)__B); 429 /* A == NAN OR B == NAN converts too: 430 NOT(A != NAN) OR NOT(B != NAN). */ 431 __c = vec_nor(__c, __c); 432 return ((__m128d)vec_orc(__c, __d)); 433 #else 434 __v2du __c, __d; 435 /* Compare against self will return false (0's) if NAN. */ 436 __c = (__v2du)vec_cmpeq((__v2df)__A, (__v2df)__A); 437 __d = (__v2du)vec_cmpeq((__v2df)__B, (__v2df)__B); 438 /* Convert the true ('1's) is NAN. */ 439 __c = vec_nor(__c, __c); 440 __d = vec_nor(__d, __d); 441 return ((__m128d)vec_or(__c, __d)); 442 #endif 443 } 444 445 extern __inline __m128d 446 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 447 _mm_cmpeq_sd(__m128d __A, __m128d __B) { 448 __v2df __a, __b, __c; 449 /* PowerISA VSX does not allow partial (for just lower double) 450 results. So to insure we don't generate spurious exceptions 451 (from the upper double values) we splat the lower double 452 before we do the operation. */ 453 __a = vec_splats(__A[0]); 454 __b = vec_splats(__B[0]); 455 __c = (__v2df)vec_cmpeq(__a, __b); 456 /* Then we merge the lower double result with the original upper 457 double from __A. */ 458 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 459 } 460 461 extern __inline __m128d 462 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 463 _mm_cmplt_sd(__m128d __A, __m128d __B) { 464 __v2df __a, __b, __c; 465 __a = vec_splats(__A[0]); 466 __b = vec_splats(__B[0]); 467 __c = (__v2df)vec_cmplt(__a, __b); 468 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 469 } 470 471 extern __inline __m128d 472 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 473 _mm_cmple_sd(__m128d __A, __m128d __B) { 474 __v2df __a, __b, __c; 475 __a = vec_splats(__A[0]); 476 __b = vec_splats(__B[0]); 477 __c = (__v2df)vec_cmple(__a, __b); 478 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 479 } 480 481 extern __inline __m128d 482 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 483 _mm_cmpgt_sd(__m128d __A, __m128d __B) { 484 __v2df __a, __b, __c; 485 __a = vec_splats(__A[0]); 486 __b = vec_splats(__B[0]); 487 __c = (__v2df)vec_cmpgt(__a, __b); 488 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 489 } 490 491 extern __inline __m128d 492 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 493 _mm_cmpge_sd(__m128d __A, __m128d __B) { 494 __v2df __a, __b, __c; 495 __a = vec_splats(__A[0]); 496 __b = vec_splats(__B[0]); 497 __c = (__v2df)vec_cmpge(__a, __b); 498 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 499 } 500 501 extern __inline __m128d 502 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 503 _mm_cmpneq_sd(__m128d __A, __m128d __B) { 504 __v2df __a, __b, __c; 505 __a = vec_splats(__A[0]); 506 __b = vec_splats(__B[0]); 507 __c = (__v2df)vec_cmpeq(__a, __b); 508 __c = vec_nor(__c, __c); 509 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 510 } 511 512 extern __inline __m128d 513 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 514 _mm_cmpnlt_sd(__m128d __A, __m128d __B) { 515 __v2df __a, __b, __c; 516 __a = vec_splats(__A[0]); 517 __b = vec_splats(__B[0]); 518 /* Not less than is just greater than or equal. */ 519 __c = (__v2df)vec_cmpge(__a, __b); 520 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 521 } 522 523 extern __inline __m128d 524 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 525 _mm_cmpnle_sd(__m128d __A, __m128d __B) { 526 __v2df __a, __b, __c; 527 __a = vec_splats(__A[0]); 528 __b = vec_splats(__B[0]); 529 /* Not less than or equal is just greater than. */ 530 __c = (__v2df)vec_cmpge(__a, __b); 531 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 532 } 533 534 extern __inline __m128d 535 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 536 _mm_cmpngt_sd(__m128d __A, __m128d __B) { 537 __v2df __a, __b, __c; 538 __a = vec_splats(__A[0]); 539 __b = vec_splats(__B[0]); 540 /* Not greater than is just less than or equal. */ 541 __c = (__v2df)vec_cmple(__a, __b); 542 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 543 } 544 545 extern __inline __m128d 546 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 547 _mm_cmpnge_sd(__m128d __A, __m128d __B) { 548 __v2df __a, __b, __c; 549 __a = vec_splats(__A[0]); 550 __b = vec_splats(__B[0]); 551 /* Not greater than or equal is just less than. */ 552 __c = (__v2df)vec_cmplt(__a, __b); 553 return (__m128d)_mm_setr_pd(__c[0], __A[1]); 554 } 555 556 extern __inline __m128d 557 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 558 _mm_cmpord_sd(__m128d __A, __m128d __B) { 559 __v2df __r; 560 __r = (__v2df)_mm_cmpord_pd(vec_splats(__A[0]), vec_splats(__B[0])); 561 return (__m128d)_mm_setr_pd(__r[0], ((__v2df)__A)[1]); 562 } 563 564 extern __inline __m128d 565 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 566 _mm_cmpunord_sd(__m128d __A, __m128d __B) { 567 __v2df __r; 568 __r = _mm_cmpunord_pd(vec_splats(__A[0]), vec_splats(__B[0])); 569 return (__m128d)_mm_setr_pd(__r[0], __A[1]); 570 } 571 572 /* FIXME 573 The __mm_comi??_sd and __mm_ucomi??_sd implementations below are 574 exactly the same because GCC for PowerPC only generates unordered 575 compares (scalar and vector). 576 Technically __mm_comieq_sp et all should be using the ordered 577 compare and signal for QNaNs. The __mm_ucomieq_sd et all should 578 be OK. */ 579 extern __inline int 580 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 581 _mm_comieq_sd(__m128d __A, __m128d __B) { 582 return (__A[0] == __B[0]); 583 } 584 585 extern __inline int 586 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 587 _mm_comilt_sd(__m128d __A, __m128d __B) { 588 return (__A[0] < __B[0]); 589 } 590 591 extern __inline int 592 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 593 _mm_comile_sd(__m128d __A, __m128d __B) { 594 return (__A[0] <= __B[0]); 595 } 596 597 extern __inline int 598 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 599 _mm_comigt_sd(__m128d __A, __m128d __B) { 600 return (__A[0] > __B[0]); 601 } 602 603 extern __inline int 604 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 605 _mm_comige_sd(__m128d __A, __m128d __B) { 606 return (__A[0] >= __B[0]); 607 } 608 609 extern __inline int 610 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 611 _mm_comineq_sd(__m128d __A, __m128d __B) { 612 return (__A[0] != __B[0]); 613 } 614 615 extern __inline int 616 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 617 _mm_ucomieq_sd(__m128d __A, __m128d __B) { 618 return (__A[0] == __B[0]); 619 } 620 621 extern __inline int 622 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 623 _mm_ucomilt_sd(__m128d __A, __m128d __B) { 624 return (__A[0] < __B[0]); 625 } 626 627 extern __inline int 628 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 629 _mm_ucomile_sd(__m128d __A, __m128d __B) { 630 return (__A[0] <= __B[0]); 631 } 632 633 extern __inline int 634 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 635 _mm_ucomigt_sd(__m128d __A, __m128d __B) { 636 return (__A[0] > __B[0]); 637 } 638 639 extern __inline int 640 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 641 _mm_ucomige_sd(__m128d __A, __m128d __B) { 642 return (__A[0] >= __B[0]); 643 } 644 645 extern __inline int 646 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 647 _mm_ucomineq_sd(__m128d __A, __m128d __B) { 648 return (__A[0] != __B[0]); 649 } 650 651 /* Create a vector of Qi, where i is the element number. */ 652 extern __inline __m128i 653 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 654 _mm_set_epi64x(long long __q1, long long __q0) { 655 return __extension__(__m128i)(__v2di){__q0, __q1}; 656 } 657 658 extern __inline __m128i 659 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 660 _mm_set_epi64(__m64 __q1, __m64 __q0) { 661 return _mm_set_epi64x((long long)__q1, (long long)__q0); 662 } 663 664 extern __inline __m128i 665 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 666 _mm_set_epi32(int __q3, int __q2, int __q1, int __q0) { 667 return __extension__(__m128i)(__v4si){__q0, __q1, __q2, __q3}; 668 } 669 670 extern __inline __m128i 671 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 672 _mm_set_epi16(short __q7, short __q6, short __q5, short __q4, short __q3, 673 short __q2, short __q1, short __q0) { 674 return __extension__(__m128i)(__v8hi){__q0, __q1, __q2, __q3, 675 __q4, __q5, __q6, __q7}; 676 } 677 678 extern __inline __m128i 679 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 680 _mm_set_epi8(char __q15, char __q14, char __q13, char __q12, char __q11, 681 char __q10, char __q09, char __q08, char __q07, char __q06, 682 char __q05, char __q04, char __q03, char __q02, char __q01, 683 char __q00) { 684 return __extension__(__m128i)(__v16qi){ 685 __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, 686 __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15}; 687 } 688 689 /* Set all of the elements of the vector to A. */ 690 extern __inline __m128i 691 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 692 _mm_set1_epi64x(long long __A) { 693 return _mm_set_epi64x(__A, __A); 694 } 695 696 extern __inline __m128i 697 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 698 _mm_set1_epi64(__m64 __A) { 699 return _mm_set_epi64(__A, __A); 700 } 701 702 extern __inline __m128i 703 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 704 _mm_set1_epi32(int __A) { 705 return _mm_set_epi32(__A, __A, __A, __A); 706 } 707 708 extern __inline __m128i 709 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 710 _mm_set1_epi16(short __A) { 711 return _mm_set_epi16(__A, __A, __A, __A, __A, __A, __A, __A); 712 } 713 714 extern __inline __m128i 715 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 716 _mm_set1_epi8(char __A) { 717 return _mm_set_epi8(__A, __A, __A, __A, __A, __A, __A, __A, __A, __A, __A, 718 __A, __A, __A, __A, __A); 719 } 720 721 /* Create a vector of Qi, where i is the element number. 722 The parameter order is reversed from the _mm_set_epi* functions. */ 723 extern __inline __m128i 724 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 725 _mm_setr_epi64(__m64 __q0, __m64 __q1) { 726 return _mm_set_epi64(__q1, __q0); 727 } 728 729 extern __inline __m128i 730 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 731 _mm_setr_epi32(int __q0, int __q1, int __q2, int __q3) { 732 return _mm_set_epi32(__q3, __q2, __q1, __q0); 733 } 734 735 extern __inline __m128i 736 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 737 _mm_setr_epi16(short __q0, short __q1, short __q2, short __q3, short __q4, 738 short __q5, short __q6, short __q7) { 739 return _mm_set_epi16(__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0); 740 } 741 742 extern __inline __m128i 743 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 744 _mm_setr_epi8(char __q00, char __q01, char __q02, char __q03, char __q04, 745 char __q05, char __q06, char __q07, char __q08, char __q09, 746 char __q10, char __q11, char __q12, char __q13, char __q14, 747 char __q15) { 748 return _mm_set_epi8(__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08, 749 __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00); 750 } 751 752 /* Create a vector with element 0 as *P and the rest zero. */ 753 extern __inline __m128i 754 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 755 _mm_load_si128(__m128i const *__P) { 756 return *__P; 757 } 758 759 extern __inline __m128i 760 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 761 _mm_loadu_si128(__m128i_u const *__P) { 762 return (__m128i)(vec_vsx_ld(0, (signed int const *)__P)); 763 } 764 765 extern __inline __m128i 766 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 767 _mm_loadl_epi64(__m128i_u const *__P) { 768 return _mm_set_epi64((__m64)0LL, *(__m64 *)__P); 769 } 770 771 extern __inline void 772 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 773 _mm_store_si128(__m128i *__P, __m128i __B) { 774 vec_st((__v16qu)__B, 0, (__v16qu *)__P); 775 } 776 777 extern __inline void 778 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 779 _mm_storeu_si128(__m128i_u *__P, __m128i __B) { 780 *__P = __B; 781 } 782 783 extern __inline void 784 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 785 _mm_storel_epi64(__m128i_u *__P, __m128i __B) { 786 *(long long *)__P = ((__v2di)__B)[0]; 787 } 788 789 extern __inline __m64 790 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 791 _mm_movepi64_pi64(__m128i_u __B) { 792 return (__m64)((__v2di)__B)[0]; 793 } 794 795 extern __inline __m128i 796 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 797 _mm_movpi64_epi64(__m64 __A) { 798 return _mm_set_epi64((__m64)0LL, __A); 799 } 800 801 extern __inline __m128i 802 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 803 _mm_move_epi64(__m128i __A) { 804 return _mm_set_epi64((__m64)0LL, (__m64)__A[0]); 805 } 806 807 /* Create an undefined vector. */ 808 extern __inline __m128i 809 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 810 _mm_undefined_si128(void) { 811 __m128i __Y = __Y; 812 return __Y; 813 } 814 815 /* Create a vector of zeros. */ 816 extern __inline __m128i 817 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 818 _mm_setzero_si128(void) { 819 return __extension__(__m128i)(__v4si){0, 0, 0, 0}; 820 } 821 822 #ifdef _ARCH_PWR8 823 extern __inline __m128d 824 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 825 _mm_cvtepi32_pd(__m128i __A) { 826 __v2di __val; 827 /* For LE need to generate Vector Unpack Low Signed Word. 828 Which is generated from unpackh. */ 829 __val = (__v2di)vec_unpackh((__v4si)__A); 830 831 return (__m128d)vec_ctf(__val, 0); 832 } 833 #endif 834 835 extern __inline __m128 836 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 837 _mm_cvtepi32_ps(__m128i __A) { 838 return ((__m128)vec_ctf((__v4si)__A, 0)); 839 } 840 841 extern __inline __m128i 842 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 843 _mm_cvtpd_epi32(__m128d __A) { 844 __v2df __rounded = vec_rint(__A); 845 __v4si __result, __temp; 846 const __v4si __vzero = {0, 0, 0, 0}; 847 848 /* VSX Vector truncate Double-Precision to integer and Convert to 849 Signed Integer Word format with Saturate. */ 850 __asm__("xvcvdpsxws %x0,%x1" : "=wa"(__temp) : "wa"(__rounded) :); 851 852 #ifdef _ARCH_PWR8 853 #ifdef __LITTLE_ENDIAN__ 854 __temp = vec_mergeo(__temp, __temp); 855 #else 856 __temp = vec_mergee(__temp, __temp); 857 #endif 858 __result = (__v4si)vec_vpkudum((__vector long long)__temp, 859 (__vector long long)__vzero); 860 #else 861 { 862 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, 863 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f}; 864 __result = (__v4si)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm); 865 } 866 #endif 867 return (__m128i)__result; 868 } 869 870 extern __inline __m64 871 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 872 _mm_cvtpd_pi32(__m128d __A) { 873 __m128i __result = _mm_cvtpd_epi32(__A); 874 875 return (__m64)__result[0]; 876 } 877 878 extern __inline __m128 879 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 880 _mm_cvtpd_ps(__m128d __A) { 881 __v4sf __result; 882 __v4si __temp; 883 const __v4si __vzero = {0, 0, 0, 0}; 884 885 __asm__("xvcvdpsp %x0,%x1" : "=wa"(__temp) : "wa"(__A) :); 886 887 #ifdef _ARCH_PWR8 888 #ifdef __LITTLE_ENDIAN__ 889 __temp = vec_mergeo(__temp, __temp); 890 #else 891 __temp = vec_mergee(__temp, __temp); 892 #endif 893 __result = (__v4sf)vec_vpkudum((__vector long long)__temp, 894 (__vector long long)__vzero); 895 #else 896 { 897 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, 898 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f}; 899 __result = (__v4sf)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm); 900 } 901 #endif 902 return ((__m128)__result); 903 } 904 905 extern __inline __m128i 906 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 907 _mm_cvttpd_epi32(__m128d __A) { 908 __v4si __result; 909 __v4si __temp; 910 const __v4si __vzero = {0, 0, 0, 0}; 911 912 /* VSX Vector truncate Double-Precision to integer and Convert to 913 Signed Integer Word format with Saturate. */ 914 __asm__("xvcvdpsxws %x0,%x1" : "=wa"(__temp) : "wa"(__A) :); 915 916 #ifdef _ARCH_PWR8 917 #ifdef __LITTLE_ENDIAN__ 918 __temp = vec_mergeo(__temp, __temp); 919 #else 920 __temp = vec_mergee(__temp, __temp); 921 #endif 922 __result = (__v4si)vec_vpkudum((__vector long long)__temp, 923 (__vector long long)__vzero); 924 #else 925 { 926 const __v16qu __pkperm = {0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, 927 0x14, 0x15, 0x16, 0x17, 0x1c, 0x1d, 0x1e, 0x1f}; 928 __result = (__v4si)vec_perm((__v16qu)__temp, (__v16qu)__vzero, __pkperm); 929 } 930 #endif 931 932 return ((__m128i)__result); 933 } 934 935 extern __inline __m64 936 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 937 _mm_cvttpd_pi32(__m128d __A) { 938 __m128i __result = _mm_cvttpd_epi32(__A); 939 940 return (__m64)__result[0]; 941 } 942 943 extern __inline int 944 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 945 _mm_cvtsi128_si32(__m128i __A) { 946 return ((__v4si)__A)[0]; 947 } 948 949 #ifdef _ARCH_PWR8 950 extern __inline __m128d 951 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 952 _mm_cvtpi32_pd(__m64 __A) { 953 __v4si __temp; 954 __v2di __tmp2; 955 __v4f __result; 956 957 __temp = (__v4si)vec_splats(__A); 958 __tmp2 = (__v2di)vec_unpackl(__temp); 959 __result = vec_ctf((__vector signed long long)__tmp2, 0); 960 return (__m128d)__result; 961 } 962 #endif 963 964 extern __inline __m128i 965 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 966 _mm_cvtps_epi32(__m128 __A) { 967 __v4sf __rounded; 968 __v4si __result; 969 970 __rounded = vec_rint((__v4sf)__A); 971 __result = vec_cts(__rounded, 0); 972 return (__m128i)__result; 973 } 974 975 extern __inline __m128i 976 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 977 _mm_cvttps_epi32(__m128 __A) { 978 __v4si __result; 979 980 __result = vec_cts((__v4sf)__A, 0); 981 return (__m128i)__result; 982 } 983 984 extern __inline __m128d 985 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 986 _mm_cvtps_pd(__m128 __A) { 987 /* Check if vec_doubleh is defined by <altivec.h>. If so use that. */ 988 #ifdef vec_doubleh 989 return (__m128d)vec_doubleh((__v4sf)__A); 990 #else 991 /* Otherwise the compiler is not current and so need to generate the 992 equivalent code. */ 993 __v4sf __a = (__v4sf)__A; 994 __v4sf __temp; 995 __v2df __result; 996 #ifdef __LITTLE_ENDIAN__ 997 /* The input float values are in elements {[0], [1]} but the convert 998 instruction needs them in elements {[1], [3]}, So we use two 999 shift left double vector word immediates to get the elements 1000 lined up. */ 1001 __temp = __builtin_vsx_xxsldwi(__a, __a, 3); 1002 __temp = __builtin_vsx_xxsldwi(__a, __temp, 2); 1003 #else 1004 /* The input float values are in elements {[0], [1]} but the convert 1005 instruction needs them in elements {[0], [2]}, So we use two 1006 shift left double vector word immediates to get the elements 1007 lined up. */ 1008 __temp = vec_vmrghw(__a, __a); 1009 #endif 1010 __asm__(" xvcvspdp %x0,%x1" : "=wa"(__result) : "wa"(__temp) :); 1011 return (__m128d)__result; 1012 #endif 1013 } 1014 1015 extern __inline int 1016 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1017 _mm_cvtsd_si32(__m128d __A) { 1018 __v2df __rounded = vec_rint((__v2df)__A); 1019 int __result = ((__v2df)__rounded)[0]; 1020 1021 return __result; 1022 } 1023 /* Intel intrinsic. */ 1024 extern __inline long long 1025 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1026 _mm_cvtsd_si64(__m128d __A) { 1027 __v2df __rounded = vec_rint((__v2df)__A); 1028 long long __result = ((__v2df)__rounded)[0]; 1029 1030 return __result; 1031 } 1032 1033 /* Microsoft intrinsic. */ 1034 extern __inline long long 1035 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1036 _mm_cvtsd_si64x(__m128d __A) { 1037 return _mm_cvtsd_si64((__v2df)__A); 1038 } 1039 1040 extern __inline int 1041 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1042 _mm_cvttsd_si32(__m128d __A) { 1043 int __result = ((__v2df)__A)[0]; 1044 1045 return __result; 1046 } 1047 1048 /* Intel intrinsic. */ 1049 extern __inline long long 1050 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1051 _mm_cvttsd_si64(__m128d __A) { 1052 long long __result = ((__v2df)__A)[0]; 1053 1054 return __result; 1055 } 1056 1057 /* Microsoft intrinsic. */ 1058 extern __inline long long 1059 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1060 _mm_cvttsd_si64x(__m128d __A) { 1061 return _mm_cvttsd_si64(__A); 1062 } 1063 1064 extern __inline __m128 1065 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1066 _mm_cvtsd_ss(__m128 __A, __m128d __B) { 1067 __v4sf __result = (__v4sf)__A; 1068 1069 #ifdef __LITTLE_ENDIAN__ 1070 __v4sf __temp_s; 1071 /* Copy double element[0] to element [1] for conversion. */ 1072 __v2df __temp_b = vec_splat((__v2df)__B, 0); 1073 1074 /* Pre-rotate __A left 3 (logically right 1) elements. */ 1075 __result = __builtin_vsx_xxsldwi(__result, __result, 3); 1076 /* Convert double to single float scalar in a vector. */ 1077 __asm__("xscvdpsp %x0,%x1" : "=wa"(__temp_s) : "wa"(__temp_b) :); 1078 /* Shift the resulting scalar into vector element [0]. */ 1079 __result = __builtin_vsx_xxsldwi(__result, __temp_s, 1); 1080 #else 1081 __result[0] = ((__v2df)__B)[0]; 1082 #endif 1083 return (__m128)__result; 1084 } 1085 1086 extern __inline __m128d 1087 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1088 _mm_cvtsi32_sd(__m128d __A, int __B) { 1089 __v2df __result = (__v2df)__A; 1090 double __db = __B; 1091 __result[0] = __db; 1092 return (__m128d)__result; 1093 } 1094 1095 /* Intel intrinsic. */ 1096 extern __inline __m128d 1097 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1098 _mm_cvtsi64_sd(__m128d __A, long long __B) { 1099 __v2df __result = (__v2df)__A; 1100 double __db = __B; 1101 __result[0] = __db; 1102 return (__m128d)__result; 1103 } 1104 1105 /* Microsoft intrinsic. */ 1106 extern __inline __m128d 1107 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1108 _mm_cvtsi64x_sd(__m128d __A, long long __B) { 1109 return _mm_cvtsi64_sd(__A, __B); 1110 } 1111 1112 extern __inline __m128d 1113 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1114 _mm_cvtss_sd(__m128d __A, __m128 __B) { 1115 #ifdef __LITTLE_ENDIAN__ 1116 /* Use splat to move element [0] into position for the convert. */ 1117 __v4sf __temp = vec_splat((__v4sf)__B, 0); 1118 __v2df __res; 1119 /* Convert single float scalar to double in a vector. */ 1120 __asm__("xscvspdp %x0,%x1" : "=wa"(__res) : "wa"(__temp) :); 1121 return (__m128d)vec_mergel(__res, (__v2df)__A); 1122 #else 1123 __v2df __res = (__v2df)__A; 1124 __res[0] = ((__v4sf)__B)[0]; 1125 return (__m128d)__res; 1126 #endif 1127 } 1128 1129 extern __inline __m128d 1130 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1131 _mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask) { 1132 __vector double __result; 1133 const int __litmsk = __mask & 0x3; 1134 1135 if (__litmsk == 0) 1136 __result = vec_mergeh(__A, __B); 1137 #if __GNUC__ < 6 1138 else if (__litmsk == 1) 1139 __result = vec_xxpermdi(__B, __A, 2); 1140 else if (__litmsk == 2) 1141 __result = vec_xxpermdi(__B, __A, 1); 1142 #else 1143 else if (__litmsk == 1) 1144 __result = vec_xxpermdi(__A, __B, 2); 1145 else if (__litmsk == 2) 1146 __result = vec_xxpermdi(__A, __B, 1); 1147 #endif 1148 else 1149 __result = vec_mergel(__A, __B); 1150 1151 return __result; 1152 } 1153 1154 extern __inline __m128d 1155 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1156 _mm_unpackhi_pd(__m128d __A, __m128d __B) { 1157 return (__m128d)vec_mergel((__v2df)__A, (__v2df)__B); 1158 } 1159 1160 extern __inline __m128d 1161 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1162 _mm_unpacklo_pd(__m128d __A, __m128d __B) { 1163 return (__m128d)vec_mergeh((__v2df)__A, (__v2df)__B); 1164 } 1165 1166 extern __inline __m128d 1167 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1168 _mm_loadh_pd(__m128d __A, double const *__B) { 1169 __v2df __result = (__v2df)__A; 1170 __result[1] = *__B; 1171 return (__m128d)__result; 1172 } 1173 1174 extern __inline __m128d 1175 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1176 _mm_loadl_pd(__m128d __A, double const *__B) { 1177 __v2df __result = (__v2df)__A; 1178 __result[0] = *__B; 1179 return (__m128d)__result; 1180 } 1181 1182 #ifdef _ARCH_PWR8 1183 /* Intrinsic functions that require PowerISA 2.07 minimum. */ 1184 1185 /* Creates a 2-bit mask from the most significant bits of the DPFP values. */ 1186 extern __inline int 1187 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1188 _mm_movemask_pd(__m128d __A) { 1189 #ifdef _ARCH_PWR10 1190 return vec_extractm((__v2du)__A); 1191 #else 1192 __vector unsigned long long __result; 1193 static const __vector unsigned int __perm_mask = { 1194 #ifdef __LITTLE_ENDIAN__ 1195 0x80800040, 0x80808080, 0x80808080, 0x80808080 1196 #else 1197 0x80808080, 0x80808080, 0x80808080, 0x80804000 1198 #endif 1199 }; 1200 1201 __result = ((__vector unsigned long long)vec_vbpermq( 1202 (__vector unsigned char)__A, (__vector unsigned char)__perm_mask)); 1203 1204 #ifdef __LITTLE_ENDIAN__ 1205 return __result[1]; 1206 #else 1207 return __result[0]; 1208 #endif 1209 #endif /* !_ARCH_PWR10 */ 1210 } 1211 #endif /* _ARCH_PWR8 */ 1212 1213 extern __inline __m128i 1214 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1215 _mm_packs_epi16(__m128i __A, __m128i __B) { 1216 return (__m128i)vec_packs((__v8hi)__A, (__v8hi)__B); 1217 } 1218 1219 extern __inline __m128i 1220 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1221 _mm_packs_epi32(__m128i __A, __m128i __B) { 1222 return (__m128i)vec_packs((__v4si)__A, (__v4si)__B); 1223 } 1224 1225 extern __inline __m128i 1226 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1227 _mm_packus_epi16(__m128i __A, __m128i __B) { 1228 return (__m128i)vec_packsu((__v8hi)__A, (__v8hi)__B); 1229 } 1230 1231 extern __inline __m128i 1232 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1233 _mm_unpackhi_epi8(__m128i __A, __m128i __B) { 1234 return (__m128i)vec_mergel((__v16qu)__A, (__v16qu)__B); 1235 } 1236 1237 extern __inline __m128i 1238 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1239 _mm_unpackhi_epi16(__m128i __A, __m128i __B) { 1240 return (__m128i)vec_mergel((__v8hu)__A, (__v8hu)__B); 1241 } 1242 1243 extern __inline __m128i 1244 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1245 _mm_unpackhi_epi32(__m128i __A, __m128i __B) { 1246 return (__m128i)vec_mergel((__v4su)__A, (__v4su)__B); 1247 } 1248 1249 extern __inline __m128i 1250 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1251 _mm_unpackhi_epi64(__m128i __A, __m128i __B) { 1252 return (__m128i)vec_mergel((__vector long long)__A, (__vector long long)__B); 1253 } 1254 1255 extern __inline __m128i 1256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1257 _mm_unpacklo_epi8(__m128i __A, __m128i __B) { 1258 return (__m128i)vec_mergeh((__v16qu)__A, (__v16qu)__B); 1259 } 1260 1261 extern __inline __m128i 1262 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1263 _mm_unpacklo_epi16(__m128i __A, __m128i __B) { 1264 return (__m128i)vec_mergeh((__v8hi)__A, (__v8hi)__B); 1265 } 1266 1267 extern __inline __m128i 1268 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1269 _mm_unpacklo_epi32(__m128i __A, __m128i __B) { 1270 return (__m128i)vec_mergeh((__v4si)__A, (__v4si)__B); 1271 } 1272 1273 extern __inline __m128i 1274 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1275 _mm_unpacklo_epi64(__m128i __A, __m128i __B) { 1276 return (__m128i)vec_mergeh((__vector long long)__A, (__vector long long)__B); 1277 } 1278 1279 extern __inline __m128i 1280 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1281 _mm_add_epi8(__m128i __A, __m128i __B) { 1282 return (__m128i)((__v16qu)__A + (__v16qu)__B); 1283 } 1284 1285 extern __inline __m128i 1286 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1287 _mm_add_epi16(__m128i __A, __m128i __B) { 1288 return (__m128i)((__v8hu)__A + (__v8hu)__B); 1289 } 1290 1291 extern __inline __m128i 1292 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1293 _mm_add_epi32(__m128i __A, __m128i __B) { 1294 return (__m128i)((__v4su)__A + (__v4su)__B); 1295 } 1296 1297 extern __inline __m128i 1298 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1299 _mm_add_epi64(__m128i __A, __m128i __B) { 1300 return (__m128i)((__v2du)__A + (__v2du)__B); 1301 } 1302 1303 extern __inline __m128i 1304 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1305 _mm_adds_epi8(__m128i __A, __m128i __B) { 1306 return (__m128i)vec_adds((__v16qi)__A, (__v16qi)__B); 1307 } 1308 1309 extern __inline __m128i 1310 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1311 _mm_adds_epi16(__m128i __A, __m128i __B) { 1312 return (__m128i)vec_adds((__v8hi)__A, (__v8hi)__B); 1313 } 1314 1315 extern __inline __m128i 1316 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1317 _mm_adds_epu8(__m128i __A, __m128i __B) { 1318 return (__m128i)vec_adds((__v16qu)__A, (__v16qu)__B); 1319 } 1320 1321 extern __inline __m128i 1322 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1323 _mm_adds_epu16(__m128i __A, __m128i __B) { 1324 return (__m128i)vec_adds((__v8hu)__A, (__v8hu)__B); 1325 } 1326 1327 extern __inline __m128i 1328 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1329 _mm_sub_epi8(__m128i __A, __m128i __B) { 1330 return (__m128i)((__v16qu)__A - (__v16qu)__B); 1331 } 1332 1333 extern __inline __m128i 1334 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1335 _mm_sub_epi16(__m128i __A, __m128i __B) { 1336 return (__m128i)((__v8hu)__A - (__v8hu)__B); 1337 } 1338 1339 extern __inline __m128i 1340 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1341 _mm_sub_epi32(__m128i __A, __m128i __B) { 1342 return (__m128i)((__v4su)__A - (__v4su)__B); 1343 } 1344 1345 extern __inline __m128i 1346 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1347 _mm_sub_epi64(__m128i __A, __m128i __B) { 1348 return (__m128i)((__v2du)__A - (__v2du)__B); 1349 } 1350 1351 extern __inline __m128i 1352 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1353 _mm_subs_epi8(__m128i __A, __m128i __B) { 1354 return (__m128i)vec_subs((__v16qi)__A, (__v16qi)__B); 1355 } 1356 1357 extern __inline __m128i 1358 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1359 _mm_subs_epi16(__m128i __A, __m128i __B) { 1360 return (__m128i)vec_subs((__v8hi)__A, (__v8hi)__B); 1361 } 1362 1363 extern __inline __m128i 1364 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1365 _mm_subs_epu8(__m128i __A, __m128i __B) { 1366 return (__m128i)vec_subs((__v16qu)__A, (__v16qu)__B); 1367 } 1368 1369 extern __inline __m128i 1370 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1371 _mm_subs_epu16(__m128i __A, __m128i __B) { 1372 return (__m128i)vec_subs((__v8hu)__A, (__v8hu)__B); 1373 } 1374 1375 extern __inline __m128i 1376 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1377 _mm_madd_epi16(__m128i __A, __m128i __B) { 1378 __vector signed int __zero = {0, 0, 0, 0}; 1379 1380 return (__m128i)vec_vmsumshm((__v8hi)__A, (__v8hi)__B, __zero); 1381 } 1382 1383 extern __inline __m128i 1384 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1385 _mm_mulhi_epi16(__m128i __A, __m128i __B) { 1386 __vector signed int __w0, __w1; 1387 1388 __vector unsigned char __xform1 = { 1389 #ifdef __LITTLE_ENDIAN__ 1390 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A, 1391 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F 1392 #else 1393 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x08, 1394 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D 1395 #endif 1396 }; 1397 1398 __w0 = vec_vmulesh((__v8hi)__A, (__v8hi)__B); 1399 __w1 = vec_vmulosh((__v8hi)__A, (__v8hi)__B); 1400 return (__m128i)vec_perm(__w0, __w1, __xform1); 1401 } 1402 1403 extern __inline __m128i 1404 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1405 _mm_mullo_epi16(__m128i __A, __m128i __B) { 1406 return (__m128i)((__v8hi)__A * (__v8hi)__B); 1407 } 1408 1409 extern __inline __m64 1410 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1411 _mm_mul_su32(__m64 __A, __m64 __B) { 1412 unsigned int __a = __A; 1413 unsigned int __b = __B; 1414 1415 return ((__m64)__a * (__m64)__b); 1416 } 1417 1418 #ifdef _ARCH_PWR8 1419 extern __inline __m128i 1420 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1421 _mm_mul_epu32(__m128i __A, __m128i __B) { 1422 #if __GNUC__ < 8 1423 __v2du __result; 1424 1425 #ifdef __LITTLE_ENDIAN__ 1426 /* VMX Vector Multiply Odd Unsigned Word. */ 1427 __asm__("vmulouw %0,%1,%2" : "=v"(__result) : "v"(__A), "v"(__B) :); 1428 #else 1429 /* VMX Vector Multiply Even Unsigned Word. */ 1430 __asm__("vmuleuw %0,%1,%2" : "=v"(__result) : "v"(__A), "v"(__B) :); 1431 #endif 1432 return (__m128i)__result; 1433 #else 1434 return (__m128i)vec_mule((__v4su)__A, (__v4su)__B); 1435 #endif 1436 } 1437 #endif 1438 1439 extern __inline __m128i 1440 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1441 _mm_slli_epi16(__m128i __A, int __B) { 1442 __v8hu __lshift; 1443 __v8hi __result = {0, 0, 0, 0, 0, 0, 0, 0}; 1444 1445 if (__B >= 0 && __B < 16) { 1446 if (__builtin_constant_p(__B)) 1447 __lshift = (__v8hu)vec_splat_s16(__B); 1448 else 1449 __lshift = vec_splats((unsigned short)__B); 1450 1451 __result = vec_sl((__v8hi)__A, __lshift); 1452 } 1453 1454 return (__m128i)__result; 1455 } 1456 1457 extern __inline __m128i 1458 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1459 _mm_slli_epi32(__m128i __A, int __B) { 1460 __v4su __lshift; 1461 __v4si __result = {0, 0, 0, 0}; 1462 1463 if (__B >= 0 && __B < 32) { 1464 if (__builtin_constant_p(__B) && __B < 16) 1465 __lshift = (__v4su)vec_splat_s32(__B); 1466 else 1467 __lshift = vec_splats((unsigned int)__B); 1468 1469 __result = vec_sl((__v4si)__A, __lshift); 1470 } 1471 1472 return (__m128i)__result; 1473 } 1474 1475 #ifdef _ARCH_PWR8 1476 extern __inline __m128i 1477 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1478 _mm_slli_epi64(__m128i __A, int __B) { 1479 __v2du __lshift; 1480 __v2di __result = {0, 0}; 1481 1482 if (__B >= 0 && __B < 64) { 1483 if (__builtin_constant_p(__B) && __B < 16) 1484 __lshift = (__v2du)vec_splat_s32(__B); 1485 else 1486 __lshift = (__v2du)vec_splats((unsigned int)__B); 1487 1488 __result = vec_sl((__v2di)__A, __lshift); 1489 } 1490 1491 return (__m128i)__result; 1492 } 1493 #endif 1494 1495 extern __inline __m128i 1496 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1497 _mm_srai_epi16(__m128i __A, int __B) { 1498 __v8hu __rshift = {15, 15, 15, 15, 15, 15, 15, 15}; 1499 __v8hi __result; 1500 1501 if (__B < 16) { 1502 if (__builtin_constant_p(__B)) 1503 __rshift = (__v8hu)vec_splat_s16(__B); 1504 else 1505 __rshift = vec_splats((unsigned short)__B); 1506 } 1507 __result = vec_sra((__v8hi)__A, __rshift); 1508 1509 return (__m128i)__result; 1510 } 1511 1512 extern __inline __m128i 1513 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1514 _mm_srai_epi32(__m128i __A, int __B) { 1515 __v4su __rshift = {31, 31, 31, 31}; 1516 __v4si __result; 1517 1518 if (__B < 32) { 1519 if (__builtin_constant_p(__B)) { 1520 if (__B < 16) 1521 __rshift = (__v4su)vec_splat_s32(__B); 1522 else 1523 __rshift = (__v4su)vec_splats((unsigned int)__B); 1524 } else 1525 __rshift = vec_splats((unsigned int)__B); 1526 } 1527 __result = vec_sra((__v4si)__A, __rshift); 1528 1529 return (__m128i)__result; 1530 } 1531 1532 extern __inline __m128i 1533 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1534 _mm_bslli_si128(__m128i __A, const int __N) { 1535 __v16qu __result; 1536 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 1537 1538 if (__N < 16) 1539 __result = vec_sld((__v16qu)__A, __zeros, __N); 1540 else 1541 __result = __zeros; 1542 1543 return (__m128i)__result; 1544 } 1545 1546 extern __inline __m128i 1547 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1548 _mm_bsrli_si128(__m128i __A, const int __N) { 1549 __v16qu __result; 1550 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 1551 1552 if (__N < 16) 1553 #ifdef __LITTLE_ENDIAN__ 1554 if (__builtin_constant_p(__N)) 1555 /* Would like to use Vector Shift Left Double by Octet 1556 Immediate here to use the immediate form and avoid 1557 load of __N * 8 value into a separate VR. */ 1558 __result = vec_sld(__zeros, (__v16qu)__A, (16 - __N)); 1559 else 1560 #endif 1561 { 1562 __v16qu __shift = vec_splats((unsigned char)(__N * 8)); 1563 #ifdef __LITTLE_ENDIAN__ 1564 __result = vec_sro((__v16qu)__A, __shift); 1565 #else 1566 __result = vec_slo((__v16qu)__A, __shift); 1567 #endif 1568 } 1569 else 1570 __result = __zeros; 1571 1572 return (__m128i)__result; 1573 } 1574 1575 extern __inline __m128i 1576 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1577 _mm_srli_si128(__m128i __A, const int __N) { 1578 return _mm_bsrli_si128(__A, __N); 1579 } 1580 1581 extern __inline __m128i 1582 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1583 _mm_slli_si128(__m128i __A, const int _imm5) { 1584 __v16qu __result; 1585 const __v16qu __zeros = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 1586 1587 if (_imm5 < 16) 1588 #ifdef __LITTLE_ENDIAN__ 1589 __result = vec_sld((__v16qu)__A, __zeros, _imm5); 1590 #else 1591 __result = vec_sld(__zeros, (__v16qu)__A, (16 - _imm5)); 1592 #endif 1593 else 1594 __result = __zeros; 1595 1596 return (__m128i)__result; 1597 } 1598 1599 extern __inline __m128i 1600 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1601 1602 _mm_srli_epi16(__m128i __A, int __B) { 1603 __v8hu __rshift; 1604 __v8hi __result = {0, 0, 0, 0, 0, 0, 0, 0}; 1605 1606 if (__B < 16) { 1607 if (__builtin_constant_p(__B)) 1608 __rshift = (__v8hu)vec_splat_s16(__B); 1609 else 1610 __rshift = vec_splats((unsigned short)__B); 1611 1612 __result = vec_sr((__v8hi)__A, __rshift); 1613 } 1614 1615 return (__m128i)__result; 1616 } 1617 1618 extern __inline __m128i 1619 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1620 _mm_srli_epi32(__m128i __A, int __B) { 1621 __v4su __rshift; 1622 __v4si __result = {0, 0, 0, 0}; 1623 1624 if (__B < 32) { 1625 if (__builtin_constant_p(__B)) { 1626 if (__B < 16) 1627 __rshift = (__v4su)vec_splat_s32(__B); 1628 else 1629 __rshift = (__v4su)vec_splats((unsigned int)__B); 1630 } else 1631 __rshift = vec_splats((unsigned int)__B); 1632 1633 __result = vec_sr((__v4si)__A, __rshift); 1634 } 1635 1636 return (__m128i)__result; 1637 } 1638 1639 #ifdef _ARCH_PWR8 1640 extern __inline __m128i 1641 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1642 _mm_srli_epi64(__m128i __A, int __B) { 1643 __v2du __rshift; 1644 __v2di __result = {0, 0}; 1645 1646 if (__B < 64) { 1647 if (__builtin_constant_p(__B)) { 1648 if (__B < 16) 1649 __rshift = (__v2du)vec_splat_s32(__B); 1650 else 1651 __rshift = (__v2du)vec_splats((unsigned long long)__B); 1652 } else 1653 __rshift = (__v2du)vec_splats((unsigned int)__B); 1654 1655 __result = vec_sr((__v2di)__A, __rshift); 1656 } 1657 1658 return (__m128i)__result; 1659 } 1660 #endif 1661 1662 extern __inline __m128i 1663 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1664 _mm_sll_epi16(__m128i __A, __m128i __B) { 1665 __v8hu __lshift; 1666 __vector __bool short __shmask; 1667 const __v8hu __shmax = {15, 15, 15, 15, 15, 15, 15, 15}; 1668 __v8hu __result; 1669 1670 #ifdef __LITTLE_ENDIAN__ 1671 __lshift = vec_splat((__v8hu)__B, 0); 1672 #else 1673 __lshift = vec_splat((__v8hu)__B, 3); 1674 #endif 1675 __shmask = vec_cmple(__lshift, __shmax); 1676 __result = vec_sl((__v8hu)__A, __lshift); 1677 __result = vec_sel((__v8hu)__shmask, __result, __shmask); 1678 1679 return (__m128i)__result; 1680 } 1681 1682 extern __inline __m128i 1683 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1684 _mm_sll_epi32(__m128i __A, __m128i __B) { 1685 __v4su __lshift; 1686 __vector __bool int __shmask; 1687 const __v4su __shmax = {32, 32, 32, 32}; 1688 __v4su __result; 1689 #ifdef __LITTLE_ENDIAN__ 1690 __lshift = vec_splat((__v4su)__B, 0); 1691 #else 1692 __lshift = vec_splat((__v4su)__B, 1); 1693 #endif 1694 __shmask = vec_cmplt(__lshift, __shmax); 1695 __result = vec_sl((__v4su)__A, __lshift); 1696 __result = vec_sel((__v4su)__shmask, __result, __shmask); 1697 1698 return (__m128i)__result; 1699 } 1700 1701 #ifdef _ARCH_PWR8 1702 extern __inline __m128i 1703 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1704 _mm_sll_epi64(__m128i __A, __m128i __B) { 1705 __v2du __lshift; 1706 __vector __bool long long __shmask; 1707 const __v2du __shmax = {64, 64}; 1708 __v2du __result; 1709 1710 __lshift = vec_splat((__v2du)__B, 0); 1711 __shmask = vec_cmplt(__lshift, __shmax); 1712 __result = vec_sl((__v2du)__A, __lshift); 1713 __result = vec_sel((__v2du)__shmask, __result, __shmask); 1714 1715 return (__m128i)__result; 1716 } 1717 #endif 1718 1719 extern __inline __m128i 1720 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1721 _mm_sra_epi16(__m128i __A, __m128i __B) { 1722 const __v8hu __rshmax = {15, 15, 15, 15, 15, 15, 15, 15}; 1723 __v8hu __rshift; 1724 __v8hi __result; 1725 1726 #ifdef __LITTLE_ENDIAN__ 1727 __rshift = vec_splat((__v8hu)__B, 0); 1728 #else 1729 __rshift = vec_splat((__v8hu)__B, 3); 1730 #endif 1731 __rshift = vec_min(__rshift, __rshmax); 1732 __result = vec_sra((__v8hi)__A, __rshift); 1733 1734 return (__m128i)__result; 1735 } 1736 1737 extern __inline __m128i 1738 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1739 _mm_sra_epi32(__m128i __A, __m128i __B) { 1740 const __v4su __rshmax = {31, 31, 31, 31}; 1741 __v4su __rshift; 1742 __v4si __result; 1743 1744 #ifdef __LITTLE_ENDIAN__ 1745 __rshift = vec_splat((__v4su)__B, 0); 1746 #else 1747 __rshift = vec_splat((__v4su)__B, 1); 1748 #endif 1749 __rshift = vec_min(__rshift, __rshmax); 1750 __result = vec_sra((__v4si)__A, __rshift); 1751 1752 return (__m128i)__result; 1753 } 1754 1755 extern __inline __m128i 1756 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1757 _mm_srl_epi16(__m128i __A, __m128i __B) { 1758 __v8hu __rshift; 1759 __vector __bool short __shmask; 1760 const __v8hu __shmax = {15, 15, 15, 15, 15, 15, 15, 15}; 1761 __v8hu __result; 1762 1763 #ifdef __LITTLE_ENDIAN__ 1764 __rshift = vec_splat((__v8hu)__B, 0); 1765 #else 1766 __rshift = vec_splat((__v8hu)__B, 3); 1767 #endif 1768 __shmask = vec_cmple(__rshift, __shmax); 1769 __result = vec_sr((__v8hu)__A, __rshift); 1770 __result = vec_sel((__v8hu)__shmask, __result, __shmask); 1771 1772 return (__m128i)__result; 1773 } 1774 1775 extern __inline __m128i 1776 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1777 _mm_srl_epi32(__m128i __A, __m128i __B) { 1778 __v4su __rshift; 1779 __vector __bool int __shmask; 1780 const __v4su __shmax = {32, 32, 32, 32}; 1781 __v4su __result; 1782 1783 #ifdef __LITTLE_ENDIAN__ 1784 __rshift = vec_splat((__v4su)__B, 0); 1785 #else 1786 __rshift = vec_splat((__v4su)__B, 1); 1787 #endif 1788 __shmask = vec_cmplt(__rshift, __shmax); 1789 __result = vec_sr((__v4su)__A, __rshift); 1790 __result = vec_sel((__v4su)__shmask, __result, __shmask); 1791 1792 return (__m128i)__result; 1793 } 1794 1795 #ifdef _ARCH_PWR8 1796 extern __inline __m128i 1797 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1798 _mm_srl_epi64(__m128i __A, __m128i __B) { 1799 __v2du __rshift; 1800 __vector __bool long long __shmask; 1801 const __v2du __shmax = {64, 64}; 1802 __v2du __result; 1803 1804 __rshift = vec_splat((__v2du)__B, 0); 1805 __shmask = vec_cmplt(__rshift, __shmax); 1806 __result = vec_sr((__v2du)__A, __rshift); 1807 __result = vec_sel((__v2du)__shmask, __result, __shmask); 1808 1809 return (__m128i)__result; 1810 } 1811 #endif 1812 1813 extern __inline __m128d 1814 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1815 _mm_and_pd(__m128d __A, __m128d __B) { 1816 return (vec_and((__v2df)__A, (__v2df)__B)); 1817 } 1818 1819 extern __inline __m128d 1820 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1821 _mm_andnot_pd(__m128d __A, __m128d __B) { 1822 return (vec_andc((__v2df)__B, (__v2df)__A)); 1823 } 1824 1825 extern __inline __m128d 1826 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1827 _mm_or_pd(__m128d __A, __m128d __B) { 1828 return (vec_or((__v2df)__A, (__v2df)__B)); 1829 } 1830 1831 extern __inline __m128d 1832 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1833 _mm_xor_pd(__m128d __A, __m128d __B) { 1834 return (vec_xor((__v2df)__A, (__v2df)__B)); 1835 } 1836 1837 extern __inline __m128i 1838 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1839 _mm_and_si128(__m128i __A, __m128i __B) { 1840 return (__m128i)vec_and((__v2di)__A, (__v2di)__B); 1841 } 1842 1843 extern __inline __m128i 1844 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1845 _mm_andnot_si128(__m128i __A, __m128i __B) { 1846 return (__m128i)vec_andc((__v2di)__B, (__v2di)__A); 1847 } 1848 1849 extern __inline __m128i 1850 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1851 _mm_or_si128(__m128i __A, __m128i __B) { 1852 return (__m128i)vec_or((__v2di)__A, (__v2di)__B); 1853 } 1854 1855 extern __inline __m128i 1856 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1857 _mm_xor_si128(__m128i __A, __m128i __B) { 1858 return (__m128i)vec_xor((__v2di)__A, (__v2di)__B); 1859 } 1860 1861 extern __inline __m128i 1862 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1863 _mm_cmpeq_epi8(__m128i __A, __m128i __B) { 1864 return (__m128i)vec_cmpeq((__v16qi)__A, (__v16qi)__B); 1865 } 1866 1867 extern __inline __m128i 1868 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1869 _mm_cmpeq_epi16(__m128i __A, __m128i __B) { 1870 return (__m128i)vec_cmpeq((__v8hi)__A, (__v8hi)__B); 1871 } 1872 1873 extern __inline __m128i 1874 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1875 _mm_cmpeq_epi32(__m128i __A, __m128i __B) { 1876 return (__m128i)vec_cmpeq((__v4si)__A, (__v4si)__B); 1877 } 1878 1879 extern __inline __m128i 1880 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1881 _mm_cmplt_epi8(__m128i __A, __m128i __B) { 1882 return (__m128i)vec_cmplt((__v16qi)__A, (__v16qi)__B); 1883 } 1884 1885 extern __inline __m128i 1886 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1887 _mm_cmplt_epi16(__m128i __A, __m128i __B) { 1888 return (__m128i)vec_cmplt((__v8hi)__A, (__v8hi)__B); 1889 } 1890 1891 extern __inline __m128i 1892 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1893 _mm_cmplt_epi32(__m128i __A, __m128i __B) { 1894 return (__m128i)vec_cmplt((__v4si)__A, (__v4si)__B); 1895 } 1896 1897 extern __inline __m128i 1898 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1899 _mm_cmpgt_epi8(__m128i __A, __m128i __B) { 1900 return (__m128i)vec_cmpgt((__v16qi)__A, (__v16qi)__B); 1901 } 1902 1903 extern __inline __m128i 1904 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1905 _mm_cmpgt_epi16(__m128i __A, __m128i __B) { 1906 return (__m128i)vec_cmpgt((__v8hi)__A, (__v8hi)__B); 1907 } 1908 1909 extern __inline __m128i 1910 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1911 _mm_cmpgt_epi32(__m128i __A, __m128i __B) { 1912 return (__m128i)vec_cmpgt((__v4si)__A, (__v4si)__B); 1913 } 1914 1915 extern __inline int 1916 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1917 _mm_extract_epi16(__m128i const __A, int const __N) { 1918 return (unsigned short)((__v8hi)__A)[__N & 7]; 1919 } 1920 1921 extern __inline __m128i 1922 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1923 _mm_insert_epi16(__m128i const __A, int const __D, int const __N) { 1924 __v8hi __result = (__v8hi)__A; 1925 1926 __result[(__N & 7)] = __D; 1927 1928 return (__m128i)__result; 1929 } 1930 1931 extern __inline __m128i 1932 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1933 _mm_max_epi16(__m128i __A, __m128i __B) { 1934 return (__m128i)vec_max((__v8hi)__A, (__v8hi)__B); 1935 } 1936 1937 extern __inline __m128i 1938 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1939 _mm_max_epu8(__m128i __A, __m128i __B) { 1940 return (__m128i)vec_max((__v16qu)__A, (__v16qu)__B); 1941 } 1942 1943 extern __inline __m128i 1944 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1945 _mm_min_epi16(__m128i __A, __m128i __B) { 1946 return (__m128i)vec_min((__v8hi)__A, (__v8hi)__B); 1947 } 1948 1949 extern __inline __m128i 1950 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1951 _mm_min_epu8(__m128i __A, __m128i __B) { 1952 return (__m128i)vec_min((__v16qu)__A, (__v16qu)__B); 1953 } 1954 1955 #ifdef _ARCH_PWR8 1956 /* Intrinsic functions that require PowerISA 2.07 minimum. */ 1957 1958 /* Return a mask created from the most significant bit of each 8-bit 1959 element in A. */ 1960 extern __inline int 1961 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1962 _mm_movemask_epi8(__m128i __A) { 1963 #ifdef _ARCH_PWR10 1964 return vec_extractm((__v16qu)__A); 1965 #else 1966 __vector unsigned long long __result; 1967 static const __vector unsigned char __perm_mask = { 1968 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, 1969 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00}; 1970 1971 __result = ((__vector unsigned long long)vec_vbpermq( 1972 (__vector unsigned char)__A, (__vector unsigned char)__perm_mask)); 1973 1974 #ifdef __LITTLE_ENDIAN__ 1975 return __result[1]; 1976 #else 1977 return __result[0]; 1978 #endif 1979 #endif /* !_ARCH_PWR10 */ 1980 } 1981 #endif /* _ARCH_PWR8 */ 1982 1983 extern __inline __m128i 1984 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1985 _mm_mulhi_epu16(__m128i __A, __m128i __B) { 1986 __v4su __w0, __w1; 1987 __v16qu __xform1 = { 1988 #ifdef __LITTLE_ENDIAN__ 1989 0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A, 1990 0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F 1991 #else 1992 0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x08, 1993 0x09, 0x18, 0x19, 0x0C, 0x0D, 0x1C, 0x1D 1994 #endif 1995 }; 1996 1997 __w0 = vec_vmuleuh((__v8hu)__A, (__v8hu)__B); 1998 __w1 = vec_vmulouh((__v8hu)__A, (__v8hu)__B); 1999 return (__m128i)vec_perm(__w0, __w1, __xform1); 2000 } 2001 2002 extern __inline __m128i 2003 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2004 _mm_shufflehi_epi16(__m128i __A, const int __mask) { 2005 unsigned long __element_selector_98 = __mask & 0x03; 2006 unsigned long __element_selector_BA = (__mask >> 2) & 0x03; 2007 unsigned long __element_selector_DC = (__mask >> 4) & 0x03; 2008 unsigned long __element_selector_FE = (__mask >> 6) & 0x03; 2009 static const unsigned short __permute_selectors[4] = { 2010 #ifdef __LITTLE_ENDIAN__ 2011 0x0908, 0x0B0A, 0x0D0C, 0x0F0E 2012 #else 2013 0x0809, 0x0A0B, 0x0C0D, 0x0E0F 2014 #endif 2015 }; 2016 __v2du __pmask = 2017 #ifdef __LITTLE_ENDIAN__ 2018 {0x1716151413121110UL, 0UL}; 2019 #else 2020 {0x1011121314151617UL, 0UL}; 2021 #endif 2022 __m64_union __t; 2023 __v2du __a, __r; 2024 2025 __t.as_short[0] = __permute_selectors[__element_selector_98]; 2026 __t.as_short[1] = __permute_selectors[__element_selector_BA]; 2027 __t.as_short[2] = __permute_selectors[__element_selector_DC]; 2028 __t.as_short[3] = __permute_selectors[__element_selector_FE]; 2029 __pmask[1] = __t.as_m64; 2030 __a = (__v2du)__A; 2031 __r = vec_perm(__a, __a, (__vector unsigned char)__pmask); 2032 return (__m128i)__r; 2033 } 2034 2035 extern __inline __m128i 2036 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2037 _mm_shufflelo_epi16(__m128i __A, const int __mask) { 2038 unsigned long __element_selector_10 = __mask & 0x03; 2039 unsigned long __element_selector_32 = (__mask >> 2) & 0x03; 2040 unsigned long __element_selector_54 = (__mask >> 4) & 0x03; 2041 unsigned long __element_selector_76 = (__mask >> 6) & 0x03; 2042 static const unsigned short __permute_selectors[4] = { 2043 #ifdef __LITTLE_ENDIAN__ 2044 0x0100, 0x0302, 0x0504, 0x0706 2045 #else 2046 0x0001, 0x0203, 0x0405, 0x0607 2047 #endif 2048 }; 2049 __v2du __pmask = 2050 #ifdef __LITTLE_ENDIAN__ 2051 {0UL, 0x1f1e1d1c1b1a1918UL}; 2052 #else 2053 {0UL, 0x18191a1b1c1d1e1fUL}; 2054 #endif 2055 __m64_union __t; 2056 __v2du __a, __r; 2057 __t.as_short[0] = __permute_selectors[__element_selector_10]; 2058 __t.as_short[1] = __permute_selectors[__element_selector_32]; 2059 __t.as_short[2] = __permute_selectors[__element_selector_54]; 2060 __t.as_short[3] = __permute_selectors[__element_selector_76]; 2061 __pmask[0] = __t.as_m64; 2062 __a = (__v2du)__A; 2063 __r = vec_perm(__a, __a, (__vector unsigned char)__pmask); 2064 return (__m128i)__r; 2065 } 2066 2067 extern __inline __m128i 2068 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2069 _mm_shuffle_epi32(__m128i __A, const int __mask) { 2070 unsigned long __element_selector_10 = __mask & 0x03; 2071 unsigned long __element_selector_32 = (__mask >> 2) & 0x03; 2072 unsigned long __element_selector_54 = (__mask >> 4) & 0x03; 2073 unsigned long __element_selector_76 = (__mask >> 6) & 0x03; 2074 static const unsigned int __permute_selectors[4] = { 2075 #ifdef __LITTLE_ENDIAN__ 2076 0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C 2077 #else 2078 0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F 2079 #endif 2080 }; 2081 __v4su __t; 2082 2083 __t[0] = __permute_selectors[__element_selector_10]; 2084 __t[1] = __permute_selectors[__element_selector_32]; 2085 __t[2] = __permute_selectors[__element_selector_54] + 0x10101010; 2086 __t[3] = __permute_selectors[__element_selector_76] + 0x10101010; 2087 return (__m128i)vec_perm((__v4si)__A, (__v4si)__A, 2088 (__vector unsigned char)__t); 2089 } 2090 2091 extern __inline void 2092 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2093 _mm_maskmoveu_si128(__m128i __A, __m128i __B, char *__C) { 2094 __v2du __hibit = {0x7f7f7f7f7f7f7f7fUL, 0x7f7f7f7f7f7f7f7fUL}; 2095 __v16qu __mask, __tmp; 2096 __m128i_u *__p = (__m128i_u *)__C; 2097 2098 __tmp = (__v16qu)_mm_loadu_si128(__p); 2099 __mask = (__v16qu)vec_cmpgt((__v16qu)__B, (__v16qu)__hibit); 2100 __tmp = vec_sel(__tmp, (__v16qu)__A, __mask); 2101 _mm_storeu_si128(__p, (__m128i)__tmp); 2102 } 2103 2104 extern __inline __m128i 2105 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2106 _mm_avg_epu8(__m128i __A, __m128i __B) { 2107 return (__m128i)vec_avg((__v16qu)__A, (__v16qu)__B); 2108 } 2109 2110 extern __inline __m128i 2111 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2112 _mm_avg_epu16(__m128i __A, __m128i __B) { 2113 return (__m128i)vec_avg((__v8hu)__A, (__v8hu)__B); 2114 } 2115 2116 extern __inline __m128i 2117 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2118 _mm_sad_epu8(__m128i __A, __m128i __B) { 2119 __v16qu __a, __b; 2120 __v16qu __vabsdiff; 2121 __v4si __vsum; 2122 const __v4su __zero = {0, 0, 0, 0}; 2123 __v4si __result; 2124 2125 __a = (__v16qu)__A; 2126 __b = (__v16qu)__B; 2127 #ifndef _ARCH_PWR9 2128 __v16qu __vmin = vec_min(__a, __b); 2129 __v16qu __vmax = vec_max(__a, __b); 2130 __vabsdiff = vec_sub(__vmax, __vmin); 2131 #else 2132 __vabsdiff = vec_absd(__a, __b); 2133 #endif 2134 /* Sum four groups of bytes into integers. */ 2135 __vsum = (__vector signed int)vec_sum4s(__vabsdiff, __zero); 2136 #ifdef __LITTLE_ENDIAN__ 2137 /* Sum across four integers with two integer results. */ 2138 __asm__("vsum2sws %0,%1,%2" : "=v"(__result) : "v"(__vsum), "v"(__zero)); 2139 /* Note: vec_sum2s could be used here, but on little-endian, vector 2140 shifts are added that are not needed for this use-case. 2141 A vector shift to correctly position the 32-bit integer results 2142 (currently at [0] and [2]) to [1] and [3] would then need to be 2143 swapped back again since the desired results are two 64-bit 2144 integers ([1]|[0] and [3]|[2]). Thus, no shift is performed. */ 2145 #else 2146 /* Sum across four integers with two integer results. */ 2147 __result = vec_sum2s(__vsum, (__vector signed int)__zero); 2148 /* Rotate the sums into the correct position. */ 2149 __result = vec_sld(__result, __result, 6); 2150 #endif 2151 return (__m128i)__result; 2152 } 2153 2154 extern __inline void 2155 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2156 _mm_stream_si32(int *__A, int __B) { 2157 /* Use the data cache block touch for store transient. */ 2158 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory"); 2159 *__A = __B; 2160 } 2161 2162 extern __inline void 2163 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2164 _mm_stream_si64(long long int *__A, long long int __B) { 2165 /* Use the data cache block touch for store transient. */ 2166 __asm__(" dcbtstt 0,%0" : : "b"(__A) : "memory"); 2167 *__A = __B; 2168 } 2169 2170 extern __inline void 2171 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2172 _mm_stream_si128(__m128i *__A, __m128i __B) { 2173 /* Use the data cache block touch for store transient. */ 2174 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory"); 2175 *__A = __B; 2176 } 2177 2178 extern __inline void 2179 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2180 _mm_stream_pd(double *__A, __m128d __B) { 2181 /* Use the data cache block touch for store transient. */ 2182 __asm__("dcbtstt 0,%0" : : "b"(__A) : "memory"); 2183 *(__m128d *)__A = __B; 2184 } 2185 2186 extern __inline void 2187 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2188 _mm_clflush(void const *__A) { 2189 /* Use the data cache block flush. */ 2190 __asm__("dcbf 0,%0" : : "b"(__A) : "memory"); 2191 } 2192 2193 extern __inline void 2194 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2195 _mm_lfence(void) { 2196 /* Use light weight sync for load to load ordering. */ 2197 __atomic_thread_fence(__ATOMIC_RELEASE); 2198 } 2199 2200 extern __inline void 2201 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2202 _mm_mfence(void) { 2203 /* Use heavy weight sync for any to any ordering. */ 2204 __atomic_thread_fence(__ATOMIC_SEQ_CST); 2205 } 2206 2207 extern __inline __m128i 2208 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2209 _mm_cvtsi32_si128(int __A) { 2210 return _mm_set_epi32(0, 0, 0, __A); 2211 } 2212 2213 extern __inline __m128i 2214 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2215 _mm_cvtsi64_si128(long long __A) { 2216 return __extension__(__m128i)(__v2di){__A, 0LL}; 2217 } 2218 2219 /* Microsoft intrinsic. */ 2220 extern __inline __m128i 2221 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2222 _mm_cvtsi64x_si128(long long __A) { 2223 return __extension__(__m128i)(__v2di){__A, 0LL}; 2224 } 2225 2226 /* Casts between various SP, DP, INT vector types. Note that these do no 2227 conversion of values, they just change the type. */ 2228 extern __inline __m128 2229 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2230 _mm_castpd_ps(__m128d __A) { 2231 return (__m128)__A; 2232 } 2233 2234 extern __inline __m128i 2235 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2236 _mm_castpd_si128(__m128d __A) { 2237 return (__m128i)__A; 2238 } 2239 2240 extern __inline __m128d 2241 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2242 _mm_castps_pd(__m128 __A) { 2243 return (__m128d)__A; 2244 } 2245 2246 extern __inline __m128i 2247 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2248 _mm_castps_si128(__m128 __A) { 2249 return (__m128i)__A; 2250 } 2251 2252 extern __inline __m128 2253 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2254 _mm_castsi128_ps(__m128i __A) { 2255 return (__m128)__A; 2256 } 2257 2258 extern __inline __m128d 2259 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 2260 _mm_castsi128_pd(__m128i __A) { 2261 return (__m128d)__A; 2262 } 2263 2264 #else 2265 #include_next <emmintrin.h> 2266 #endif /* defined(__powerpc64__) && \ 2267 * (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) */ 2268 2269 #endif /* EMMINTRIN_H_ */ 2270