10b57cec5SDimitry Andric /*===---- immintrin.h - Intel intrinsics -----------------------------------=== 20b57cec5SDimitry Andric * 30b57cec5SDimitry Andric * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric * See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric * 70b57cec5SDimitry Andric *===-----------------------------------------------------------------------=== 80b57cec5SDimitry Andric */ 90b57cec5SDimitry Andric 100b57cec5SDimitry Andric #ifndef __IMMINTRIN_H 110b57cec5SDimitry Andric #define __IMMINTRIN_H 120b57cec5SDimitry Andric 13e8d8bef9SDimitry Andric #include <x86gprintrin.h> 14e8d8bef9SDimitry Andric 155ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 165ffd83dbSDimitry Andric defined(__MMX__) 170b57cec5SDimitry Andric #include <mmintrin.h> 180b57cec5SDimitry Andric #endif 190b57cec5SDimitry Andric 205ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 215ffd83dbSDimitry Andric defined(__SSE__) 220b57cec5SDimitry Andric #include <xmmintrin.h> 230b57cec5SDimitry Andric #endif 240b57cec5SDimitry Andric 255ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 265ffd83dbSDimitry Andric defined(__SSE2__) 270b57cec5SDimitry Andric #include <emmintrin.h> 280b57cec5SDimitry Andric #endif 290b57cec5SDimitry Andric 305ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 315ffd83dbSDimitry Andric defined(__SSE3__) 320b57cec5SDimitry Andric #include <pmmintrin.h> 330b57cec5SDimitry Andric #endif 340b57cec5SDimitry Andric 355ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 365ffd83dbSDimitry Andric defined(__SSSE3__) 370b57cec5SDimitry Andric #include <tmmintrin.h> 380b57cec5SDimitry Andric #endif 390b57cec5SDimitry Andric 405ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 410b57cec5SDimitry Andric (defined(__SSE4_2__) || defined(__SSE4_1__)) 420b57cec5SDimitry Andric #include <smmintrin.h> 430b57cec5SDimitry Andric #endif 440b57cec5SDimitry Andric 455ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 460b57cec5SDimitry Andric (defined(__AES__) || defined(__PCLMUL__)) 470b57cec5SDimitry Andric #include <wmmintrin.h> 480b57cec5SDimitry Andric #endif 490b57cec5SDimitry Andric 505ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 515ffd83dbSDimitry Andric defined(__CLFLUSHOPT__) 520b57cec5SDimitry Andric #include <clflushoptintrin.h> 530b57cec5SDimitry Andric #endif 540b57cec5SDimitry Andric 555ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 565ffd83dbSDimitry Andric defined(__CLWB__) 570b57cec5SDimitry Andric #include <clwbintrin.h> 580b57cec5SDimitry Andric #endif 590b57cec5SDimitry Andric 605ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 615ffd83dbSDimitry Andric defined(__AVX__) 620b57cec5SDimitry Andric #include <avxintrin.h> 630b57cec5SDimitry Andric #endif 640b57cec5SDimitry Andric 655ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 665ffd83dbSDimitry Andric defined(__AVX2__) 670b57cec5SDimitry Andric #include <avx2intrin.h> 680b57cec5SDimitry Andric #endif 690b57cec5SDimitry Andric 705ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 715ffd83dbSDimitry Andric defined(__F16C__) 720b57cec5SDimitry Andric #include <f16cintrin.h> 730b57cec5SDimitry Andric #endif 740b57cec5SDimitry Andric 75a7dea167SDimitry Andric /* No feature check desired due to internal checks */ 760b57cec5SDimitry Andric #include <bmiintrin.h> 770b57cec5SDimitry Andric 785ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 795ffd83dbSDimitry Andric defined(__BMI2__) 800b57cec5SDimitry Andric #include <bmi2intrin.h> 810b57cec5SDimitry Andric #endif 820b57cec5SDimitry Andric 835ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 845ffd83dbSDimitry Andric defined(__LZCNT__) 850b57cec5SDimitry Andric #include <lzcntintrin.h> 860b57cec5SDimitry Andric #endif 870b57cec5SDimitry Andric 885ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 895ffd83dbSDimitry Andric defined(__POPCNT__) 900b57cec5SDimitry Andric #include <popcntintrin.h> 910b57cec5SDimitry Andric #endif 920b57cec5SDimitry Andric 935ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 945ffd83dbSDimitry Andric defined(__FMA__) 950b57cec5SDimitry Andric #include <fmaintrin.h> 960b57cec5SDimitry Andric #endif 970b57cec5SDimitry Andric 985ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 995ffd83dbSDimitry Andric defined(__AVX512F__) 1000b57cec5SDimitry Andric #include <avx512fintrin.h> 1010b57cec5SDimitry Andric #endif 1020b57cec5SDimitry Andric 1035ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1045ffd83dbSDimitry Andric defined(__AVX512VL__) 1050b57cec5SDimitry Andric #include <avx512vlintrin.h> 1060b57cec5SDimitry Andric #endif 1070b57cec5SDimitry Andric 1085ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1095ffd83dbSDimitry Andric defined(__AVX512BW__) 1100b57cec5SDimitry Andric #include <avx512bwintrin.h> 1110b57cec5SDimitry Andric #endif 1120b57cec5SDimitry Andric 1135ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1145ffd83dbSDimitry Andric defined(__AVX512BITALG__) 1150b57cec5SDimitry Andric #include <avx512bitalgintrin.h> 1160b57cec5SDimitry Andric #endif 1170b57cec5SDimitry Andric 1185ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1195ffd83dbSDimitry Andric defined(__AVX512CD__) 1200b57cec5SDimitry Andric #include <avx512cdintrin.h> 1210b57cec5SDimitry Andric #endif 1220b57cec5SDimitry Andric 1235ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1245ffd83dbSDimitry Andric defined(__AVX512VPOPCNTDQ__) 1250b57cec5SDimitry Andric #include <avx512vpopcntdqintrin.h> 1260b57cec5SDimitry Andric #endif 1270b57cec5SDimitry Andric 1285ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1290b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 1300b57cec5SDimitry Andric #include <avx512vpopcntdqvlintrin.h> 1310b57cec5SDimitry Andric #endif 1320b57cec5SDimitry Andric 1335ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1345ffd83dbSDimitry Andric defined(__AVX512VNNI__) 1350b57cec5SDimitry Andric #include <avx512vnniintrin.h> 1360b57cec5SDimitry Andric #endif 1370b57cec5SDimitry Andric 1385ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1390b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 1400b57cec5SDimitry Andric #include <avx512vlvnniintrin.h> 1410b57cec5SDimitry Andric #endif 1420b57cec5SDimitry Andric 1435ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 144e8d8bef9SDimitry Andric defined(__AVXVNNI__) 145e8d8bef9SDimitry Andric #include <avxvnniintrin.h> 146e8d8bef9SDimitry Andric #endif 147e8d8bef9SDimitry Andric 148e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1495ffd83dbSDimitry Andric defined(__AVX512DQ__) 1500b57cec5SDimitry Andric #include <avx512dqintrin.h> 1510b57cec5SDimitry Andric #endif 1520b57cec5SDimitry Andric 1535ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1540b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 1550b57cec5SDimitry Andric #include <avx512vlbitalgintrin.h> 1560b57cec5SDimitry Andric #endif 1570b57cec5SDimitry Andric 1585ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1590b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BW__)) 1600b57cec5SDimitry Andric #include <avx512vlbwintrin.h> 1610b57cec5SDimitry Andric #endif 1620b57cec5SDimitry Andric 1635ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1640b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512CD__)) 1650b57cec5SDimitry Andric #include <avx512vlcdintrin.h> 1660b57cec5SDimitry Andric #endif 1670b57cec5SDimitry Andric 1685ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1690b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512DQ__)) 1700b57cec5SDimitry Andric #include <avx512vldqintrin.h> 1710b57cec5SDimitry Andric #endif 1720b57cec5SDimitry Andric 1735ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1745ffd83dbSDimitry Andric defined(__AVX512ER__) 1750b57cec5SDimitry Andric #include <avx512erintrin.h> 1760b57cec5SDimitry Andric #endif 1770b57cec5SDimitry Andric 1785ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1795ffd83dbSDimitry Andric defined(__AVX512IFMA__) 1800b57cec5SDimitry Andric #include <avx512ifmaintrin.h> 1810b57cec5SDimitry Andric #endif 1820b57cec5SDimitry Andric 1835ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1840b57cec5SDimitry Andric (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 1850b57cec5SDimitry Andric #include <avx512ifmavlintrin.h> 1860b57cec5SDimitry Andric #endif 1870b57cec5SDimitry Andric 1885ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1895ffd83dbSDimitry Andric defined(__AVX512VBMI__) 1900b57cec5SDimitry Andric #include <avx512vbmiintrin.h> 1910b57cec5SDimitry Andric #endif 1920b57cec5SDimitry Andric 1935ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1940b57cec5SDimitry Andric (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 1950b57cec5SDimitry Andric #include <avx512vbmivlintrin.h> 1960b57cec5SDimitry Andric #endif 1970b57cec5SDimitry Andric 1985ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1995ffd83dbSDimitry Andric defined(__AVX512VBMI2__) 2000b57cec5SDimitry Andric #include <avx512vbmi2intrin.h> 2010b57cec5SDimitry Andric #endif 2020b57cec5SDimitry Andric 2035ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2040b57cec5SDimitry Andric (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 2050b57cec5SDimitry Andric #include <avx512vlvbmi2intrin.h> 2060b57cec5SDimitry Andric #endif 2070b57cec5SDimitry Andric 2085ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2095ffd83dbSDimitry Andric defined(__AVX512PF__) 2100b57cec5SDimitry Andric #include <avx512pfintrin.h> 2110b57cec5SDimitry Andric #endif 2120b57cec5SDimitry Andric 2135ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2145ffd83dbSDimitry Andric defined(__AVX512BF16__) 2150b57cec5SDimitry Andric #include <avx512bf16intrin.h> 2160b57cec5SDimitry Andric #endif 2170b57cec5SDimitry Andric 2185ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2190b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BF16__)) 2200b57cec5SDimitry Andric #include <avx512vlbf16intrin.h> 2210b57cec5SDimitry Andric #endif 2220b57cec5SDimitry Andric 2235ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2245ffd83dbSDimitry Andric defined(__PKU__) 2250b57cec5SDimitry Andric #include <pkuintrin.h> 2260b57cec5SDimitry Andric #endif 2270b57cec5SDimitry Andric 2285ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 229fe6060f1SDimitry Andric defined(__VPCLMULQDQ__) 230fe6060f1SDimitry Andric #include <vpclmulqdqintrin.h> 231fe6060f1SDimitry Andric #endif 232fe6060f1SDimitry Andric 233fe6060f1SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2345ffd83dbSDimitry Andric defined(__VAES__) 2350b57cec5SDimitry Andric #include <vaesintrin.h> 2360b57cec5SDimitry Andric #endif 2370b57cec5SDimitry Andric 2385ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2395ffd83dbSDimitry Andric defined(__GFNI__) 2400b57cec5SDimitry Andric #include <gfniintrin.h> 2410b57cec5SDimitry Andric #endif 2420b57cec5SDimitry Andric 2435ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2445ffd83dbSDimitry Andric defined(__RDPID__) 2450b57cec5SDimitry Andric /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103). 2460b57cec5SDimitry Andric /// 2470b57cec5SDimitry Andric /// \headerfile <immintrin.h> 2480b57cec5SDimitry Andric /// 2490b57cec5SDimitry Andric /// This intrinsic corresponds to the <c> RDPID </c> instruction. 2500b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) 2510b57cec5SDimitry Andric _rdpid_u32(void) { 2520b57cec5SDimitry Andric return __builtin_ia32_rdpid(); 2530b57cec5SDimitry Andric } 2540b57cec5SDimitry Andric #endif // __RDPID__ 2550b57cec5SDimitry Andric 2565ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2575ffd83dbSDimitry Andric defined(__RDRND__) 2580b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 2590b57cec5SDimitry Andric _rdrand16_step(unsigned short *__p) 2600b57cec5SDimitry Andric { 2610b57cec5SDimitry Andric return __builtin_ia32_rdrand16_step(__p); 2620b57cec5SDimitry Andric } 2630b57cec5SDimitry Andric 2640b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 2650b57cec5SDimitry Andric _rdrand32_step(unsigned int *__p) 2660b57cec5SDimitry Andric { 2670b57cec5SDimitry Andric return __builtin_ia32_rdrand32_step(__p); 2680b57cec5SDimitry Andric } 2690b57cec5SDimitry Andric 2700b57cec5SDimitry Andric #ifdef __x86_64__ 2710b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 2720b57cec5SDimitry Andric _rdrand64_step(unsigned long long *__p) 2730b57cec5SDimitry Andric { 2740b57cec5SDimitry Andric return __builtin_ia32_rdrand64_step(__p); 2750b57cec5SDimitry Andric } 2760b57cec5SDimitry Andric #endif 2770b57cec5SDimitry Andric #endif /* __RDRND__ */ 2780b57cec5SDimitry Andric 2795ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2805ffd83dbSDimitry Andric defined(__FSGSBASE__) 2810b57cec5SDimitry Andric #ifdef __x86_64__ 2820b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 2830b57cec5SDimitry Andric _readfsbase_u32(void) 2840b57cec5SDimitry Andric { 2850b57cec5SDimitry Andric return __builtin_ia32_rdfsbase32(); 2860b57cec5SDimitry Andric } 2870b57cec5SDimitry Andric 2880b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 2890b57cec5SDimitry Andric _readfsbase_u64(void) 2900b57cec5SDimitry Andric { 2910b57cec5SDimitry Andric return __builtin_ia32_rdfsbase64(); 2920b57cec5SDimitry Andric } 2930b57cec5SDimitry Andric 2940b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 2950b57cec5SDimitry Andric _readgsbase_u32(void) 2960b57cec5SDimitry Andric { 2970b57cec5SDimitry Andric return __builtin_ia32_rdgsbase32(); 2980b57cec5SDimitry Andric } 2990b57cec5SDimitry Andric 3000b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3010b57cec5SDimitry Andric _readgsbase_u64(void) 3020b57cec5SDimitry Andric { 3030b57cec5SDimitry Andric return __builtin_ia32_rdgsbase64(); 3040b57cec5SDimitry Andric } 3050b57cec5SDimitry Andric 3060b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3070b57cec5SDimitry Andric _writefsbase_u32(unsigned int __V) 3080b57cec5SDimitry Andric { 3090b57cec5SDimitry Andric __builtin_ia32_wrfsbase32(__V); 3100b57cec5SDimitry Andric } 3110b57cec5SDimitry Andric 3120b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3130b57cec5SDimitry Andric _writefsbase_u64(unsigned long long __V) 3140b57cec5SDimitry Andric { 3150b57cec5SDimitry Andric __builtin_ia32_wrfsbase64(__V); 3160b57cec5SDimitry Andric } 3170b57cec5SDimitry Andric 3180b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3190b57cec5SDimitry Andric _writegsbase_u32(unsigned int __V) 3200b57cec5SDimitry Andric { 3210b57cec5SDimitry Andric __builtin_ia32_wrgsbase32(__V); 3220b57cec5SDimitry Andric } 3230b57cec5SDimitry Andric 3240b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3250b57cec5SDimitry Andric _writegsbase_u64(unsigned long long __V) 3260b57cec5SDimitry Andric { 3270b57cec5SDimitry Andric __builtin_ia32_wrgsbase64(__V); 3280b57cec5SDimitry Andric } 3290b57cec5SDimitry Andric 3300b57cec5SDimitry Andric #endif 3310b57cec5SDimitry Andric #endif /* __FSGSBASE__ */ 3320b57cec5SDimitry Andric 3335ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3345ffd83dbSDimitry Andric defined(__MOVBE__) 3350b57cec5SDimitry Andric 3360b57cec5SDimitry Andric /* The structs used below are to force the load/store to be unaligned. This 3370b57cec5SDimitry Andric * is accomplished with the __packed__ attribute. The __may_alias__ prevents 3380b57cec5SDimitry Andric * tbaa metadata from being generated based on the struct and the type of the 3390b57cec5SDimitry Andric * field inside of it. 3400b57cec5SDimitry Andric */ 3410b57cec5SDimitry Andric 3420b57cec5SDimitry Andric static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3430b57cec5SDimitry Andric _loadbe_i16(void const * __P) { 3440b57cec5SDimitry Andric struct __loadu_i16 { 3450b57cec5SDimitry Andric short __v; 3460b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 347480093f4SDimitry Andric return __builtin_bswap16(((const struct __loadu_i16*)__P)->__v); 3480b57cec5SDimitry Andric } 3490b57cec5SDimitry Andric 3500b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3510b57cec5SDimitry Andric _storebe_i16(void * __P, short __D) { 3520b57cec5SDimitry Andric struct __storeu_i16 { 3530b57cec5SDimitry Andric short __v; 3540b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 3550b57cec5SDimitry Andric ((struct __storeu_i16*)__P)->__v = __builtin_bswap16(__D); 3560b57cec5SDimitry Andric } 3570b57cec5SDimitry Andric 3580b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3590b57cec5SDimitry Andric _loadbe_i32(void const * __P) { 3600b57cec5SDimitry Andric struct __loadu_i32 { 3610b57cec5SDimitry Andric int __v; 3620b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 363480093f4SDimitry Andric return __builtin_bswap32(((const struct __loadu_i32*)__P)->__v); 3640b57cec5SDimitry Andric } 3650b57cec5SDimitry Andric 3660b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3670b57cec5SDimitry Andric _storebe_i32(void * __P, int __D) { 3680b57cec5SDimitry Andric struct __storeu_i32 { 3690b57cec5SDimitry Andric int __v; 3700b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 3710b57cec5SDimitry Andric ((struct __storeu_i32*)__P)->__v = __builtin_bswap32(__D); 3720b57cec5SDimitry Andric } 3730b57cec5SDimitry Andric 3740b57cec5SDimitry Andric #ifdef __x86_64__ 3750b57cec5SDimitry Andric static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3760b57cec5SDimitry Andric _loadbe_i64(void const * __P) { 3770b57cec5SDimitry Andric struct __loadu_i64 { 3780b57cec5SDimitry Andric long long __v; 3790b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 380480093f4SDimitry Andric return __builtin_bswap64(((const struct __loadu_i64*)__P)->__v); 3810b57cec5SDimitry Andric } 3820b57cec5SDimitry Andric 3830b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3840b57cec5SDimitry Andric _storebe_i64(void * __P, long long __D) { 3850b57cec5SDimitry Andric struct __storeu_i64 { 3860b57cec5SDimitry Andric long long __v; 3870b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 3880b57cec5SDimitry Andric ((struct __storeu_i64*)__P)->__v = __builtin_bswap64(__D); 3890b57cec5SDimitry Andric } 3900b57cec5SDimitry Andric #endif 3910b57cec5SDimitry Andric #endif /* __MOVBE */ 3920b57cec5SDimitry Andric 3935ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3945ffd83dbSDimitry Andric defined(__RTM__) 3950b57cec5SDimitry Andric #include <rtmintrin.h> 3960b57cec5SDimitry Andric #include <xtestintrin.h> 3970b57cec5SDimitry Andric #endif 3980b57cec5SDimitry Andric 3995ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4005ffd83dbSDimitry Andric defined(__SHA__) 4010b57cec5SDimitry Andric #include <shaintrin.h> 4020b57cec5SDimitry Andric #endif 4030b57cec5SDimitry Andric 4045ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4055ffd83dbSDimitry Andric defined(__FXSR__) 4060b57cec5SDimitry Andric #include <fxsrintrin.h> 4070b57cec5SDimitry Andric #endif 4080b57cec5SDimitry Andric 4090b57cec5SDimitry Andric /* No feature check desired due to internal MSC_VER checks */ 4100b57cec5SDimitry Andric #include <xsaveintrin.h> 4110b57cec5SDimitry Andric 4125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4135ffd83dbSDimitry Andric defined(__XSAVEOPT__) 4140b57cec5SDimitry Andric #include <xsaveoptintrin.h> 4150b57cec5SDimitry Andric #endif 4160b57cec5SDimitry Andric 4175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4185ffd83dbSDimitry Andric defined(__XSAVEC__) 4190b57cec5SDimitry Andric #include <xsavecintrin.h> 4200b57cec5SDimitry Andric #endif 4210b57cec5SDimitry Andric 4225ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4235ffd83dbSDimitry Andric defined(__XSAVES__) 4240b57cec5SDimitry Andric #include <xsavesintrin.h> 4250b57cec5SDimitry Andric #endif 4260b57cec5SDimitry Andric 4275ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4285ffd83dbSDimitry Andric defined(__SHSTK__) 4290b57cec5SDimitry Andric #include <cetintrin.h> 4300b57cec5SDimitry Andric #endif 4310b57cec5SDimitry Andric 4320b57cec5SDimitry Andric /* Some intrinsics inside adxintrin.h are available only on processors with ADX, 4330b57cec5SDimitry Andric * whereas others are also available at all times. */ 4340b57cec5SDimitry Andric #include <adxintrin.h> 4350b57cec5SDimitry Andric 4365ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4375ffd83dbSDimitry Andric defined(__RDSEED__) 4380b57cec5SDimitry Andric #include <rdseedintrin.h> 4390b57cec5SDimitry Andric #endif 4400b57cec5SDimitry Andric 4415ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4425ffd83dbSDimitry Andric defined(__WBNOINVD__) 4430b57cec5SDimitry Andric #include <wbnoinvdintrin.h> 4440b57cec5SDimitry Andric #endif 4450b57cec5SDimitry Andric 4465ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4475ffd83dbSDimitry Andric defined(__CLDEMOTE__) 4480b57cec5SDimitry Andric #include <cldemoteintrin.h> 4490b57cec5SDimitry Andric #endif 4500b57cec5SDimitry Andric 4515ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4525ffd83dbSDimitry Andric defined(__WAITPKG__) 4530b57cec5SDimitry Andric #include <waitpkgintrin.h> 4540b57cec5SDimitry Andric #endif 4550b57cec5SDimitry Andric 4565ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4570b57cec5SDimitry Andric defined(__MOVDIRI__) || defined(__MOVDIR64B__) 4580b57cec5SDimitry Andric #include <movdirintrin.h> 4590b57cec5SDimitry Andric #endif 4600b57cec5SDimitry Andric 4615ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4625ffd83dbSDimitry Andric defined(__PCONFIG__) 4630b57cec5SDimitry Andric #include <pconfigintrin.h> 4640b57cec5SDimitry Andric #endif 4650b57cec5SDimitry Andric 4665ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4675ffd83dbSDimitry Andric defined(__SGX__) 4680b57cec5SDimitry Andric #include <sgxintrin.h> 4690b57cec5SDimitry Andric #endif 4700b57cec5SDimitry Andric 4715ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4725ffd83dbSDimitry Andric defined(__PTWRITE__) 4730b57cec5SDimitry Andric #include <ptwriteintrin.h> 4740b57cec5SDimitry Andric #endif 4750b57cec5SDimitry Andric 4765ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4775ffd83dbSDimitry Andric defined(__INVPCID__) 4780b57cec5SDimitry Andric #include <invpcidintrin.h> 4790b57cec5SDimitry Andric #endif 4800b57cec5SDimitry Andric 4815ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 482e8d8bef9SDimitry Andric defined(__KL__) || defined(__WIDEKL__) 483e8d8bef9SDimitry Andric #include <keylockerintrin.h> 484e8d8bef9SDimitry Andric #endif 485e8d8bef9SDimitry Andric 486e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4875ffd83dbSDimitry Andric defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__) 4885ffd83dbSDimitry Andric #include <amxintrin.h> 4895ffd83dbSDimitry Andric #endif 4905ffd83dbSDimitry Andric 4915ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4920b57cec5SDimitry Andric defined(__AVX512VP2INTERSECT__) 4930b57cec5SDimitry Andric #include <avx512vp2intersectintrin.h> 4940b57cec5SDimitry Andric #endif 4950b57cec5SDimitry Andric 4965ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4970b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__)) 4980b57cec5SDimitry Andric #include <avx512vlvp2intersectintrin.h> 4990b57cec5SDimitry Andric #endif 5000b57cec5SDimitry Andric 5015ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5025ffd83dbSDimitry Andric defined(__ENQCMD__) 5030b57cec5SDimitry Andric #include <enqcmdintrin.h> 5040b57cec5SDimitry Andric #endif 5050b57cec5SDimitry Andric 5065ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5075ffd83dbSDimitry Andric defined(__SERIALIZE__) 5085ffd83dbSDimitry Andric #include <serializeintrin.h> 5095ffd83dbSDimitry Andric #endif 5105ffd83dbSDimitry Andric 5115ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5125ffd83dbSDimitry Andric defined(__TSXLDTRK__) 5135ffd83dbSDimitry Andric #include <tsxldtrkintrin.h> 5145ffd83dbSDimitry Andric #endif 5155ffd83dbSDimitry Andric 5160b57cec5SDimitry Andric #if defined(_MSC_VER) && __has_extension(gnu_asm) 5170b57cec5SDimitry Andric /* Define the default attributes for these intrinsics */ 5180b57cec5SDimitry Andric #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 5190b57cec5SDimitry Andric #ifdef __cplusplus 5200b57cec5SDimitry Andric extern "C" { 5210b57cec5SDimitry Andric #endif 5220b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 5230b57cec5SDimitry Andric |* Interlocked Exchange HLE 5240b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 5250b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 5260b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5270b57cec5SDimitry Andric _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) { 5280b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1" 5290b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5300b57cec5SDimitry Andric return _Value; 5310b57cec5SDimitry Andric } 5320b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5330b57cec5SDimitry Andric _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) { 5340b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1" 5350b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5360b57cec5SDimitry Andric return _Value; 5370b57cec5SDimitry Andric } 5380b57cec5SDimitry Andric #endif 5390b57cec5SDimitry Andric #if defined(__x86_64__) 5400b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5410b57cec5SDimitry Andric _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) { 5420b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg %0, %1" 5430b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5440b57cec5SDimitry Andric return _Value; 5450b57cec5SDimitry Andric } 5460b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5470b57cec5SDimitry Andric _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) { 5480b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg %0, %1" 5490b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5500b57cec5SDimitry Andric return _Value; 5510b57cec5SDimitry Andric } 5520b57cec5SDimitry Andric #endif 5530b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 5540b57cec5SDimitry Andric |* Interlocked Compare Exchange HLE 5550b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 5560b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 5570b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5580b57cec5SDimitry Andric _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination, 5590b57cec5SDimitry Andric long _Exchange, long _Comparand) { 5600b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1" 5610b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 5620b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 5630b57cec5SDimitry Andric return _Comparand; 5640b57cec5SDimitry Andric } 5650b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5660b57cec5SDimitry Andric _InterlockedCompareExchange_HLERelease(long volatile *_Destination, 5670b57cec5SDimitry Andric long _Exchange, long _Comparand) { 5680b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1" 5690b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 5700b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 5710b57cec5SDimitry Andric return _Comparand; 5720b57cec5SDimitry Andric } 5730b57cec5SDimitry Andric #endif 5740b57cec5SDimitry Andric #if defined(__x86_64__) 5750b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5760b57cec5SDimitry Andric _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination, 5770b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 5780b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg %2, %1" 5790b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 5800b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 5810b57cec5SDimitry Andric return _Comparand; 5820b57cec5SDimitry Andric } 5830b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5840b57cec5SDimitry Andric _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination, 5850b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 5860b57cec5SDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg %2, %1" 5870b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 5880b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 5890b57cec5SDimitry Andric return _Comparand; 5900b57cec5SDimitry Andric } 5910b57cec5SDimitry Andric #endif 5920b57cec5SDimitry Andric #ifdef __cplusplus 5930b57cec5SDimitry Andric } 5940b57cec5SDimitry Andric #endif 5950b57cec5SDimitry Andric 5960b57cec5SDimitry Andric #undef __DEFAULT_FN_ATTRS 5970b57cec5SDimitry Andric 5980b57cec5SDimitry Andric #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */ 5990b57cec5SDimitry Andric 6000b57cec5SDimitry Andric #endif /* __IMMINTRIN_H */ 601