10b57cec5SDimitry Andric /*===---- immintrin.h - Intel intrinsics -----------------------------------=== 20b57cec5SDimitry Andric * 30b57cec5SDimitry Andric * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric * See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric * 70b57cec5SDimitry Andric *===-----------------------------------------------------------------------=== 80b57cec5SDimitry Andric */ 90b57cec5SDimitry Andric 100b57cec5SDimitry Andric #ifndef __IMMINTRIN_H 110b57cec5SDimitry Andric #define __IMMINTRIN_H 120b57cec5SDimitry Andric 13349cc55cSDimitry Andric #if !defined(__i386__) && !defined(__x86_64__) 14349cc55cSDimitry Andric #error "This header is only meant to be used on x86 and x64 architecture" 15349cc55cSDimitry Andric #endif 16349cc55cSDimitry Andric 17e8d8bef9SDimitry Andric #include <x86gprintrin.h> 18e8d8bef9SDimitry Andric 195ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 205ffd83dbSDimitry Andric defined(__MMX__) 210b57cec5SDimitry Andric #include <mmintrin.h> 220b57cec5SDimitry Andric #endif 230b57cec5SDimitry Andric 245ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 255ffd83dbSDimitry Andric defined(__SSE__) 260b57cec5SDimitry Andric #include <xmmintrin.h> 270b57cec5SDimitry Andric #endif 280b57cec5SDimitry Andric 295ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 305ffd83dbSDimitry Andric defined(__SSE2__) 310b57cec5SDimitry Andric #include <emmintrin.h> 320b57cec5SDimitry Andric #endif 330b57cec5SDimitry Andric 345ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 355ffd83dbSDimitry Andric defined(__SSE3__) 360b57cec5SDimitry Andric #include <pmmintrin.h> 370b57cec5SDimitry Andric #endif 380b57cec5SDimitry Andric 395ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 405ffd83dbSDimitry Andric defined(__SSSE3__) 410b57cec5SDimitry Andric #include <tmmintrin.h> 420b57cec5SDimitry Andric #endif 430b57cec5SDimitry Andric 445ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 450b57cec5SDimitry Andric (defined(__SSE4_2__) || defined(__SSE4_1__)) 460b57cec5SDimitry Andric #include <smmintrin.h> 470b57cec5SDimitry Andric #endif 480b57cec5SDimitry Andric 495ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 500b57cec5SDimitry Andric (defined(__AES__) || defined(__PCLMUL__)) 510b57cec5SDimitry Andric #include <wmmintrin.h> 520b57cec5SDimitry Andric #endif 530b57cec5SDimitry Andric 545ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 555ffd83dbSDimitry Andric defined(__CLFLUSHOPT__) 560b57cec5SDimitry Andric #include <clflushoptintrin.h> 570b57cec5SDimitry Andric #endif 580b57cec5SDimitry Andric 595ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 605ffd83dbSDimitry Andric defined(__CLWB__) 610b57cec5SDimitry Andric #include <clwbintrin.h> 620b57cec5SDimitry Andric #endif 630b57cec5SDimitry Andric 645ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 655ffd83dbSDimitry Andric defined(__AVX__) 660b57cec5SDimitry Andric #include <avxintrin.h> 670b57cec5SDimitry Andric #endif 680b57cec5SDimitry Andric 695ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 705ffd83dbSDimitry Andric defined(__AVX2__) 710b57cec5SDimitry Andric #include <avx2intrin.h> 720b57cec5SDimitry Andric #endif 730b57cec5SDimitry Andric 745ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 755ffd83dbSDimitry Andric defined(__F16C__) 760b57cec5SDimitry Andric #include <f16cintrin.h> 770b57cec5SDimitry Andric #endif 780b57cec5SDimitry Andric 79a7dea167SDimitry Andric /* No feature check desired due to internal checks */ 800b57cec5SDimitry Andric #include <bmiintrin.h> 810b57cec5SDimitry Andric 825ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 835ffd83dbSDimitry Andric defined(__BMI2__) 840b57cec5SDimitry Andric #include <bmi2intrin.h> 850b57cec5SDimitry Andric #endif 860b57cec5SDimitry Andric 875ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 885ffd83dbSDimitry Andric defined(__LZCNT__) 890b57cec5SDimitry Andric #include <lzcntintrin.h> 900b57cec5SDimitry Andric #endif 910b57cec5SDimitry Andric 925ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 935ffd83dbSDimitry Andric defined(__POPCNT__) 940b57cec5SDimitry Andric #include <popcntintrin.h> 950b57cec5SDimitry Andric #endif 960b57cec5SDimitry Andric 975ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 985ffd83dbSDimitry Andric defined(__FMA__) 990b57cec5SDimitry Andric #include <fmaintrin.h> 1000b57cec5SDimitry Andric #endif 1010b57cec5SDimitry Andric 1025ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1035ffd83dbSDimitry Andric defined(__AVX512F__) 1040b57cec5SDimitry Andric #include <avx512fintrin.h> 1050b57cec5SDimitry Andric #endif 1060b57cec5SDimitry Andric 1075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1085ffd83dbSDimitry Andric defined(__AVX512VL__) 1090b57cec5SDimitry Andric #include <avx512vlintrin.h> 1100b57cec5SDimitry Andric #endif 1110b57cec5SDimitry Andric 1125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1135ffd83dbSDimitry Andric defined(__AVX512BW__) 1140b57cec5SDimitry Andric #include <avx512bwintrin.h> 1150b57cec5SDimitry Andric #endif 1160b57cec5SDimitry Andric 1175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1185ffd83dbSDimitry Andric defined(__AVX512BITALG__) 1190b57cec5SDimitry Andric #include <avx512bitalgintrin.h> 1200b57cec5SDimitry Andric #endif 1210b57cec5SDimitry Andric 1225ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1235ffd83dbSDimitry Andric defined(__AVX512CD__) 1240b57cec5SDimitry Andric #include <avx512cdintrin.h> 1250b57cec5SDimitry Andric #endif 1260b57cec5SDimitry Andric 1275ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1285ffd83dbSDimitry Andric defined(__AVX512VPOPCNTDQ__) 1290b57cec5SDimitry Andric #include <avx512vpopcntdqintrin.h> 1300b57cec5SDimitry Andric #endif 1310b57cec5SDimitry Andric 1325ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1330b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 1340b57cec5SDimitry Andric #include <avx512vpopcntdqvlintrin.h> 1350b57cec5SDimitry Andric #endif 1360b57cec5SDimitry Andric 1375ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1385ffd83dbSDimitry Andric defined(__AVX512VNNI__) 1390b57cec5SDimitry Andric #include <avx512vnniintrin.h> 1400b57cec5SDimitry Andric #endif 1410b57cec5SDimitry Andric 1425ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1430b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 1440b57cec5SDimitry Andric #include <avx512vlvnniintrin.h> 1450b57cec5SDimitry Andric #endif 1460b57cec5SDimitry Andric 1475ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 148e8d8bef9SDimitry Andric defined(__AVXVNNI__) 149e8d8bef9SDimitry Andric #include <avxvnniintrin.h> 150e8d8bef9SDimitry Andric #endif 151e8d8bef9SDimitry Andric 152e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1535ffd83dbSDimitry Andric defined(__AVX512DQ__) 1540b57cec5SDimitry Andric #include <avx512dqintrin.h> 1550b57cec5SDimitry Andric #endif 1560b57cec5SDimitry Andric 1575ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1580b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 1590b57cec5SDimitry Andric #include <avx512vlbitalgintrin.h> 1600b57cec5SDimitry Andric #endif 1610b57cec5SDimitry Andric 1625ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1630b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BW__)) 1640b57cec5SDimitry Andric #include <avx512vlbwintrin.h> 1650b57cec5SDimitry Andric #endif 1660b57cec5SDimitry Andric 1675ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1680b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512CD__)) 1690b57cec5SDimitry Andric #include <avx512vlcdintrin.h> 1700b57cec5SDimitry Andric #endif 1710b57cec5SDimitry Andric 1725ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1730b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512DQ__)) 1740b57cec5SDimitry Andric #include <avx512vldqintrin.h> 1750b57cec5SDimitry Andric #endif 1760b57cec5SDimitry Andric 1775ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1785ffd83dbSDimitry Andric defined(__AVX512ER__) 1790b57cec5SDimitry Andric #include <avx512erintrin.h> 1800b57cec5SDimitry Andric #endif 1810b57cec5SDimitry Andric 1825ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1835ffd83dbSDimitry Andric defined(__AVX512IFMA__) 1840b57cec5SDimitry Andric #include <avx512ifmaintrin.h> 1850b57cec5SDimitry Andric #endif 1860b57cec5SDimitry Andric 1875ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1880b57cec5SDimitry Andric (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 1890b57cec5SDimitry Andric #include <avx512ifmavlintrin.h> 1900b57cec5SDimitry Andric #endif 1910b57cec5SDimitry Andric 1925ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 193bdd1243dSDimitry Andric defined(__AVXIFMA__) 194bdd1243dSDimitry Andric #include <avxifmaintrin.h> 195bdd1243dSDimitry Andric #endif 196bdd1243dSDimitry Andric 197bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1985ffd83dbSDimitry Andric defined(__AVX512VBMI__) 1990b57cec5SDimitry Andric #include <avx512vbmiintrin.h> 2000b57cec5SDimitry Andric #endif 2010b57cec5SDimitry Andric 2025ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2030b57cec5SDimitry Andric (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 2040b57cec5SDimitry Andric #include <avx512vbmivlintrin.h> 2050b57cec5SDimitry Andric #endif 2060b57cec5SDimitry Andric 2075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2085ffd83dbSDimitry Andric defined(__AVX512VBMI2__) 2090b57cec5SDimitry Andric #include <avx512vbmi2intrin.h> 2100b57cec5SDimitry Andric #endif 2110b57cec5SDimitry Andric 2125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2130b57cec5SDimitry Andric (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 2140b57cec5SDimitry Andric #include <avx512vlvbmi2intrin.h> 2150b57cec5SDimitry Andric #endif 2160b57cec5SDimitry Andric 2175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2185ffd83dbSDimitry Andric defined(__AVX512PF__) 2190b57cec5SDimitry Andric #include <avx512pfintrin.h> 2200b57cec5SDimitry Andric #endif 2210b57cec5SDimitry Andric 222bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 223bdd1243dSDimitry Andric defined(__AVX512FP16__) 224349cc55cSDimitry Andric #include <avx512fp16intrin.h> 225349cc55cSDimitry Andric #endif 226349cc55cSDimitry Andric 227bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 228bdd1243dSDimitry Andric (defined(__AVX512VL__) && defined(__AVX512FP16__)) 229349cc55cSDimitry Andric #include <avx512vlfp16intrin.h> 230349cc55cSDimitry Andric #endif 231349cc55cSDimitry Andric 2325ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2335ffd83dbSDimitry Andric defined(__AVX512BF16__) 2340b57cec5SDimitry Andric #include <avx512bf16intrin.h> 2350b57cec5SDimitry Andric #endif 2360b57cec5SDimitry Andric 2375ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2380b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BF16__)) 2390b57cec5SDimitry Andric #include <avx512vlbf16intrin.h> 2400b57cec5SDimitry Andric #endif 2410b57cec5SDimitry Andric 2425ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2435ffd83dbSDimitry Andric defined(__PKU__) 2440b57cec5SDimitry Andric #include <pkuintrin.h> 2450b57cec5SDimitry Andric #endif 2460b57cec5SDimitry Andric 2475ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 248fe6060f1SDimitry Andric defined(__VPCLMULQDQ__) 249fe6060f1SDimitry Andric #include <vpclmulqdqintrin.h> 250fe6060f1SDimitry Andric #endif 251fe6060f1SDimitry Andric 252fe6060f1SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2535ffd83dbSDimitry Andric defined(__VAES__) 2540b57cec5SDimitry Andric #include <vaesintrin.h> 2550b57cec5SDimitry Andric #endif 2560b57cec5SDimitry Andric 2575ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2585ffd83dbSDimitry Andric defined(__GFNI__) 2590b57cec5SDimitry Andric #include <gfniintrin.h> 2600b57cec5SDimitry Andric #endif 2610b57cec5SDimitry Andric 2625ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 263bdd1243dSDimitry Andric defined(__AVXVNNIINT8__) 264bdd1243dSDimitry Andric #include <avxvnniint8intrin.h> 265bdd1243dSDimitry Andric #endif 266bdd1243dSDimitry Andric 267bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 268bdd1243dSDimitry Andric defined(__AVXNECONVERT__) 269bdd1243dSDimitry Andric #include <avxneconvertintrin.h> 270bdd1243dSDimitry Andric #endif 271bdd1243dSDimitry Andric 272bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2735ffd83dbSDimitry Andric defined(__RDPID__) 2740b57cec5SDimitry Andric /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103). 2750b57cec5SDimitry Andric /// 2760b57cec5SDimitry Andric /// \headerfile <immintrin.h> 2770b57cec5SDimitry Andric /// 2780b57cec5SDimitry Andric /// This intrinsic corresponds to the <c> RDPID </c> instruction. 2790b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) 2800b57cec5SDimitry Andric _rdpid_u32(void) { 2810b57cec5SDimitry Andric return __builtin_ia32_rdpid(); 2820b57cec5SDimitry Andric } 2830b57cec5SDimitry Andric #endif // __RDPID__ 2840b57cec5SDimitry Andric 2855ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2865ffd83dbSDimitry Andric defined(__RDRND__) 2870b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 2880b57cec5SDimitry Andric _rdrand16_step(unsigned short *__p) 2890b57cec5SDimitry Andric { 29081ad6265SDimitry Andric return (int)__builtin_ia32_rdrand16_step(__p); 2910b57cec5SDimitry Andric } 2920b57cec5SDimitry Andric 2930b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 2940b57cec5SDimitry Andric _rdrand32_step(unsigned int *__p) 2950b57cec5SDimitry Andric { 29681ad6265SDimitry Andric return (int)__builtin_ia32_rdrand32_step(__p); 2970b57cec5SDimitry Andric } 2980b57cec5SDimitry Andric 2990b57cec5SDimitry Andric #ifdef __x86_64__ 3000b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 3010b57cec5SDimitry Andric _rdrand64_step(unsigned long long *__p) 3020b57cec5SDimitry Andric { 30381ad6265SDimitry Andric return (int)__builtin_ia32_rdrand64_step(__p); 3040b57cec5SDimitry Andric } 305bdd1243dSDimitry Andric #else 306bdd1243dSDimitry Andric // We need to emulate the functionality of 64-bit rdrand with 2 32-bit 307bdd1243dSDimitry Andric // rdrand instructions. 308bdd1243dSDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 309bdd1243dSDimitry Andric _rdrand64_step(unsigned long long *__p) 310bdd1243dSDimitry Andric { 311bdd1243dSDimitry Andric unsigned int __lo, __hi; 312bdd1243dSDimitry Andric unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo); 313bdd1243dSDimitry Andric unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi); 314bdd1243dSDimitry Andric if (__res_lo && __res_hi) { 315bdd1243dSDimitry Andric *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo; 316bdd1243dSDimitry Andric return 1; 317bdd1243dSDimitry Andric } else { 318bdd1243dSDimitry Andric *__p = 0; 319bdd1243dSDimitry Andric return 0; 320bdd1243dSDimitry Andric } 321bdd1243dSDimitry Andric } 3220b57cec5SDimitry Andric #endif 3230b57cec5SDimitry Andric #endif /* __RDRND__ */ 3240b57cec5SDimitry Andric 3255ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3265ffd83dbSDimitry Andric defined(__FSGSBASE__) 3270b57cec5SDimitry Andric #ifdef __x86_64__ 3280b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3290b57cec5SDimitry Andric _readfsbase_u32(void) 3300b57cec5SDimitry Andric { 3310b57cec5SDimitry Andric return __builtin_ia32_rdfsbase32(); 3320b57cec5SDimitry Andric } 3330b57cec5SDimitry Andric 3340b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3350b57cec5SDimitry Andric _readfsbase_u64(void) 3360b57cec5SDimitry Andric { 3370b57cec5SDimitry Andric return __builtin_ia32_rdfsbase64(); 3380b57cec5SDimitry Andric } 3390b57cec5SDimitry Andric 3400b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3410b57cec5SDimitry Andric _readgsbase_u32(void) 3420b57cec5SDimitry Andric { 3430b57cec5SDimitry Andric return __builtin_ia32_rdgsbase32(); 3440b57cec5SDimitry Andric } 3450b57cec5SDimitry Andric 3460b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3470b57cec5SDimitry Andric _readgsbase_u64(void) 3480b57cec5SDimitry Andric { 3490b57cec5SDimitry Andric return __builtin_ia32_rdgsbase64(); 3500b57cec5SDimitry Andric } 3510b57cec5SDimitry Andric 3520b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3530b57cec5SDimitry Andric _writefsbase_u32(unsigned int __V) 3540b57cec5SDimitry Andric { 3550b57cec5SDimitry Andric __builtin_ia32_wrfsbase32(__V); 3560b57cec5SDimitry Andric } 3570b57cec5SDimitry Andric 3580b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3590b57cec5SDimitry Andric _writefsbase_u64(unsigned long long __V) 3600b57cec5SDimitry Andric { 3610b57cec5SDimitry Andric __builtin_ia32_wrfsbase64(__V); 3620b57cec5SDimitry Andric } 3630b57cec5SDimitry Andric 3640b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3650b57cec5SDimitry Andric _writegsbase_u32(unsigned int __V) 3660b57cec5SDimitry Andric { 3670b57cec5SDimitry Andric __builtin_ia32_wrgsbase32(__V); 3680b57cec5SDimitry Andric } 3690b57cec5SDimitry Andric 3700b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3710b57cec5SDimitry Andric _writegsbase_u64(unsigned long long __V) 3720b57cec5SDimitry Andric { 3730b57cec5SDimitry Andric __builtin_ia32_wrgsbase64(__V); 3740b57cec5SDimitry Andric } 3750b57cec5SDimitry Andric 3760b57cec5SDimitry Andric #endif 3770b57cec5SDimitry Andric #endif /* __FSGSBASE__ */ 3780b57cec5SDimitry Andric 3795ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3805ffd83dbSDimitry Andric defined(__MOVBE__) 3810b57cec5SDimitry Andric 3820b57cec5SDimitry Andric /* The structs used below are to force the load/store to be unaligned. This 3830b57cec5SDimitry Andric * is accomplished with the __packed__ attribute. The __may_alias__ prevents 3840b57cec5SDimitry Andric * tbaa metadata from being generated based on the struct and the type of the 3850b57cec5SDimitry Andric * field inside of it. 3860b57cec5SDimitry Andric */ 3870b57cec5SDimitry Andric 3880b57cec5SDimitry Andric static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3890b57cec5SDimitry Andric _loadbe_i16(void const * __P) { 3900b57cec5SDimitry Andric struct __loadu_i16 { 39181ad6265SDimitry Andric unsigned short __v; 3920b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 39381ad6265SDimitry Andric return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v); 3940b57cec5SDimitry Andric } 3950b57cec5SDimitry Andric 3960b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 3970b57cec5SDimitry Andric _storebe_i16(void * __P, short __D) { 3980b57cec5SDimitry Andric struct __storeu_i16 { 39981ad6265SDimitry Andric unsigned short __v; 4000b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 40181ad6265SDimitry Andric ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D); 4020b57cec5SDimitry Andric } 4030b57cec5SDimitry Andric 4040b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 4050b57cec5SDimitry Andric _loadbe_i32(void const * __P) { 4060b57cec5SDimitry Andric struct __loadu_i32 { 40781ad6265SDimitry Andric unsigned int __v; 4080b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 40981ad6265SDimitry Andric return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v); 4100b57cec5SDimitry Andric } 4110b57cec5SDimitry Andric 4120b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 4130b57cec5SDimitry Andric _storebe_i32(void * __P, int __D) { 4140b57cec5SDimitry Andric struct __storeu_i32 { 41581ad6265SDimitry Andric unsigned int __v; 4160b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 41781ad6265SDimitry Andric ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D); 4180b57cec5SDimitry Andric } 4190b57cec5SDimitry Andric 4200b57cec5SDimitry Andric #ifdef __x86_64__ 4210b57cec5SDimitry Andric static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 4220b57cec5SDimitry Andric _loadbe_i64(void const * __P) { 4230b57cec5SDimitry Andric struct __loadu_i64 { 42481ad6265SDimitry Andric unsigned long long __v; 4250b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 42681ad6265SDimitry Andric return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v); 4270b57cec5SDimitry Andric } 4280b57cec5SDimitry Andric 4290b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 4300b57cec5SDimitry Andric _storebe_i64(void * __P, long long __D) { 4310b57cec5SDimitry Andric struct __storeu_i64 { 43281ad6265SDimitry Andric unsigned long long __v; 4330b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 43481ad6265SDimitry Andric ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D); 4350b57cec5SDimitry Andric } 4360b57cec5SDimitry Andric #endif 4370b57cec5SDimitry Andric #endif /* __MOVBE */ 4380b57cec5SDimitry Andric 4395ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4405ffd83dbSDimitry Andric defined(__RTM__) 4410b57cec5SDimitry Andric #include <rtmintrin.h> 4420b57cec5SDimitry Andric #include <xtestintrin.h> 4430b57cec5SDimitry Andric #endif 4440b57cec5SDimitry Andric 4455ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4465ffd83dbSDimitry Andric defined(__SHA__) 4470b57cec5SDimitry Andric #include <shaintrin.h> 4480b57cec5SDimitry Andric #endif 4490b57cec5SDimitry Andric 4505ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4515ffd83dbSDimitry Andric defined(__FXSR__) 4520b57cec5SDimitry Andric #include <fxsrintrin.h> 4530b57cec5SDimitry Andric #endif 4540b57cec5SDimitry Andric 4550b57cec5SDimitry Andric /* No feature check desired due to internal MSC_VER checks */ 4560b57cec5SDimitry Andric #include <xsaveintrin.h> 4570b57cec5SDimitry Andric 4585ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4595ffd83dbSDimitry Andric defined(__XSAVEOPT__) 4600b57cec5SDimitry Andric #include <xsaveoptintrin.h> 4610b57cec5SDimitry Andric #endif 4620b57cec5SDimitry Andric 4635ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4645ffd83dbSDimitry Andric defined(__XSAVEC__) 4650b57cec5SDimitry Andric #include <xsavecintrin.h> 4660b57cec5SDimitry Andric #endif 4670b57cec5SDimitry Andric 4685ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4695ffd83dbSDimitry Andric defined(__XSAVES__) 4700b57cec5SDimitry Andric #include <xsavesintrin.h> 4710b57cec5SDimitry Andric #endif 4720b57cec5SDimitry Andric 4735ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4745ffd83dbSDimitry Andric defined(__SHSTK__) 4750b57cec5SDimitry Andric #include <cetintrin.h> 4760b57cec5SDimitry Andric #endif 4770b57cec5SDimitry Andric 4780b57cec5SDimitry Andric /* Some intrinsics inside adxintrin.h are available only on processors with ADX, 4790b57cec5SDimitry Andric * whereas others are also available at all times. */ 4800b57cec5SDimitry Andric #include <adxintrin.h> 4810b57cec5SDimitry Andric 4825ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4835ffd83dbSDimitry Andric defined(__RDSEED__) 4840b57cec5SDimitry Andric #include <rdseedintrin.h> 4850b57cec5SDimitry Andric #endif 4860b57cec5SDimitry Andric 4875ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4885ffd83dbSDimitry Andric defined(__WBNOINVD__) 4890b57cec5SDimitry Andric #include <wbnoinvdintrin.h> 4900b57cec5SDimitry Andric #endif 4910b57cec5SDimitry Andric 4925ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4935ffd83dbSDimitry Andric defined(__CLDEMOTE__) 4940b57cec5SDimitry Andric #include <cldemoteintrin.h> 4950b57cec5SDimitry Andric #endif 4960b57cec5SDimitry Andric 4975ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4985ffd83dbSDimitry Andric defined(__WAITPKG__) 4990b57cec5SDimitry Andric #include <waitpkgintrin.h> 5000b57cec5SDimitry Andric #endif 5010b57cec5SDimitry Andric 5025ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5030b57cec5SDimitry Andric defined(__MOVDIRI__) || defined(__MOVDIR64B__) 5040b57cec5SDimitry Andric #include <movdirintrin.h> 5050b57cec5SDimitry Andric #endif 5060b57cec5SDimitry Andric 5075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5085ffd83dbSDimitry Andric defined(__PCONFIG__) 5090b57cec5SDimitry Andric #include <pconfigintrin.h> 5100b57cec5SDimitry Andric #endif 5110b57cec5SDimitry Andric 5125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5135ffd83dbSDimitry Andric defined(__SGX__) 5140b57cec5SDimitry Andric #include <sgxintrin.h> 5150b57cec5SDimitry Andric #endif 5160b57cec5SDimitry Andric 5175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5185ffd83dbSDimitry Andric defined(__PTWRITE__) 5190b57cec5SDimitry Andric #include <ptwriteintrin.h> 5200b57cec5SDimitry Andric #endif 5210b57cec5SDimitry Andric 5225ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5235ffd83dbSDimitry Andric defined(__INVPCID__) 5240b57cec5SDimitry Andric #include <invpcidintrin.h> 5250b57cec5SDimitry Andric #endif 526bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5271ac55f4cSDimitry Andric defined(__AMX_FP16__) 528bdd1243dSDimitry Andric #include <amxfp16intrin.h> 529bdd1243dSDimitry Andric #endif 5300b57cec5SDimitry Andric 5315ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 532e8d8bef9SDimitry Andric defined(__KL__) || defined(__WIDEKL__) 533e8d8bef9SDimitry Andric #include <keylockerintrin.h> 534e8d8bef9SDimitry Andric #endif 535e8d8bef9SDimitry Andric 536e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5371ac55f4cSDimitry Andric defined(__AMX_TILE__) || defined(__AMX_INT8__) || defined(__AMX_BF16__) 5385ffd83dbSDimitry Andric #include <amxintrin.h> 5395ffd83dbSDimitry Andric #endif 5405ffd83dbSDimitry Andric 5415ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5420b57cec5SDimitry Andric defined(__AVX512VP2INTERSECT__) 5430b57cec5SDimitry Andric #include <avx512vp2intersectintrin.h> 5440b57cec5SDimitry Andric #endif 5450b57cec5SDimitry Andric 5465ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5470b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__)) 5480b57cec5SDimitry Andric #include <avx512vlvp2intersectintrin.h> 5490b57cec5SDimitry Andric #endif 5500b57cec5SDimitry Andric 5515ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5525ffd83dbSDimitry Andric defined(__ENQCMD__) 5530b57cec5SDimitry Andric #include <enqcmdintrin.h> 5540b57cec5SDimitry Andric #endif 5550b57cec5SDimitry Andric 5565ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5575ffd83dbSDimitry Andric defined(__SERIALIZE__) 5585ffd83dbSDimitry Andric #include <serializeintrin.h> 5595ffd83dbSDimitry Andric #endif 5605ffd83dbSDimitry Andric 5615ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5625ffd83dbSDimitry Andric defined(__TSXLDTRK__) 5635ffd83dbSDimitry Andric #include <tsxldtrkintrin.h> 5645ffd83dbSDimitry Andric #endif 5655ffd83dbSDimitry Andric 5660b57cec5SDimitry Andric #if defined(_MSC_VER) && __has_extension(gnu_asm) 5670b57cec5SDimitry Andric /* Define the default attributes for these intrinsics */ 5680b57cec5SDimitry Andric #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 5690b57cec5SDimitry Andric #ifdef __cplusplus 5700b57cec5SDimitry Andric extern "C" { 5710b57cec5SDimitry Andric #endif 5720b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 5730b57cec5SDimitry Andric |* Interlocked Exchange HLE 5740b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 5750b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 5760b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5770b57cec5SDimitry Andric _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) { 578349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 5790b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5800b57cec5SDimitry Andric return _Value; 5810b57cec5SDimitry Andric } 5820b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 5830b57cec5SDimitry Andric _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) { 584349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 5850b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5860b57cec5SDimitry Andric return _Value; 5870b57cec5SDimitry Andric } 5880b57cec5SDimitry Andric #endif 5890b57cec5SDimitry Andric #if defined(__x86_64__) 5900b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5910b57cec5SDimitry Andric _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) { 592349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 5930b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 5940b57cec5SDimitry Andric return _Value; 5950b57cec5SDimitry Andric } 5960b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 5970b57cec5SDimitry Andric _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) { 598349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 5990b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 6000b57cec5SDimitry Andric return _Value; 6010b57cec5SDimitry Andric } 6020b57cec5SDimitry Andric #endif 6030b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 6040b57cec5SDimitry Andric |* Interlocked Compare Exchange HLE 6050b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 6060b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 6070b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 6080b57cec5SDimitry Andric _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination, 6090b57cec5SDimitry Andric long _Exchange, long _Comparand) { 610349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 6110b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 6120b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 6130b57cec5SDimitry Andric return _Comparand; 6140b57cec5SDimitry Andric } 6150b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 6160b57cec5SDimitry Andric _InterlockedCompareExchange_HLERelease(long volatile *_Destination, 6170b57cec5SDimitry Andric long _Exchange, long _Comparand) { 618349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 6190b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 6200b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 6210b57cec5SDimitry Andric return _Comparand; 6220b57cec5SDimitry Andric } 6230b57cec5SDimitry Andric #endif 6240b57cec5SDimitry Andric #if defined(__x86_64__) 6250b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 6260b57cec5SDimitry Andric _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination, 6270b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 628349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 6290b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 6300b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 6310b57cec5SDimitry Andric return _Comparand; 6320b57cec5SDimitry Andric } 6330b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 6340b57cec5SDimitry Andric _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination, 6350b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 636349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 6370b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 6380b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 6390b57cec5SDimitry Andric return _Comparand; 6400b57cec5SDimitry Andric } 6410b57cec5SDimitry Andric #endif 6420b57cec5SDimitry Andric #ifdef __cplusplus 6430b57cec5SDimitry Andric } 6440b57cec5SDimitry Andric #endif 6450b57cec5SDimitry Andric 6460b57cec5SDimitry Andric #undef __DEFAULT_FN_ATTRS 6470b57cec5SDimitry Andric 6480b57cec5SDimitry Andric #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */ 6490b57cec5SDimitry Andric 6500b57cec5SDimitry Andric #endif /* __IMMINTRIN_H */ 651