10b57cec5SDimitry Andric /*===---- immintrin.h - Intel intrinsics -----------------------------------=== 20b57cec5SDimitry Andric * 30b57cec5SDimitry Andric * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric * See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric * 70b57cec5SDimitry Andric *===-----------------------------------------------------------------------=== 80b57cec5SDimitry Andric */ 90b57cec5SDimitry Andric 100b57cec5SDimitry Andric #ifndef __IMMINTRIN_H 110b57cec5SDimitry Andric #define __IMMINTRIN_H 120b57cec5SDimitry Andric 13349cc55cSDimitry Andric #if !defined(__i386__) && !defined(__x86_64__) 14349cc55cSDimitry Andric #error "This header is only meant to be used on x86 and x64 architecture" 15349cc55cSDimitry Andric #endif 16349cc55cSDimitry Andric 17e8d8bef9SDimitry Andric #include <x86gprintrin.h> 18e8d8bef9SDimitry Andric 195ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 205ffd83dbSDimitry Andric defined(__MMX__) 210b57cec5SDimitry Andric #include <mmintrin.h> 220b57cec5SDimitry Andric #endif 230b57cec5SDimitry Andric 245ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 255ffd83dbSDimitry Andric defined(__SSE__) 260b57cec5SDimitry Andric #include <xmmintrin.h> 270b57cec5SDimitry Andric #endif 280b57cec5SDimitry Andric 295ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 305ffd83dbSDimitry Andric defined(__SSE2__) 310b57cec5SDimitry Andric #include <emmintrin.h> 320b57cec5SDimitry Andric #endif 330b57cec5SDimitry Andric 345ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 355ffd83dbSDimitry Andric defined(__SSE3__) 360b57cec5SDimitry Andric #include <pmmintrin.h> 370b57cec5SDimitry Andric #endif 380b57cec5SDimitry Andric 395ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 405ffd83dbSDimitry Andric defined(__SSSE3__) 410b57cec5SDimitry Andric #include <tmmintrin.h> 420b57cec5SDimitry Andric #endif 430b57cec5SDimitry Andric 445ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 450b57cec5SDimitry Andric (defined(__SSE4_2__) || defined(__SSE4_1__)) 460b57cec5SDimitry Andric #include <smmintrin.h> 470b57cec5SDimitry Andric #endif 480b57cec5SDimitry Andric 495ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 500b57cec5SDimitry Andric (defined(__AES__) || defined(__PCLMUL__)) 510b57cec5SDimitry Andric #include <wmmintrin.h> 520b57cec5SDimitry Andric #endif 530b57cec5SDimitry Andric 545ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 555ffd83dbSDimitry Andric defined(__CLFLUSHOPT__) 560b57cec5SDimitry Andric #include <clflushoptintrin.h> 570b57cec5SDimitry Andric #endif 580b57cec5SDimitry Andric 595ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 605ffd83dbSDimitry Andric defined(__CLWB__) 610b57cec5SDimitry Andric #include <clwbintrin.h> 620b57cec5SDimitry Andric #endif 630b57cec5SDimitry Andric 645ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 655ffd83dbSDimitry Andric defined(__AVX__) 660b57cec5SDimitry Andric #include <avxintrin.h> 670b57cec5SDimitry Andric #endif 680b57cec5SDimitry Andric 695ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 705ffd83dbSDimitry Andric defined(__AVX2__) 710b57cec5SDimitry Andric #include <avx2intrin.h> 720b57cec5SDimitry Andric #endif 730b57cec5SDimitry Andric 745ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 755ffd83dbSDimitry Andric defined(__F16C__) 760b57cec5SDimitry Andric #include <f16cintrin.h> 770b57cec5SDimitry Andric #endif 780b57cec5SDimitry Andric 79a7dea167SDimitry Andric /* No feature check desired due to internal checks */ 800b57cec5SDimitry Andric #include <bmiintrin.h> 810b57cec5SDimitry Andric 825ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 835ffd83dbSDimitry Andric defined(__BMI2__) 840b57cec5SDimitry Andric #include <bmi2intrin.h> 850b57cec5SDimitry Andric #endif 860b57cec5SDimitry Andric 875ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 885ffd83dbSDimitry Andric defined(__LZCNT__) 890b57cec5SDimitry Andric #include <lzcntintrin.h> 900b57cec5SDimitry Andric #endif 910b57cec5SDimitry Andric 925ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 935ffd83dbSDimitry Andric defined(__POPCNT__) 940b57cec5SDimitry Andric #include <popcntintrin.h> 950b57cec5SDimitry Andric #endif 960b57cec5SDimitry Andric 975ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 985ffd83dbSDimitry Andric defined(__FMA__) 990b57cec5SDimitry Andric #include <fmaintrin.h> 1000b57cec5SDimitry Andric #endif 1010b57cec5SDimitry Andric 1025ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1035ffd83dbSDimitry Andric defined(__AVX512F__) 1040b57cec5SDimitry Andric #include <avx512fintrin.h> 1050b57cec5SDimitry Andric #endif 1060b57cec5SDimitry Andric 1075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1085ffd83dbSDimitry Andric defined(__AVX512VL__) 1090b57cec5SDimitry Andric #include <avx512vlintrin.h> 1100b57cec5SDimitry Andric #endif 1110b57cec5SDimitry Andric 1125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1135ffd83dbSDimitry Andric defined(__AVX512BW__) 1140b57cec5SDimitry Andric #include <avx512bwintrin.h> 1150b57cec5SDimitry Andric #endif 1160b57cec5SDimitry Andric 1175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1185ffd83dbSDimitry Andric defined(__AVX512BITALG__) 1190b57cec5SDimitry Andric #include <avx512bitalgintrin.h> 1200b57cec5SDimitry Andric #endif 1210b57cec5SDimitry Andric 1225ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1235ffd83dbSDimitry Andric defined(__AVX512CD__) 1240b57cec5SDimitry Andric #include <avx512cdintrin.h> 1250b57cec5SDimitry Andric #endif 1260b57cec5SDimitry Andric 1275ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1285ffd83dbSDimitry Andric defined(__AVX512VPOPCNTDQ__) 1290b57cec5SDimitry Andric #include <avx512vpopcntdqintrin.h> 1300b57cec5SDimitry Andric #endif 1310b57cec5SDimitry Andric 1325ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1330b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__)) 1340b57cec5SDimitry Andric #include <avx512vpopcntdqvlintrin.h> 1350b57cec5SDimitry Andric #endif 1360b57cec5SDimitry Andric 1375ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1385ffd83dbSDimitry Andric defined(__AVX512VNNI__) 1390b57cec5SDimitry Andric #include <avx512vnniintrin.h> 1400b57cec5SDimitry Andric #endif 1410b57cec5SDimitry Andric 1425ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1430b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VNNI__)) 1440b57cec5SDimitry Andric #include <avx512vlvnniintrin.h> 1450b57cec5SDimitry Andric #endif 1460b57cec5SDimitry Andric 1475ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 148e8d8bef9SDimitry Andric defined(__AVXVNNI__) 149e8d8bef9SDimitry Andric #include <avxvnniintrin.h> 150e8d8bef9SDimitry Andric #endif 151e8d8bef9SDimitry Andric 152e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1535ffd83dbSDimitry Andric defined(__AVX512DQ__) 1540b57cec5SDimitry Andric #include <avx512dqintrin.h> 1550b57cec5SDimitry Andric #endif 1560b57cec5SDimitry Andric 1575ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1580b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BITALG__)) 1590b57cec5SDimitry Andric #include <avx512vlbitalgintrin.h> 1600b57cec5SDimitry Andric #endif 1610b57cec5SDimitry Andric 1625ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1630b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BW__)) 1640b57cec5SDimitry Andric #include <avx512vlbwintrin.h> 1650b57cec5SDimitry Andric #endif 1660b57cec5SDimitry Andric 1675ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1680b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512CD__)) 1690b57cec5SDimitry Andric #include <avx512vlcdintrin.h> 1700b57cec5SDimitry Andric #endif 1710b57cec5SDimitry Andric 1725ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1730b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512DQ__)) 1740b57cec5SDimitry Andric #include <avx512vldqintrin.h> 1750b57cec5SDimitry Andric #endif 1760b57cec5SDimitry Andric 1775ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1785ffd83dbSDimitry Andric defined(__AVX512ER__) 1790b57cec5SDimitry Andric #include <avx512erintrin.h> 1800b57cec5SDimitry Andric #endif 1810b57cec5SDimitry Andric 1825ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1835ffd83dbSDimitry Andric defined(__AVX512IFMA__) 1840b57cec5SDimitry Andric #include <avx512ifmaintrin.h> 1850b57cec5SDimitry Andric #endif 1860b57cec5SDimitry Andric 1875ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1880b57cec5SDimitry Andric (defined(__AVX512IFMA__) && defined(__AVX512VL__)) 1890b57cec5SDimitry Andric #include <avx512ifmavlintrin.h> 1900b57cec5SDimitry Andric #endif 1910b57cec5SDimitry Andric 1925ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 193bdd1243dSDimitry Andric defined(__AVXIFMA__) 194bdd1243dSDimitry Andric #include <avxifmaintrin.h> 195bdd1243dSDimitry Andric #endif 196bdd1243dSDimitry Andric 197bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 1985ffd83dbSDimitry Andric defined(__AVX512VBMI__) 1990b57cec5SDimitry Andric #include <avx512vbmiintrin.h> 2000b57cec5SDimitry Andric #endif 2010b57cec5SDimitry Andric 2025ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2030b57cec5SDimitry Andric (defined(__AVX512VBMI__) && defined(__AVX512VL__)) 2040b57cec5SDimitry Andric #include <avx512vbmivlintrin.h> 2050b57cec5SDimitry Andric #endif 2060b57cec5SDimitry Andric 2075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2085ffd83dbSDimitry Andric defined(__AVX512VBMI2__) 2090b57cec5SDimitry Andric #include <avx512vbmi2intrin.h> 2100b57cec5SDimitry Andric #endif 2110b57cec5SDimitry Andric 2125ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2130b57cec5SDimitry Andric (defined(__AVX512VBMI2__) && defined(__AVX512VL__)) 2140b57cec5SDimitry Andric #include <avx512vlvbmi2intrin.h> 2150b57cec5SDimitry Andric #endif 2160b57cec5SDimitry Andric 2175ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2185ffd83dbSDimitry Andric defined(__AVX512PF__) 2190b57cec5SDimitry Andric #include <avx512pfintrin.h> 2200b57cec5SDimitry Andric #endif 2210b57cec5SDimitry Andric 222bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 223bdd1243dSDimitry Andric defined(__AVX512FP16__) 224349cc55cSDimitry Andric #include <avx512fp16intrin.h> 225349cc55cSDimitry Andric #endif 226349cc55cSDimitry Andric 227bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 228bdd1243dSDimitry Andric (defined(__AVX512VL__) && defined(__AVX512FP16__)) 229349cc55cSDimitry Andric #include <avx512vlfp16intrin.h> 230349cc55cSDimitry Andric #endif 231349cc55cSDimitry Andric 2325ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2335ffd83dbSDimitry Andric defined(__AVX512BF16__) 2340b57cec5SDimitry Andric #include <avx512bf16intrin.h> 2350b57cec5SDimitry Andric #endif 2360b57cec5SDimitry Andric 2375ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2380b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512BF16__)) 2390b57cec5SDimitry Andric #include <avx512vlbf16intrin.h> 2400b57cec5SDimitry Andric #endif 2410b57cec5SDimitry Andric 2425ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2435ffd83dbSDimitry Andric defined(__PKU__) 2440b57cec5SDimitry Andric #include <pkuintrin.h> 2450b57cec5SDimitry Andric #endif 2460b57cec5SDimitry Andric 2475ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 248fe6060f1SDimitry Andric defined(__VPCLMULQDQ__) 249fe6060f1SDimitry Andric #include <vpclmulqdqintrin.h> 250fe6060f1SDimitry Andric #endif 251fe6060f1SDimitry Andric 252fe6060f1SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2535ffd83dbSDimitry Andric defined(__VAES__) 2540b57cec5SDimitry Andric #include <vaesintrin.h> 2550b57cec5SDimitry Andric #endif 2560b57cec5SDimitry Andric 2575ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2585ffd83dbSDimitry Andric defined(__GFNI__) 2590b57cec5SDimitry Andric #include <gfniintrin.h> 2600b57cec5SDimitry Andric #endif 2610b57cec5SDimitry Andric 2625ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 263bdd1243dSDimitry Andric defined(__AVXVNNIINT8__) 264bdd1243dSDimitry Andric #include <avxvnniint8intrin.h> 265bdd1243dSDimitry Andric #endif 266bdd1243dSDimitry Andric 267bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 268bdd1243dSDimitry Andric defined(__AVXNECONVERT__) 269bdd1243dSDimitry Andric #include <avxneconvertintrin.h> 270bdd1243dSDimitry Andric #endif 271bdd1243dSDimitry Andric 272bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 27306c3fb27SDimitry Andric defined(__SHA512__) 27406c3fb27SDimitry Andric #include <sha512intrin.h> 27506c3fb27SDimitry Andric #endif 27606c3fb27SDimitry Andric 27706c3fb27SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 27806c3fb27SDimitry Andric defined(__SM3__) 27906c3fb27SDimitry Andric #include <sm3intrin.h> 28006c3fb27SDimitry Andric #endif 28106c3fb27SDimitry Andric 28206c3fb27SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 28306c3fb27SDimitry Andric defined(__SM4__) 28406c3fb27SDimitry Andric #include <sm4intrin.h> 28506c3fb27SDimitry Andric #endif 28606c3fb27SDimitry Andric 28706c3fb27SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 28806c3fb27SDimitry Andric defined(__AVXVNNIINT16__) 28906c3fb27SDimitry Andric #include <avxvnniint16intrin.h> 29006c3fb27SDimitry Andric #endif 29106c3fb27SDimitry Andric 29206c3fb27SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 2935ffd83dbSDimitry Andric defined(__RDPID__) 2945f757f3fSDimitry Andric /// Reads the value of the IA32_TSC_AUX MSR (0xc0000103). 2950b57cec5SDimitry Andric /// 2960b57cec5SDimitry Andric /// \headerfile <immintrin.h> 2970b57cec5SDimitry Andric /// 2980b57cec5SDimitry Andric /// This intrinsic corresponds to the <c> RDPID </c> instruction. 2995f757f3fSDimitry Andric /// 3005f757f3fSDimitry Andric /// \returns The 32-bit contents of the MSR. 3010b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid"))) 3020b57cec5SDimitry Andric _rdpid_u32(void) { 3030b57cec5SDimitry Andric return __builtin_ia32_rdpid(); 3040b57cec5SDimitry Andric } 3050b57cec5SDimitry Andric #endif // __RDPID__ 3060b57cec5SDimitry Andric 3075ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3085ffd83dbSDimitry Andric defined(__RDRND__) 30906c3fb27SDimitry Andric /// Returns a 16-bit hardware-generated random value. 31006c3fb27SDimitry Andric /// 31106c3fb27SDimitry Andric /// \headerfile <immintrin.h> 31206c3fb27SDimitry Andric /// 31306c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDRAND </c> instruction. 31406c3fb27SDimitry Andric /// 31506c3fb27SDimitry Andric /// \param __p 31606c3fb27SDimitry Andric /// A pointer to a 16-bit memory location to place the random value. 31706c3fb27SDimitry Andric /// \returns 1 if the value was successfully generated, 0 otherwise. 3180b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 3190b57cec5SDimitry Andric _rdrand16_step(unsigned short *__p) 3200b57cec5SDimitry Andric { 32181ad6265SDimitry Andric return (int)__builtin_ia32_rdrand16_step(__p); 3220b57cec5SDimitry Andric } 3230b57cec5SDimitry Andric 32406c3fb27SDimitry Andric /// Returns a 32-bit hardware-generated random value. 32506c3fb27SDimitry Andric /// 32606c3fb27SDimitry Andric /// \headerfile <immintrin.h> 32706c3fb27SDimitry Andric /// 32806c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDRAND </c> instruction. 32906c3fb27SDimitry Andric /// 33006c3fb27SDimitry Andric /// \param __p 33106c3fb27SDimitry Andric /// A pointer to a 32-bit memory location to place the random value. 33206c3fb27SDimitry Andric /// \returns 1 if the value was successfully generated, 0 otherwise. 3330b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 3340b57cec5SDimitry Andric _rdrand32_step(unsigned int *__p) 3350b57cec5SDimitry Andric { 33681ad6265SDimitry Andric return (int)__builtin_ia32_rdrand32_step(__p); 3370b57cec5SDimitry Andric } 3380b57cec5SDimitry Andric 33906c3fb27SDimitry Andric /// Returns a 64-bit hardware-generated random value. 34006c3fb27SDimitry Andric /// 34106c3fb27SDimitry Andric /// \headerfile <immintrin.h> 34206c3fb27SDimitry Andric /// 34306c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDRAND </c> instruction. 34406c3fb27SDimitry Andric /// 34506c3fb27SDimitry Andric /// \param __p 34606c3fb27SDimitry Andric /// A pointer to a 64-bit memory location to place the random value. 34706c3fb27SDimitry Andric /// \returns 1 if the value was successfully generated, 0 otherwise. 3480b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd"))) 3490b57cec5SDimitry Andric _rdrand64_step(unsigned long long *__p) 3500b57cec5SDimitry Andric { 35106c3fb27SDimitry Andric #ifdef __x86_64__ 35281ad6265SDimitry Andric return (int)__builtin_ia32_rdrand64_step(__p); 353bdd1243dSDimitry Andric #else 354bdd1243dSDimitry Andric // We need to emulate the functionality of 64-bit rdrand with 2 32-bit 355bdd1243dSDimitry Andric // rdrand instructions. 356bdd1243dSDimitry Andric unsigned int __lo, __hi; 357bdd1243dSDimitry Andric unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo); 358bdd1243dSDimitry Andric unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi); 359bdd1243dSDimitry Andric if (__res_lo && __res_hi) { 360bdd1243dSDimitry Andric *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo; 361bdd1243dSDimitry Andric return 1; 362bdd1243dSDimitry Andric } else { 363bdd1243dSDimitry Andric *__p = 0; 364bdd1243dSDimitry Andric return 0; 365bdd1243dSDimitry Andric } 3660b57cec5SDimitry Andric #endif 36706c3fb27SDimitry Andric } 3680b57cec5SDimitry Andric #endif /* __RDRND__ */ 3690b57cec5SDimitry Andric 3705ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 3715ffd83dbSDimitry Andric defined(__FSGSBASE__) 3720b57cec5SDimitry Andric #ifdef __x86_64__ 37306c3fb27SDimitry Andric /// Reads the FS base register. 37406c3fb27SDimitry Andric /// 37506c3fb27SDimitry Andric /// \headerfile <immintrin.h> 37606c3fb27SDimitry Andric /// 37706c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction. 37806c3fb27SDimitry Andric /// 37906c3fb27SDimitry Andric /// \returns The lower 32 bits of the FS base register. 3800b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3810b57cec5SDimitry Andric _readfsbase_u32(void) 3820b57cec5SDimitry Andric { 3830b57cec5SDimitry Andric return __builtin_ia32_rdfsbase32(); 3840b57cec5SDimitry Andric } 3850b57cec5SDimitry Andric 38606c3fb27SDimitry Andric /// Reads the FS base register. 38706c3fb27SDimitry Andric /// 38806c3fb27SDimitry Andric /// \headerfile <immintrin.h> 38906c3fb27SDimitry Andric /// 39006c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction. 39106c3fb27SDimitry Andric /// 39206c3fb27SDimitry Andric /// \returns The contents of the FS base register. 3930b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 3940b57cec5SDimitry Andric _readfsbase_u64(void) 3950b57cec5SDimitry Andric { 3960b57cec5SDimitry Andric return __builtin_ia32_rdfsbase64(); 3970b57cec5SDimitry Andric } 3980b57cec5SDimitry Andric 39906c3fb27SDimitry Andric /// Reads the GS base register. 40006c3fb27SDimitry Andric /// 40106c3fb27SDimitry Andric /// \headerfile <immintrin.h> 40206c3fb27SDimitry Andric /// 40306c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction. 40406c3fb27SDimitry Andric /// 40506c3fb27SDimitry Andric /// \returns The lower 32 bits of the GS base register. 4060b57cec5SDimitry Andric static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4070b57cec5SDimitry Andric _readgsbase_u32(void) 4080b57cec5SDimitry Andric { 4090b57cec5SDimitry Andric return __builtin_ia32_rdgsbase32(); 4100b57cec5SDimitry Andric } 4110b57cec5SDimitry Andric 41206c3fb27SDimitry Andric /// Reads the GS base register. 41306c3fb27SDimitry Andric /// 41406c3fb27SDimitry Andric /// \headerfile <immintrin.h> 41506c3fb27SDimitry Andric /// 41606c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction. 41706c3fb27SDimitry Andric /// 41806c3fb27SDimitry Andric /// \returns The contents of the GS base register. 4190b57cec5SDimitry Andric static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4200b57cec5SDimitry Andric _readgsbase_u64(void) 4210b57cec5SDimitry Andric { 4220b57cec5SDimitry Andric return __builtin_ia32_rdgsbase64(); 4230b57cec5SDimitry Andric } 4240b57cec5SDimitry Andric 42506c3fb27SDimitry Andric /// Modifies the FS base register. 42606c3fb27SDimitry Andric /// 42706c3fb27SDimitry Andric /// \headerfile <immintrin.h> 42806c3fb27SDimitry Andric /// 42906c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. 43006c3fb27SDimitry Andric /// 43106c3fb27SDimitry Andric /// \param __V 43206c3fb27SDimitry Andric /// Value to use for the lower 32 bits of the FS base register. 4330b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4340b57cec5SDimitry Andric _writefsbase_u32(unsigned int __V) 4350b57cec5SDimitry Andric { 4360b57cec5SDimitry Andric __builtin_ia32_wrfsbase32(__V); 4370b57cec5SDimitry Andric } 4380b57cec5SDimitry Andric 43906c3fb27SDimitry Andric /// Modifies the FS base register. 44006c3fb27SDimitry Andric /// 44106c3fb27SDimitry Andric /// \headerfile <immintrin.h> 44206c3fb27SDimitry Andric /// 44306c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. 44406c3fb27SDimitry Andric /// 44506c3fb27SDimitry Andric /// \param __V 44606c3fb27SDimitry Andric /// Value to use for the FS base register. 4470b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4480b57cec5SDimitry Andric _writefsbase_u64(unsigned long long __V) 4490b57cec5SDimitry Andric { 4500b57cec5SDimitry Andric __builtin_ia32_wrfsbase64(__V); 4510b57cec5SDimitry Andric } 4520b57cec5SDimitry Andric 45306c3fb27SDimitry Andric /// Modifies the GS base register. 45406c3fb27SDimitry Andric /// 45506c3fb27SDimitry Andric /// \headerfile <immintrin.h> 45606c3fb27SDimitry Andric /// 45706c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> WRGSBASE </c> instruction. 45806c3fb27SDimitry Andric /// 45906c3fb27SDimitry Andric /// \param __V 46006c3fb27SDimitry Andric /// Value to use for the lower 32 bits of the GS base register. 4610b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4620b57cec5SDimitry Andric _writegsbase_u32(unsigned int __V) 4630b57cec5SDimitry Andric { 4640b57cec5SDimitry Andric __builtin_ia32_wrgsbase32(__V); 4650b57cec5SDimitry Andric } 4660b57cec5SDimitry Andric 46706c3fb27SDimitry Andric /// Modifies the GS base register. 46806c3fb27SDimitry Andric /// 46906c3fb27SDimitry Andric /// \headerfile <immintrin.h> 47006c3fb27SDimitry Andric /// 47106c3fb27SDimitry Andric /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction. 47206c3fb27SDimitry Andric /// 47306c3fb27SDimitry Andric /// \param __V 47406c3fb27SDimitry Andric /// Value to use for GS base register. 4750b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase"))) 4760b57cec5SDimitry Andric _writegsbase_u64(unsigned long long __V) 4770b57cec5SDimitry Andric { 4780b57cec5SDimitry Andric __builtin_ia32_wrgsbase64(__V); 4790b57cec5SDimitry Andric } 4800b57cec5SDimitry Andric 4810b57cec5SDimitry Andric #endif 4820b57cec5SDimitry Andric #endif /* __FSGSBASE__ */ 4830b57cec5SDimitry Andric 4845ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 4855ffd83dbSDimitry Andric defined(__MOVBE__) 4860b57cec5SDimitry Andric 4870b57cec5SDimitry Andric /* The structs used below are to force the load/store to be unaligned. This 4880b57cec5SDimitry Andric * is accomplished with the __packed__ attribute. The __may_alias__ prevents 4890b57cec5SDimitry Andric * tbaa metadata from being generated based on the struct and the type of the 4900b57cec5SDimitry Andric * field inside of it. 4910b57cec5SDimitry Andric */ 4920b57cec5SDimitry Andric 4930b57cec5SDimitry Andric static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 4940b57cec5SDimitry Andric _loadbe_i16(void const * __P) { 4950b57cec5SDimitry Andric struct __loadu_i16 { 49681ad6265SDimitry Andric unsigned short __v; 4970b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 49881ad6265SDimitry Andric return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v); 4990b57cec5SDimitry Andric } 5000b57cec5SDimitry Andric 5010b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 5020b57cec5SDimitry Andric _storebe_i16(void * __P, short __D) { 5030b57cec5SDimitry Andric struct __storeu_i16 { 50481ad6265SDimitry Andric unsigned short __v; 5050b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 50681ad6265SDimitry Andric ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D); 5070b57cec5SDimitry Andric } 5080b57cec5SDimitry Andric 5090b57cec5SDimitry Andric static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 5100b57cec5SDimitry Andric _loadbe_i32(void const * __P) { 5110b57cec5SDimitry Andric struct __loadu_i32 { 51281ad6265SDimitry Andric unsigned int __v; 5130b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 51481ad6265SDimitry Andric return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v); 5150b57cec5SDimitry Andric } 5160b57cec5SDimitry Andric 5170b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 5180b57cec5SDimitry Andric _storebe_i32(void * __P, int __D) { 5190b57cec5SDimitry Andric struct __storeu_i32 { 52081ad6265SDimitry Andric unsigned int __v; 5210b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 52281ad6265SDimitry Andric ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D); 5230b57cec5SDimitry Andric } 5240b57cec5SDimitry Andric 5250b57cec5SDimitry Andric #ifdef __x86_64__ 5260b57cec5SDimitry Andric static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 5270b57cec5SDimitry Andric _loadbe_i64(void const * __P) { 5280b57cec5SDimitry Andric struct __loadu_i64 { 52981ad6265SDimitry Andric unsigned long long __v; 5300b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 53181ad6265SDimitry Andric return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v); 5320b57cec5SDimitry Andric } 5330b57cec5SDimitry Andric 5340b57cec5SDimitry Andric static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe"))) 5350b57cec5SDimitry Andric _storebe_i64(void * __P, long long __D) { 5360b57cec5SDimitry Andric struct __storeu_i64 { 53781ad6265SDimitry Andric unsigned long long __v; 5380b57cec5SDimitry Andric } __attribute__((__packed__, __may_alias__)); 53981ad6265SDimitry Andric ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D); 5400b57cec5SDimitry Andric } 5410b57cec5SDimitry Andric #endif 5420b57cec5SDimitry Andric #endif /* __MOVBE */ 5430b57cec5SDimitry Andric 5445ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5455ffd83dbSDimitry Andric defined(__RTM__) 5460b57cec5SDimitry Andric #include <rtmintrin.h> 5470b57cec5SDimitry Andric #include <xtestintrin.h> 5480b57cec5SDimitry Andric #endif 5490b57cec5SDimitry Andric 5505ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5515ffd83dbSDimitry Andric defined(__SHA__) 5520b57cec5SDimitry Andric #include <shaintrin.h> 5530b57cec5SDimitry Andric #endif 5540b57cec5SDimitry Andric 5555ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5565ffd83dbSDimitry Andric defined(__FXSR__) 5570b57cec5SDimitry Andric #include <fxsrintrin.h> 5580b57cec5SDimitry Andric #endif 5590b57cec5SDimitry Andric 5600b57cec5SDimitry Andric /* No feature check desired due to internal MSC_VER checks */ 5610b57cec5SDimitry Andric #include <xsaveintrin.h> 5620b57cec5SDimitry Andric 5635ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5645ffd83dbSDimitry Andric defined(__XSAVEOPT__) 5650b57cec5SDimitry Andric #include <xsaveoptintrin.h> 5660b57cec5SDimitry Andric #endif 5670b57cec5SDimitry Andric 5685ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5695ffd83dbSDimitry Andric defined(__XSAVEC__) 5700b57cec5SDimitry Andric #include <xsavecintrin.h> 5710b57cec5SDimitry Andric #endif 5720b57cec5SDimitry Andric 5735ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5745ffd83dbSDimitry Andric defined(__XSAVES__) 5750b57cec5SDimitry Andric #include <xsavesintrin.h> 5760b57cec5SDimitry Andric #endif 5770b57cec5SDimitry Andric 5785ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5795ffd83dbSDimitry Andric defined(__SHSTK__) 5800b57cec5SDimitry Andric #include <cetintrin.h> 5810b57cec5SDimitry Andric #endif 5820b57cec5SDimitry Andric 583cb14a3feSDimitry Andric /* Intrinsics inside adcintrin.h are available at all times. */ 584cb14a3feSDimitry Andric #include <adcintrin.h> 585cb14a3feSDimitry Andric 586cb14a3feSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 587cb14a3feSDimitry Andric defined(__ADX__) 5880b57cec5SDimitry Andric #include <adxintrin.h> 589cb14a3feSDimitry Andric #endif 5900b57cec5SDimitry Andric 5915ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5925ffd83dbSDimitry Andric defined(__RDSEED__) 5930b57cec5SDimitry Andric #include <rdseedintrin.h> 5940b57cec5SDimitry Andric #endif 5950b57cec5SDimitry Andric 5965ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 5975ffd83dbSDimitry Andric defined(__WBNOINVD__) 5980b57cec5SDimitry Andric #include <wbnoinvdintrin.h> 5990b57cec5SDimitry Andric #endif 6000b57cec5SDimitry Andric 6015ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6025ffd83dbSDimitry Andric defined(__CLDEMOTE__) 6030b57cec5SDimitry Andric #include <cldemoteintrin.h> 6040b57cec5SDimitry Andric #endif 6050b57cec5SDimitry Andric 6065ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6075ffd83dbSDimitry Andric defined(__WAITPKG__) 6080b57cec5SDimitry Andric #include <waitpkgintrin.h> 6090b57cec5SDimitry Andric #endif 6100b57cec5SDimitry Andric 6115ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6120b57cec5SDimitry Andric defined(__MOVDIRI__) || defined(__MOVDIR64B__) 6130b57cec5SDimitry Andric #include <movdirintrin.h> 6140b57cec5SDimitry Andric #endif 6150b57cec5SDimitry Andric 6165ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6175ffd83dbSDimitry Andric defined(__PCONFIG__) 6180b57cec5SDimitry Andric #include <pconfigintrin.h> 6190b57cec5SDimitry Andric #endif 6200b57cec5SDimitry Andric 6215ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6225ffd83dbSDimitry Andric defined(__SGX__) 6230b57cec5SDimitry Andric #include <sgxintrin.h> 6240b57cec5SDimitry Andric #endif 6250b57cec5SDimitry Andric 6265ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6275ffd83dbSDimitry Andric defined(__PTWRITE__) 6280b57cec5SDimitry Andric #include <ptwriteintrin.h> 6290b57cec5SDimitry Andric #endif 6300b57cec5SDimitry Andric 6315ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6325ffd83dbSDimitry Andric defined(__INVPCID__) 6330b57cec5SDimitry Andric #include <invpcidintrin.h> 6340b57cec5SDimitry Andric #endif 635bdd1243dSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6361ac55f4cSDimitry Andric defined(__AMX_FP16__) 637bdd1243dSDimitry Andric #include <amxfp16intrin.h> 638bdd1243dSDimitry Andric #endif 6390b57cec5SDimitry Andric 6405ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 641e8d8bef9SDimitry Andric defined(__KL__) || defined(__WIDEKL__) 642e8d8bef9SDimitry Andric #include <keylockerintrin.h> 643e8d8bef9SDimitry Andric #endif 644e8d8bef9SDimitry Andric 645e8d8bef9SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6461ac55f4cSDimitry Andric defined(__AMX_TILE__) || defined(__AMX_INT8__) || defined(__AMX_BF16__) 6475ffd83dbSDimitry Andric #include <amxintrin.h> 6485ffd83dbSDimitry Andric #endif 6495ffd83dbSDimitry Andric 6505ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 65106c3fb27SDimitry Andric defined(__AMX_COMPLEX__) 65206c3fb27SDimitry Andric #include <amxcomplexintrin.h> 65306c3fb27SDimitry Andric #endif 65406c3fb27SDimitry Andric 65506c3fb27SDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6560b57cec5SDimitry Andric defined(__AVX512VP2INTERSECT__) 6570b57cec5SDimitry Andric #include <avx512vp2intersectintrin.h> 6580b57cec5SDimitry Andric #endif 6590b57cec5SDimitry Andric 6605ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6610b57cec5SDimitry Andric (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__)) 6620b57cec5SDimitry Andric #include <avx512vlvp2intersectintrin.h> 6630b57cec5SDimitry Andric #endif 6640b57cec5SDimitry Andric 6655ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6665ffd83dbSDimitry Andric defined(__ENQCMD__) 6670b57cec5SDimitry Andric #include <enqcmdintrin.h> 6680b57cec5SDimitry Andric #endif 6690b57cec5SDimitry Andric 6705ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6715ffd83dbSDimitry Andric defined(__SERIALIZE__) 6725ffd83dbSDimitry Andric #include <serializeintrin.h> 6735ffd83dbSDimitry Andric #endif 6745ffd83dbSDimitry Andric 6755ffd83dbSDimitry Andric #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) || \ 6765ffd83dbSDimitry Andric defined(__TSXLDTRK__) 6775ffd83dbSDimitry Andric #include <tsxldtrkintrin.h> 6785ffd83dbSDimitry Andric #endif 6795ffd83dbSDimitry Andric 6800b57cec5SDimitry Andric #if defined(_MSC_VER) && __has_extension(gnu_asm) 6810b57cec5SDimitry Andric /* Define the default attributes for these intrinsics */ 6820b57cec5SDimitry Andric #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__)) 6830b57cec5SDimitry Andric #ifdef __cplusplus 6840b57cec5SDimitry Andric extern "C" { 6850b57cec5SDimitry Andric #endif 6860b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 6870b57cec5SDimitry Andric |* Interlocked Exchange HLE 6880b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 6890b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 6900b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 6910b57cec5SDimitry Andric _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) { 692349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 6930b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 6940b57cec5SDimitry Andric return _Value; 6950b57cec5SDimitry Andric } 6960b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 6970b57cec5SDimitry Andric _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) { 698349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 6990b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 7000b57cec5SDimitry Andric return _Value; 7010b57cec5SDimitry Andric } 7020b57cec5SDimitry Andric #endif 7030b57cec5SDimitry Andric #if defined(__x86_64__) 7040b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 7050b57cec5SDimitry Andric _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) { 706349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}" 7070b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 7080b57cec5SDimitry Andric return _Value; 7090b57cec5SDimitry Andric } 7100b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 7110b57cec5SDimitry Andric _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) { 712349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}" 7130b57cec5SDimitry Andric : "+r" (_Value), "+m" (*_Target) :: "memory"); 7140b57cec5SDimitry Andric return _Value; 7150b57cec5SDimitry Andric } 7160b57cec5SDimitry Andric #endif 7170b57cec5SDimitry Andric /*----------------------------------------------------------------------------*\ 7180b57cec5SDimitry Andric |* Interlocked Compare Exchange HLE 7190b57cec5SDimitry Andric \*----------------------------------------------------------------------------*/ 7200b57cec5SDimitry Andric #if defined(__i386__) || defined(__x86_64__) 7210b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 7220b57cec5SDimitry Andric _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination, 7230b57cec5SDimitry Andric long _Exchange, long _Comparand) { 724349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 7250b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 7260b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 7270b57cec5SDimitry Andric return _Comparand; 7280b57cec5SDimitry Andric } 7290b57cec5SDimitry Andric static __inline__ long __DEFAULT_FN_ATTRS 7300b57cec5SDimitry Andric _InterlockedCompareExchange_HLERelease(long volatile *_Destination, 7310b57cec5SDimitry Andric long _Exchange, long _Comparand) { 732349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 7330b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 7340b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 7350b57cec5SDimitry Andric return _Comparand; 7360b57cec5SDimitry Andric } 7370b57cec5SDimitry Andric #endif 7380b57cec5SDimitry Andric #if defined(__x86_64__) 7390b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 7400b57cec5SDimitry Andric _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination, 7410b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 742349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}" 7430b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 7440b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 7450b57cec5SDimitry Andric return _Comparand; 7460b57cec5SDimitry Andric } 7470b57cec5SDimitry Andric static __inline__ __int64 __DEFAULT_FN_ATTRS 7480b57cec5SDimitry Andric _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination, 7490b57cec5SDimitry Andric __int64 _Exchange, __int64 _Comparand) { 750349cc55cSDimitry Andric __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}" 7510b57cec5SDimitry Andric : "+a" (_Comparand), "+m" (*_Destination) 7520b57cec5SDimitry Andric : "r" (_Exchange) : "memory"); 7530b57cec5SDimitry Andric return _Comparand; 7540b57cec5SDimitry Andric } 7550b57cec5SDimitry Andric #endif 7560b57cec5SDimitry Andric #ifdef __cplusplus 7570b57cec5SDimitry Andric } 7580b57cec5SDimitry Andric #endif 7590b57cec5SDimitry Andric 7600b57cec5SDimitry Andric #undef __DEFAULT_FN_ATTRS 7610b57cec5SDimitry Andric 7620b57cec5SDimitry Andric #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */ 7630b57cec5SDimitry Andric 7640b57cec5SDimitry Andric #endif /* __IMMINTRIN_H */ 765