1 #include <stddef.h> 2 #include <stdint.h> 3 #ifdef HAVE_ANDROID_GETCPUFEATURES 4 # include <cpu-features.h> 5 #endif 6 7 #include "private/common.h" 8 #include "runtime.h" 9 10 typedef struct CPUFeatures_ { 11 int initialized; 12 int has_neon; 13 int has_sse2; 14 int has_sse3; 15 int has_ssse3; 16 int has_sse41; 17 int has_avx; 18 int has_avx2; 19 int has_avx512f; 20 int has_pclmul; 21 int has_aesni; 22 int has_rdrand; 23 } CPUFeatures; 24 25 static CPUFeatures _cpu_features; 26 27 #define CPUID_EBX_AVX2 0x00000020 28 #define CPUID_EBX_AVX512F 0x00010000 29 30 #define CPUID_ECX_SSE3 0x00000001 31 #define CPUID_ECX_PCLMUL 0x00000002 32 #define CPUID_ECX_SSSE3 0x00000200 33 #define CPUID_ECX_SSE41 0x00080000 34 #define CPUID_ECX_AESNI 0x02000000 35 #define CPUID_ECX_XSAVE 0x04000000 36 #define CPUID_ECX_OSXSAVE 0x08000000 37 #define CPUID_ECX_AVX 0x10000000 38 #define CPUID_ECX_RDRAND 0x40000000 39 40 #define CPUID_EDX_SSE2 0x04000000 41 42 #define XCR0_SSE 0x00000002 43 #define XCR0_AVX 0x00000004 44 45 static int 46 _sodium_runtime_arm_cpu_features(CPUFeatures * const cpu_features) 47 { 48 #ifndef __arm__ 49 cpu_features->has_neon = 0; 50 return -1; 51 #else 52 # ifdef __APPLE__ 53 # ifdef __ARM_NEON__ 54 cpu_features->has_neon = 1; 55 # else 56 cpu_features->has_neon = 0; 57 # endif 58 # elif defined(HAVE_ANDROID_GETCPUFEATURES) && \ 59 defined(ANDROID_CPU_ARM_FEATURE_NEON) 60 cpu_features->has_neon = 61 (android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0x0; 62 # else 63 cpu_features->has_neon = 0; 64 # endif 65 return 0; 66 #endif 67 } 68 69 static void 70 _cpuid(unsigned int cpu_info[4U], const unsigned int cpu_info_type) 71 { 72 #if defined(_MSC_VER) && \ 73 (defined(_M_X64) || defined(_M_AMD64) || defined(_M_IX86)) 74 __cpuid((int *) cpu_info, cpu_info_type); 75 #elif defined(HAVE_CPUID) 76 cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; 77 # ifdef __i386__ 78 __asm__ __volatile__( 79 "pushfl; pushfl; " 80 "popl %0; " 81 "movl %0, %1; xorl %2, %0; " 82 "pushl %0; " 83 "popfl; pushfl; popl %0; popfl" 84 : "=&r"(cpu_info[0]), "=&r"(cpu_info[1]) 85 : "i"(0x200000)); 86 if (((cpu_info[0] ^ cpu_info[1]) & 0x200000) == 0x0) { 87 return; /* LCOV_EXCL_LINE */ 88 } 89 # endif 90 # ifdef __i386__ 91 __asm__ __volatile__("xchgl %%ebx, %k1; cpuid; xchgl %%ebx, %k1" 92 : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), 93 "=c"(cpu_info[2]), "=d"(cpu_info[3]) 94 : "0"(cpu_info_type), "2"(0U)); 95 # elif defined(__x86_64__) 96 __asm__ __volatile__("xchgq %%rbx, %q1; cpuid; xchgq %%rbx, %q1" 97 : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), 98 "=c"(cpu_info[2]), "=d"(cpu_info[3]) 99 : "0"(cpu_info_type), "2"(0U)); 100 # else 101 __asm__ __volatile__("cpuid" 102 : "=a"(cpu_info[0]), "=b"(cpu_info[1]), 103 "=c"(cpu_info[2]), "=d"(cpu_info[3]) 104 : "0"(cpu_info_type), "2"(0U)); 105 # endif 106 #else 107 (void) cpu_info_type; 108 cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; 109 #endif 110 } 111 112 static int 113 _sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features) 114 { 115 unsigned int cpu_info[4]; 116 unsigned int id; 117 118 _cpuid(cpu_info, 0x0); 119 if ((id = cpu_info[0]) == 0U) { 120 return -1; /* LCOV_EXCL_LINE */ 121 } 122 _cpuid(cpu_info, 0x00000001); 123 #ifdef HAVE_EMMINTRIN_H 124 cpu_features->has_sse2 = ((cpu_info[3] & CPUID_EDX_SSE2) != 0x0); 125 #else 126 cpu_features->has_sse2 = 0; 127 #endif 128 129 #ifdef HAVE_PMMINTRIN_H 130 cpu_features->has_sse3 = ((cpu_info[2] & CPUID_ECX_SSE3) != 0x0); 131 #else 132 cpu_features->has_sse3 = 0; 133 #endif 134 135 #ifdef HAVE_TMMINTRIN_H 136 cpu_features->has_ssse3 = ((cpu_info[2] & CPUID_ECX_SSSE3) != 0x0); 137 #else 138 cpu_features->has_ssse3 = 0; 139 #endif 140 141 #ifdef HAVE_SMMINTRIN_H 142 cpu_features->has_sse41 = ((cpu_info[2] & CPUID_ECX_SSE41) != 0x0); 143 #else 144 cpu_features->has_sse41 = 0; 145 #endif 146 147 cpu_features->has_avx = 0; 148 #ifdef HAVE_AVXINTRIN_H 149 if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == 150 (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { 151 uint32_t xcr0 = 0U; 152 # if defined(HAVE__XGETBV) || \ 153 (defined(_MSC_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) && _MSC_FULL_VER >= 160040219) 154 xcr0 = (uint32_t) _xgetbv(0); 155 # elif defined(_MSC_VER) && defined(_M_IX86) 156 /* 157 * Visual Studio documentation states that eax/ecx/edx don't need to 158 * be preserved in inline assembly code. But that doesn't seem to 159 * always hold true on Visual Studio 2010. 160 */ 161 __asm { 162 push eax 163 push ecx 164 push edx 165 xor ecx, ecx 166 _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 167 mov xcr0, eax 168 pop edx 169 pop ecx 170 pop eax 171 } 172 # elif defined(HAVE_AVX_ASM) 173 __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" /* XGETBV */ 174 : "=a"(xcr0) 175 : "c"((uint32_t) 0U) 176 : "%edx"); 177 # endif 178 if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { 179 cpu_features->has_avx = 1; 180 } 181 } 182 #endif 183 184 cpu_features->has_avx2 = 0; 185 #ifdef HAVE_AVX2INTRIN_H 186 if (cpu_features->has_avx) { 187 unsigned int cpu_info7[4]; 188 189 _cpuid(cpu_info7, 0x00000007); 190 cpu_features->has_avx2 = ((cpu_info7[1] & CPUID_EBX_AVX2) != 0x0); 191 } 192 #endif 193 194 cpu_features->has_avx512f = 0; 195 #ifdef HAVE_AVX512FINTRIN_H 196 if (cpu_features->has_avx2) { 197 unsigned int cpu_info7[4]; 198 199 _cpuid(cpu_info7, 0x00000007); 200 cpu_features->has_avx512f = ((cpu_info7[1] & CPUID_EBX_AVX512F) != 0x0); 201 } 202 #endif 203 204 #ifdef HAVE_WMMINTRIN_H 205 cpu_features->has_pclmul = ((cpu_info[2] & CPUID_ECX_PCLMUL) != 0x0); 206 cpu_features->has_aesni = ((cpu_info[2] & CPUID_ECX_AESNI) != 0x0); 207 #else 208 cpu_features->has_pclmul = 0; 209 cpu_features->has_aesni = 0; 210 #endif 211 212 #ifdef HAVE_RDRAND 213 cpu_features->has_rdrand = ((cpu_info[2] & CPUID_ECX_RDRAND) != 0x0); 214 #else 215 cpu_features->has_rdrand = 0; 216 #endif 217 218 return 0; 219 } 220 221 int 222 _sodium_runtime_get_cpu_features(void) 223 { 224 int ret = -1; 225 226 ret &= _sodium_runtime_arm_cpu_features(&_cpu_features); 227 ret &= _sodium_runtime_intel_cpu_features(&_cpu_features); 228 _cpu_features.initialized = 1; 229 230 return ret; 231 } 232 233 int 234 sodium_runtime_has_neon(void) 235 { 236 return _cpu_features.has_neon; 237 } 238 239 int 240 sodium_runtime_has_sse2(void) 241 { 242 return _cpu_features.has_sse2; 243 } 244 245 int 246 sodium_runtime_has_sse3(void) 247 { 248 return _cpu_features.has_sse3; 249 } 250 251 int 252 sodium_runtime_has_ssse3(void) 253 { 254 return _cpu_features.has_ssse3; 255 } 256 257 int 258 sodium_runtime_has_sse41(void) 259 { 260 return _cpu_features.has_sse41; 261 } 262 263 int 264 sodium_runtime_has_avx(void) 265 { 266 return _cpu_features.has_avx; 267 } 268 269 int 270 sodium_runtime_has_avx2(void) 271 { 272 return _cpu_features.has_avx2; 273 } 274 275 int 276 sodium_runtime_has_avx512f(void) 277 { 278 return _cpu_features.has_avx512f; 279 } 280 281 int 282 sodium_runtime_has_pclmul(void) 283 { 284 return _cpu_features.has_pclmul; 285 } 286 287 int 288 sodium_runtime_has_aesni(void) 289 { 290 return _cpu_features.has_aesni; 291 } 292 293 int 294 sodium_runtime_has_rdrand(void) 295 { 296 return _cpu_features.has_rdrand; 297 } 298