1 /*
2 * Copyright 2011 The LibYuv Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "libyuv/cpu_id.h"
12
13 #if defined(_MSC_VER)
14 #include <intrin.h> // For __cpuidex()
15 #endif
16 #if !defined(__pnacl__) && !defined(__CLR_VER) && \
17 !defined(__native_client__) && (defined(_M_IX86) || defined(_M_X64)) && \
18 defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
19 #include <immintrin.h> // For _xgetbv()
20 #endif
21
22 #if !defined(__native_client__)
23 #include <stdlib.h> // For getenv()
24 #endif
25
26 // For ArmCpuCaps() but unittested on all platforms
27 #include <stdio.h>
28 #include <string.h>
29
30 #include "libyuv/basic_types.h" // For CPU_X86
31
32 #ifdef __cplusplus
33 namespace libyuv {
34 extern "C" {
35 #endif
36
37 // For functions that use the stack and have runtime checks for overflow,
38 // use SAFEBUFFERS to avoid additional check.
39 #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219) && \
40 !defined(__clang__)
41 #define SAFEBUFFERS __declspec(safebuffers)
42 #else
43 #define SAFEBUFFERS
44 #endif
45
46 // cpu_info_ variable for SIMD instruction sets detected.
47 LIBYUV_API int cpu_info_ = 0;
48
49 // TODO(fbarchard): Consider using int for cpuid so casting is not needed.
50 // Low level cpuid for X86.
51 #if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
52 defined(__x86_64__)) && \
53 !defined(__pnacl__) && !defined(__CLR_VER)
54 LIBYUV_API
CpuId(int info_eax,int info_ecx,int * cpu_info)55 void CpuId(int info_eax, int info_ecx, int* cpu_info) {
56 #if defined(_MSC_VER)
57 // Visual C version uses intrinsic or inline x86 assembly.
58 #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
59 __cpuidex(cpu_info, info_eax, info_ecx);
60 #elif defined(_M_IX86)
61 __asm {
62 mov eax, info_eax
63 mov ecx, info_ecx
64 mov edi, cpu_info
65 cpuid
66 mov [edi], eax
67 mov [edi + 4], ebx
68 mov [edi + 8], ecx
69 mov [edi + 12], edx
70 }
71 #else // Visual C but not x86
72 if (info_ecx == 0) {
73 __cpuid(cpu_info, info_eax);
74 } else {
75 cpu_info[3] = cpu_info[2] = cpu_info[1] = cpu_info[0] = 0u;
76 }
77 #endif
78 // GCC version uses inline x86 assembly.
79 #else // defined(_MSC_VER)
80 int info_ebx, info_edx;
81 asm volatile(
82 #if defined(__i386__) && defined(__PIC__)
83 // Preserve ebx for fpic 32 bit.
84 "mov %%ebx, %%edi \n"
85 "cpuid \n"
86 "xchg %%edi, %%ebx \n"
87 : "=D"(info_ebx),
88 #else
89 "cpuid \n"
90 : "=b"(info_ebx),
91 #endif // defined( __i386__) && defined(__PIC__)
92 "+a"(info_eax), "+c"(info_ecx), "=d"(info_edx));
93 cpu_info[0] = info_eax;
94 cpu_info[1] = info_ebx;
95 cpu_info[2] = info_ecx;
96 cpu_info[3] = info_edx;
97 #endif // defined(_MSC_VER)
98 }
99 #else // (defined(_M_IX86) || defined(_M_X64) ...
100 LIBYUV_API
CpuId(int eax,int ecx,int * cpu_info)101 void CpuId(int eax, int ecx, int* cpu_info) {
102 (void)eax;
103 (void)ecx;
104 cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0;
105 }
106 #endif
107
108 // For VS2010 and earlier emit can be used:
109 // _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 // For VS2010 and earlier.
110 // __asm {
111 // xor ecx, ecx // xcr 0
112 // xgetbv
113 // mov xcr0, eax
114 // }
115 // For VS2013 and earlier 32 bit, the _xgetbv(0) optimizer produces bad code.
116 // https://code.google.com/p/libyuv/issues/detail?id=529
117 #if defined(_M_IX86) && (_MSC_VER < 1900)
118 #pragma optimize("g", off)
119 #endif
120 #if (defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || \
121 defined(__x86_64__)) && \
122 !defined(__pnacl__) && !defined(__CLR_VER) && !defined(__native_client__)
123 // X86 CPUs have xgetbv to detect OS saves high parts of ymm registers.
GetXCR0()124 int GetXCR0() {
125 int xcr0 = 0;
126 #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219)
127 xcr0 = (int)_xgetbv(0); // VS2010 SP1 required. NOLINT
128 #elif defined(__i386__) || defined(__x86_64__)
129 asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx");
130 #endif // defined(__i386__) || defined(__x86_64__)
131 return xcr0;
132 }
133 #else
134 // xgetbv unavailable to query for OSSave support. Return 0.
135 #define GetXCR0() 0
136 #endif // defined(_M_IX86) || defined(_M_X64) ..
137 // Return optimization to previous setting.
138 #if defined(_M_IX86) && (_MSC_VER < 1900)
139 #pragma optimize("g", on)
140 #endif
141
142 // based on libvpx arm_cpudetect.c
143 // For Arm, but public to allow testing on any CPU
ArmCpuCaps(const char * cpuinfo_name)144 LIBYUV_API SAFEBUFFERS int ArmCpuCaps(const char* cpuinfo_name) {
145 char cpuinfo_line[512];
146 FILE* f = fopen(cpuinfo_name, "r");
147 if (!f) {
148 // Assume Neon if /proc/cpuinfo is unavailable.
149 // This will occur for Chrome sandbox for Pepper or Render process.
150 return kCpuHasNEON;
151 }
152 while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
153 if (memcmp(cpuinfo_line, "Features", 8) == 0) {
154 char* p = strstr(cpuinfo_line, " neon");
155 if (p && (p[5] == ' ' || p[5] == '\n')) {
156 fclose(f);
157 return kCpuHasNEON;
158 }
159 // aarch64 uses asimd for Neon.
160 p = strstr(cpuinfo_line, " asimd");
161 if (p) {
162 fclose(f);
163 return kCpuHasNEON;
164 }
165 }
166 }
167 fclose(f);
168 return 0;
169 }
170
171 // TODO(fbarchard): Consider read_msa_ir().
172 // TODO(fbarchard): Add unittest.
MipsCpuCaps(const char * cpuinfo_name,const char ase[])173 LIBYUV_API SAFEBUFFERS int MipsCpuCaps(const char* cpuinfo_name,
174 const char ase[]) {
175 char cpuinfo_line[512];
176 FILE* f = fopen(cpuinfo_name, "r");
177 if (!f) {
178 // ase enabled if /proc/cpuinfo is unavailable.
179 if (strcmp(ase, " msa") == 0) {
180 return kCpuHasMSA;
181 }
182 return kCpuHasDSPR2;
183 }
184 while (fgets(cpuinfo_line, sizeof(cpuinfo_line) - 1, f)) {
185 if (memcmp(cpuinfo_line, "ASEs implemented", 16) == 0) {
186 char* p = strstr(cpuinfo_line, ase);
187 if (p) {
188 fclose(f);
189 if (strcmp(ase, " msa") == 0) {
190 return kCpuHasMSA;
191 }
192 return kCpuHasDSPR2;
193 }
194 }
195 }
196 fclose(f);
197 return 0;
198 }
199
200 // Test environment variable for disabling CPU features. Any non-zero value
201 // to disable. Zero ignored to make it easy to set the variable on/off.
202 #if !defined(__native_client__) && !defined(_M_ARM)
203
TestEnv(const char * name)204 static LIBYUV_BOOL TestEnv(const char* name) {
205 const char* var = getenv(name);
206 if (var) {
207 if (var[0] != '0') {
208 return LIBYUV_TRUE;
209 }
210 }
211 return LIBYUV_FALSE;
212 }
213 #else // nacl does not support getenv().
TestEnv(const char *)214 static LIBYUV_BOOL TestEnv(const char*) {
215 return LIBYUV_FALSE;
216 }
217 #endif
218
GetCpuFlags(void)219 static SAFEBUFFERS int GetCpuFlags(void) {
220 int cpu_info = 0;
221 #if !defined(__pnacl__) && !defined(__CLR_VER) && defined(CPU_X86)
222 int cpu_info0[4] = {0, 0, 0, 0};
223 int cpu_info1[4] = {0, 0, 0, 0};
224 int cpu_info7[4] = {0, 0, 0, 0};
225 CpuId(0, 0, cpu_info0);
226 CpuId(1, 0, cpu_info1);
227 if (cpu_info0[0] >= 7) {
228 CpuId(7, 0, cpu_info7);
229 }
230 cpu_info = kCpuHasX86 | ((cpu_info1[3] & 0x04000000) ? kCpuHasSSE2 : 0) |
231 ((cpu_info1[2] & 0x00000200) ? kCpuHasSSSE3 : 0) |
232 ((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) |
233 ((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) |
234 ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0);
235
236 // AVX requires OS saves YMM registers.
237 if (((cpu_info1[2] & 0x1c000000) == 0x1c000000) && // AVX and OSXSave
238 ((GetXCR0() & 6) == 6)) { // Test OS saves YMM registers
239 cpu_info |= kCpuHasAVX | ((cpu_info7[1] & 0x00000020) ? kCpuHasAVX2 : 0) |
240 ((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) |
241 ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0);
242
243 // Detect AVX512bw
244 if ((GetXCR0() & 0xe0) == 0xe0) {
245 cpu_info |= (cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0;
246 cpu_info |= (cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0;
247 cpu_info |= (cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0;
248 cpu_info |= (cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0;
249 cpu_info |= (cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0;
250 cpu_info |= (cpu_info7[2] & 0x00004000) ? kCpuHasAVX512VPOPCNTDQ : 0;
251 cpu_info |= (cpu_info7[2] & 0x00000100) ? kCpuHasGFNI : 0;
252 }
253 }
254
255 // TODO(fbarchard): Consider moving these to gtest
256 // Environment variable overrides for testing.
257 if (TestEnv("LIBYUV_DISABLE_X86")) {
258 cpu_info &= ~kCpuHasX86;
259 }
260 if (TestEnv("LIBYUV_DISABLE_SSE2")) {
261 cpu_info &= ~kCpuHasSSE2;
262 }
263 if (TestEnv("LIBYUV_DISABLE_SSSE3")) {
264 cpu_info &= ~kCpuHasSSSE3;
265 }
266 if (TestEnv("LIBYUV_DISABLE_SSE41")) {
267 cpu_info &= ~kCpuHasSSE41;
268 }
269 if (TestEnv("LIBYUV_DISABLE_SSE42")) {
270 cpu_info &= ~kCpuHasSSE42;
271 }
272 if (TestEnv("LIBYUV_DISABLE_AVX")) {
273 cpu_info &= ~kCpuHasAVX;
274 }
275 if (TestEnv("LIBYUV_DISABLE_AVX2")) {
276 cpu_info &= ~kCpuHasAVX2;
277 }
278 if (TestEnv("LIBYUV_DISABLE_ERMS")) {
279 cpu_info &= ~kCpuHasERMS;
280 }
281 if (TestEnv("LIBYUV_DISABLE_FMA3")) {
282 cpu_info &= ~kCpuHasFMA3;
283 }
284 if (TestEnv("LIBYUV_DISABLE_F16C")) {
285 cpu_info &= ~kCpuHasF16C;
286 }
287 if (TestEnv("LIBYUV_DISABLE_AVX512BW")) {
288 cpu_info &= ~kCpuHasAVX512BW;
289 }
290
291 #endif
292 #if defined(__mips__) && defined(__linux__)
293 #if defined(__mips_dspr2)
294 cpu_info |= kCpuHasDSPR2;
295 #endif
296 #if defined(__mips_msa)
297 cpu_info = MipsCpuCaps("/proc/cpuinfo", " msa");
298 #endif
299 cpu_info |= kCpuHasMIPS;
300 if (getenv("LIBYUV_DISABLE_DSPR2")) {
301 cpu_info &= ~kCpuHasDSPR2;
302 }
303 if (getenv("LIBYUV_DISABLE_MSA")) {
304 cpu_info &= ~kCpuHasMSA;
305 }
306 #endif
307 #if defined(__arm__) || defined(__aarch64__)
308 // gcc -mfpu=neon defines __ARM_NEON__
309 // __ARM_NEON__ generates code that requires Neon. NaCL also requires Neon.
310 // For Linux, /proc/cpuinfo can be tested but without that assume Neon.
311 #if defined(__ARM_NEON__) || defined(__native_client__) || !defined(__linux__)
312 cpu_info = kCpuHasNEON;
313 // For aarch64(arm64), /proc/cpuinfo's feature is not complete, e.g. no neon
314 // flag in it.
315 // So for aarch64, neon enabling is hard coded here.
316 #endif
317 #if defined(__aarch64__)
318 cpu_info = kCpuHasNEON;
319 #else
320 // Linux arm parse text file for neon detect.
321 cpu_info = ArmCpuCaps("/proc/cpuinfo");
322 #endif
323 cpu_info |= kCpuHasARM;
324 if (TestEnv("LIBYUV_DISABLE_NEON")) {
325 cpu_info &= ~kCpuHasNEON;
326 }
327 #endif // __arm__
328 if (TestEnv("LIBYUV_DISABLE_ASM")) {
329 cpu_info = 0;
330 }
331 cpu_info |= kCpuInitialized;
332 return cpu_info;
333 }
334
335 // Note that use of this function is not thread safe.
336 LIBYUV_API
MaskCpuFlags(int enable_flags)337 int MaskCpuFlags(int enable_flags) {
338 int cpu_info = GetCpuFlags() & enable_flags;
339 #ifdef __ATOMIC_RELAXED
340 __atomic_store_n(&cpu_info_, cpu_info, __ATOMIC_RELAXED);
341 #else
342 cpu_info_ = cpu_info;
343 #endif
344 return cpu_info;
345 }
346
347 LIBYUV_API
InitCpuFlags(void)348 int InitCpuFlags(void) {
349 return MaskCpuFlags(-1);
350 }
351
352 #ifdef __cplusplus
353 } // extern "C"
354 } // namespace libyuv
355 #endif
356