1 // cpu.cpp - originally written and placed in the public domain by Wei Dai
2 // modified by Jeffrey Walton and the community over the years.
3
4 #include "pch.h"
5 #include "config.h"
6
7 #ifndef EXCEPTION_EXECUTE_HANDLER
8 # define EXCEPTION_EXECUTE_HANDLER 1
9 #endif
10
11 #ifndef CRYPTOPP_IMPORTS
12
13 #include "cpu.h"
14 #include "misc.h"
15 #include "stdcpp.h"
16
17 // For _xgetbv on Microsoft 32-bit and 64-bit Intel platforms
18 // https://github.com/weidai11/cryptopp/issues/972
19 #if _MSC_VER >= 1600 && (defined(_M_IX86) || defined(_M_X64))
20 # include <immintrin.h>
21 #endif
22
23 #ifdef _AIX
24 # include <sys/systemcfg.h>
25 #endif
26
27 #ifdef __linux__
28 # include <unistd.h>
29 #endif
30
31 // Capability queries, requires Glibc 2.16, http://lwn.net/Articles/519085/
32 // CRYPTOPP_GLIBC_VERSION not used because config.h is missing <feature.h>
33 #if (((__GLIBC__ * 100) + __GLIBC_MINOR__) >= 216)
34 # define CRYPTOPP_GETAUXV_AVAILABLE 1
35 #endif
36
37 #if CRYPTOPP_GETAUXV_AVAILABLE
38 # include <sys/auxv.h>
39 #else
40 #ifndef AT_HWCAP
41 # define AT_HWCAP 16
42 #endif
43 #ifndef AT_HWCAP2
44 # define AT_HWCAP2 26
45 #endif
getauxval(unsigned long int)46 unsigned long int getauxval(unsigned long int) { return 0; }
47 #endif
48
49 #if defined(__APPLE__)
50 # include <sys/utsname.h>
51 # include <sys/sysctl.h>
52 #endif
53
54 // The cpu-features header and source file are located in
55 // "$ANDROID_NDK_ROOT/sources/android/cpufeatures".
56 // setenv-android.sh will copy the header and source file
57 // into PWD and the makefile will build it in place.
58 #if defined(__ANDROID__)
59 # include "cpu-features.h"
60 #endif
61
62 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
63 # include <signal.h>
64 # include <setjmp.h>
65 #endif
66
67 // Visual Studio 2008 and below are missing _xgetbv and _cpuidex.
68 // The 32-bit versions use inline ASM below. The 64-bit versions are in x64dll.asm.
69 #if defined(_MSC_VER) && defined(_M_X64)
70 extern "C" unsigned long long __fastcall XGETBV64(unsigned int);
71 extern "C" unsigned long long __fastcall CPUID64(unsigned int, unsigned int, unsigned int*);
72 #endif
73
74 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
75 extern "C" {
76 typedef void (*SigHandler)(int);
77 }
78
79 extern "C"
80 {
81 static jmp_buf s_jmpNoCPUID;
SigIllHandler(int)82 static void SigIllHandler(int)
83 {
84 longjmp(s_jmpNoCPUID, 1);
85 }
86 }
87 #endif // CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
88
89 ANONYMOUS_NAMESPACE_BEGIN
90
91 #if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
92
93 using CryptoPP::word32;
94
IsIntel(const word32 output[4])95 inline bool IsIntel(const word32 output[4])
96 {
97 // This is the "GenuineIntel" string
98 return (output[1] /*EBX*/ == 0x756e6547) &&
99 (output[2] /*ECX*/ == 0x6c65746e) &&
100 (output[3] /*EDX*/ == 0x49656e69);
101 }
102
IsAMD(const word32 output[4])103 inline bool IsAMD(const word32 output[4])
104 {
105 // This is the "AuthenticAMD" string.
106 return ((output[1] /*EBX*/ == 0x68747541) &&
107 (output[2] /*ECX*/ == 0x444D4163) &&
108 (output[3] /*EDX*/ == 0x69746E65)) ||
109 // Early K5's can return "AMDisbetter!"
110 ((output[1] /*EBX*/ == 0x69444d41) &&
111 (output[2] /*ECX*/ == 0x74656273) &&
112 (output[3] /*EDX*/ == 0x21726574));
113 }
114
IsHygon(const word32 output[4])115 inline bool IsHygon(const word32 output[4])
116 {
117 // This is the "HygonGenuine" string.
118 return (output[1] /*EBX*/ == 0x6f677948) &&
119 (output[2] /*ECX*/ == 0x656e6975) &&
120 (output[3] /*EDX*/ == 0x6e65476e);
121 }
122
IsVIA(const word32 output[4])123 inline bool IsVIA(const word32 output[4])
124 {
125 // This is the "CentaurHauls" string.
126 return ((output[1] /*EBX*/ == 0x746e6543) &&
127 (output[2] /*ECX*/ == 0x736c7561) &&
128 (output[3] /*EDX*/ == 0x48727561)) ||
129 // Some non-PadLock's return "VIA VIA VIA "
130 ((output[1] /*EBX*/ == 0x32414956) &&
131 (output[2] /*ECX*/ == 0x32414956) &&
132 (output[3] /*EDX*/ == 0x32414956));
133 }
134
135 #endif // X86, X32 and X64
136
137 #if defined(__APPLE__)
138
139 // http://stackoverflow.com/questions/45637888/how-to-determine-armv8-features-at-runtime-on-ios
140 class AppleMachineInfo
141 {
142 public:
143 enum { PowerMac=1, Mac, iPhone, iPod, iPad, AppleTV, AppleWatch };
144 enum { PowerPC=1, I386, I686, X86_64, ARM32, ARMV8, ARMV82, ARMV83 };
145
AppleMachineInfo()146 AppleMachineInfo() : m_device(0), m_version(0), m_arch(0)
147 {
148 struct utsname systemInfo;
149 systemInfo.machine[0] = '\0';
150 uname(&systemInfo);
151
152 std::string machine(systemInfo.machine);
153
154 std::string::size_type pos = machine.find_first_of("0123456789");
155 if (pos != std::string::npos)
156 m_version = std::atoi(machine.substr(pos).c_str());
157
158 if (machine.find("iPhone") != std::string::npos)
159 {
160 m_device = iPhone;
161 if (m_version >= 6) { m_arch = ARMV8; }
162 else { m_arch = ARM32; }
163 }
164 else if (machine.find("iPod") != std::string::npos)
165 {
166 m_device = iPod;
167 if (m_version >= 6) { m_arch = ARMV8; }
168 else { m_arch = ARM32; }
169 }
170 else if (machine.find("iPad") != std::string::npos)
171 {
172 m_device = iPad;
173 if (m_version >= 5) { m_arch = ARMV8; }
174 else { m_arch = ARM32; }
175 }
176 else if (machine.find("PowerMac") != std::string::npos ||
177 machine.find("Power Macintosh") != std::string::npos)
178 {
179 m_device = PowerMac;
180 m_arch = PowerPC;
181 }
182 else if (machine.find("Mac") != std::string::npos ||
183 machine.find("Macintosh") != std::string::npos)
184 {
185 #if defined(__x86_64) || defined(__amd64)
186 m_device = Mac;
187 m_arch = X86_64;
188 #elif defined(__i386)
189 m_device = Mac;
190 m_arch = I386;
191 #elif defined(__i686)
192 m_device = Mac;
193 m_arch = I686;
194 #else
195 // Should never get here
196 m_device = Mac;
197 m_arch = 0;
198 #endif
199 }
200 else if (machine.find("AppleTV") != std::string::npos)
201 {
202 m_device = AppleTV;
203 if (m_version >= 4) { m_arch = ARMV8; }
204 else { m_arch = ARM32; }
205 }
206 else if (machine.find("AppleWatch") != std::string::npos)
207 {
208 m_device = AppleWatch;
209 if (m_version >= 4) { m_arch = ARMV8; }
210 else { m_arch = ARM32; }
211 }
212 else if (machine.find("arm64") != std::string::npos)
213 {
214 // M1 machine?
215 std::string brand;
216 size_t size = 0;
217
218 if (sysctlbyname("machdep.cpu.brand_string", NULL, &size, NULL, 0) == 0 && size > 0)
219 {
220 brand.resize(size);
221 if (sysctlbyname("machdep.cpu.brand_string", &brand[0], &size, NULL, 0) == 0 && size > 0)
222 {
223 if (brand[size-1] == '\0')
224 size--;
225 brand.resize(size);
226 }
227 }
228
229 if (brand == "Apple M1")
230 {
231 m_device = Mac;
232 m_arch = ARMV82;
233 }
234 else
235 {
236 // ???
237 m_device = 0;
238 m_arch = ARMV8;
239 }
240 }
241 else
242 {
243 CRYPTOPP_ASSERT(0);
244 }
245 }
246
Device() const247 unsigned int Device() const {
248 return m_device;
249 }
250
Version() const251 unsigned int Version() const {
252 return m_version;
253 }
254
Arch() const255 unsigned int Arch() const {
256 return m_arch;
257 }
258
IsARM32() const259 bool IsARM32() const {
260 return m_arch == ARM32;
261 }
262
IsARMv8() const263 bool IsARMv8() const {
264 return m_arch >= ARMV8;
265 }
266
IsARMv82() const267 bool IsARMv82() const {
268 return m_arch >= ARMV82;
269 }
270
IsARMv83() const271 bool IsARMv83() const {
272 return m_arch >= ARMV83;
273 }
274
275 private:
276 unsigned int m_device, m_version, m_arch;
277 };
278
GetAppleMachineInfo(unsigned int & device,unsigned int & version,unsigned int & arch)279 void GetAppleMachineInfo(unsigned int& device, unsigned int& version, unsigned int& arch)
280 {
281 #if CRYPTOPP_CXX11_STATIC_INIT
282 static const AppleMachineInfo info;
283 #else
284 using CryptoPP::Singleton;
285 const AppleMachineInfo& info = Singleton<AppleMachineInfo>().Ref();
286 #endif
287
288 device = info.Device();
289 version = info.Version();
290 arch = info.Arch();
291 }
292
IsAppleMachineARM32()293 inline bool IsAppleMachineARM32()
294 {
295 static unsigned int arch;
296 if (arch == 0)
297 {
298 unsigned int unused;
299 GetAppleMachineInfo(unused, unused, arch);
300 }
301 return arch == AppleMachineInfo::ARM32;
302 }
303
IsAppleMachineARMv8()304 inline bool IsAppleMachineARMv8()
305 {
306 static unsigned int arch;
307 if (arch == 0)
308 {
309 unsigned int unused;
310 GetAppleMachineInfo(unused, unused, arch);
311 }
312 return arch >= AppleMachineInfo::ARMV8;
313 }
314
IsAppleMachineARMv82()315 inline bool IsAppleMachineARMv82()
316 {
317 static unsigned int arch;
318 if (arch == 0)
319 {
320 unsigned int unused;
321 GetAppleMachineInfo(unused, unused, arch);
322 }
323 return arch >= AppleMachineInfo::ARMV82;
324 }
325
IsAppleMachineARMv83()326 inline bool IsAppleMachineARMv83()
327 {
328 static unsigned int arch;
329 if (arch == 0)
330 {
331 unsigned int unused;
332 GetAppleMachineInfo(unused, unused, arch);
333 }
334 return arch >= AppleMachineInfo::ARMV83;
335 }
336
337 #endif // __APPLE__
338
339 ANONYMOUS_NAMESPACE_END
340
341 NAMESPACE_BEGIN(CryptoPP)
342
343 // *************************** IA-32 CPUs ***************************
344
345 #if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
346
347 bool CRYPTOPP_SECTION_INIT g_x86DetectionDone = false;
348 bool CRYPTOPP_SECTION_INIT g_hasSSE2 = false;
349 bool CRYPTOPP_SECTION_INIT g_hasSSSE3 = false;
350 bool CRYPTOPP_SECTION_INIT g_hasSSE41 = false;
351 bool CRYPTOPP_SECTION_INIT g_hasSSE42 = false;
352 bool CRYPTOPP_SECTION_INIT g_hasAESNI = false;
353 bool CRYPTOPP_SECTION_INIT g_hasCLMUL = false;
354 bool CRYPTOPP_SECTION_INIT g_hasMOVBE = false;
355 bool CRYPTOPP_SECTION_INIT g_hasAVX = false;
356 bool CRYPTOPP_SECTION_INIT g_hasAVX2 = false;
357 bool CRYPTOPP_SECTION_INIT g_hasADX = false;
358 bool CRYPTOPP_SECTION_INIT g_hasSHA = false;
359 bool CRYPTOPP_SECTION_INIT g_hasRDRAND = false;
360 bool CRYPTOPP_SECTION_INIT g_hasRDSEED = false;
361 bool CRYPTOPP_SECTION_INIT g_isP4 = false;
362 bool CRYPTOPP_SECTION_INIT g_hasPadlockRNG = false;
363 bool CRYPTOPP_SECTION_INIT g_hasPadlockACE = false;
364 bool CRYPTOPP_SECTION_INIT g_hasPadlockACE2 = false;
365 bool CRYPTOPP_SECTION_INIT g_hasPadlockPHE = false;
366 bool CRYPTOPP_SECTION_INIT g_hasPadlockPMM = false;
367 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
368
369 // For Solaris 11
370 extern bool CPU_ProbeSSE2();
371
372 // xcr0 is available when xgetbv is present.
373 // The intrinsic is broke on GCC 8.1 and earlier. Also see
374 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85684.
XGetBV(word32 num)375 word64 XGetBV(word32 num)
376 {
377 // Visual Studio 2010 SP1 and above, 32 and 64-bit
378 // https://github.com/weidai11/cryptopp/issues/972
379 #if defined(_MSC_VER) && (_MSC_FULL_VER >= 160040219)
380
381 return _xgetbv(num);
382
383 // Visual Studio 2008 and below, 64-bit
384 #elif defined(_MSC_VER) && defined(_M_X64)
385
386 return XGETBV64(num);
387
388 // Visual Studio 2008 and below, 32-bit
389 #elif defined(_MSC_VER) && defined(_M_IX86)
390
391 word32 a=0, d=0;
392 __asm {
393 push eax
394 push edx
395 push ecx
396 mov ecx, num
397 _emit 0x0f
398 _emit 0x01
399 _emit 0xd0
400 mov a, eax
401 mov d, edx
402 pop ecx
403 pop edx
404 pop eax
405 }
406 return (static_cast<word64>(d) << 32) | a;
407
408 // GCC 4.4 and above
409 #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4))
410
411 word32 a=0, d=0;
412 __asm__
413 (
414 "xgetbv" : "=a"(a), "=d"(d) : "c"(num) : "cc"
415 );
416 return (static_cast<word64>(d) << 32) | a;
417
418 // Remainder of GCC and compatibles.
419 #elif defined(__GNUC__) || defined(__clang__) || defined(__SUNPRO_CC)
420
421 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71659 and
422 // http://www.agner.org/optimize/vectorclass/read.php?i=65
423 word32 a=0, d=0;
424 __asm__
425 (
426 ".byte 0x0f, 0x01, 0xd0" "\n\t"
427 : "=a"(a), "=d"(d) : "c"(num) : "cc"
428 );
429 return (static_cast<word64>(d) << 32) | a;
430 #else
431 # error "Need an xgetbv function"
432 #endif
433 }
434
435 // No inline due to Borland/Embarcadero and Issue 498
436 // cpu.cpp (131): E2211 Inline assembly not allowed in inline and template functions
CpuId(word32 func,word32 subfunc,word32 output[4])437 bool CpuId(word32 func, word32 subfunc, word32 output[4])
438 {
439 // Visual Studio 2010 and above, 32 and 64-bit
440 #if defined(_MSC_VER) && (_MSC_VER >= 1600)
441
442 __cpuidex((int *)output, func, subfunc);
443 return true;
444
445 // Visual Studio 2008 and below, 64-bit
446 #elif defined(_MSC_VER) && defined(_M_X64)
447
448 CPUID64(func, subfunc, output);
449 return true;
450
451 // Visual Studio 2008 and below, 32-bit
452 #elif (defined(_MSC_VER) && defined(_M_IX86)) || defined(__BORLANDC__)
453
454 __try
455 {
456 // Borland/Embarcadero and Issue 500
457 // Local variables for cpuid output
458 word32 a, b, c, d;
459 __asm
460 {
461 push ebx
462 mov eax, func
463 mov ecx, subfunc
464 cpuid
465 mov [a], eax
466 mov [b], ebx
467 mov [c], ecx
468 mov [d], edx
469 pop ebx
470 }
471 output[0] = a;
472 output[1] = b;
473 output[2] = c;
474 output[3] = d;
475 }
476 __except (EXCEPTION_EXECUTE_HANDLER)
477 {
478 return false;
479 }
480
481 return true;
482
483 // Linux, Unix, OS X, Solaris, Cygwin, MinGW
484 #else
485
486 // longjmp and clobber warnings. Volatile is required.
487 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
488 volatile bool result = true;
489
490 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
491 if (oldHandler == SIG_ERR)
492 return false;
493
494 # ifndef __MINGW32__
495 volatile sigset_t oldMask;
496 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask) != 0)
497 {
498 signal(SIGILL, oldHandler);
499 return false;
500 }
501 # endif
502
503 if (setjmp(s_jmpNoCPUID))
504 result = false;
505 else
506 {
507 asm volatile
508 (
509 // save ebx in case -fPIC is being used
510 # if CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
511 "pushq %%rbx; cpuid; mov %%ebx, %%edi; popq %%rbx"
512 # else
513 "push %%ebx; cpuid; mov %%ebx, %%edi; pop %%ebx"
514 # endif
515 : "=a" (output[0]), "=D" (output[1]), "=c" (output[2]), "=d" (output[3])
516 : "a" (func), "c" (subfunc)
517 : "cc"
518 );
519 }
520
521 # ifndef __MINGW32__
522 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
523 # endif
524
525 signal(SIGILL, oldHandler);
526 return result;
527 #endif
528 }
529
DetectX86Features()530 void DetectX86Features()
531 {
532 // Coverity finding CID 171239. Initialize arrays.
533 // Indexes: EAX=0, EBX=1, ECX=2, EDX=3
534 word32 cpuid0[4]={0}, cpuid1[4]={0}, cpuid2[4]={0};
535
536 #if defined(CRYPTOPP_DISABLE_ASM)
537 // Not available
538 goto done;
539 #else
540 if (!CpuId(0, 0, cpuid0))
541 goto done;
542 if (!CpuId(1, 0, cpuid1))
543 goto done;
544 #endif
545
546 CRYPTOPP_CONSTANT(EAX_REG = 0);
547 CRYPTOPP_CONSTANT(EBX_REG = 1);
548 CRYPTOPP_CONSTANT(ECX_REG = 2);
549 CRYPTOPP_CONSTANT(EDX_REG = 3);
550
551 CRYPTOPP_CONSTANT(MMX_FLAG = (1 << 24)); // EDX
552 CRYPTOPP_CONSTANT(SSE_FLAG = (1 << 25)); // EDX
553 CRYPTOPP_CONSTANT(SSE2_FLAG = (1 << 26)); // EDX
554
555 CRYPTOPP_CONSTANT(SSE3_FLAG = (1 << 0)); // ECX
556 CRYPTOPP_CONSTANT(SSSE3_FLAG = (1 << 9)); // ECX
557 CRYPTOPP_CONSTANT(SSE41_FLAG = (1 << 19)); // ECX
558 CRYPTOPP_CONSTANT(SSE42_FLAG = (1 << 20)); // ECX
559 CRYPTOPP_CONSTANT(MOVBE_FLAG = (1 << 22)); // ECX
560 CRYPTOPP_CONSTANT(AESNI_FLAG = (1 << 25)); // ECX
561 CRYPTOPP_CONSTANT(CLMUL_FLAG = (1 << 1)); // ECX
562
563 CRYPTOPP_CONSTANT(XSAVE_FLAG = (1 << 26)); // ECX
564 CRYPTOPP_CONSTANT(OSXSAVE_FLAG = (1 << 27)); // ECX
565
566 CRYPTOPP_CONSTANT(AVX_FLAG = (3 << 27)); // ECX
567 CRYPTOPP_CONSTANT(YMM_FLAG = (3 << 1)); // CR0
568
569 // x86_64 machines don't check some flags because SSE2
570 // is part of the core instruction set architecture
571 CRYPTOPP_UNUSED(MMX_FLAG); CRYPTOPP_UNUSED(SSE_FLAG);
572 CRYPTOPP_UNUSED(SSE3_FLAG); CRYPTOPP_UNUSED(XSAVE_FLAG);
573
574 #if (CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
575 // 64-bit core instruction set includes SSE2. Just check
576 // the OS enabled SSE2 support using OSXSAVE.
577 g_hasSSE2 = (cpuid1[ECX_REG] & OSXSAVE_FLAG) != 0;
578 #else
579 // Check the processor supports SSE2. Then use OSXSAVE to
580 // signal OS support for SSE2 to avoid probes.
581 // Also see http://stackoverflow.com/a/22521619/608639
582 // and http://github.com/weidai11/cryptopp/issues/511.
583 if ((cpuid1[EDX_REG] & SSE2_FLAG) == SSE2_FLAG)
584 g_hasSSE2 = (cpuid1[ECX_REG] & XSAVE_FLAG) != 0 &&
585 (cpuid1[ECX_REG] & OSXSAVE_FLAG) != 0;
586 #endif
587
588 // Solaris 11 i86pc does not signal SSE support using
589 // OSXSAVE. We need to probe for SSE support.
590 if (g_hasSSE2 == false)
591 g_hasSSE2 = CPU_ProbeSSE2();
592
593 if (g_hasSSE2 == false)
594 goto done;
595
596 g_hasSSSE3 = (cpuid1[ECX_REG] & SSSE3_FLAG) != 0;
597 g_hasSSE41 = (cpuid1[ECX_REG] & SSE41_FLAG) != 0;
598 g_hasSSE42 = (cpuid1[ECX_REG] & SSE42_FLAG) != 0;
599 g_hasMOVBE = (cpuid1[ECX_REG] & MOVBE_FLAG) != 0;
600 g_hasAESNI = (cpuid1[ECX_REG] & AESNI_FLAG) != 0;
601 g_hasCLMUL = (cpuid1[ECX_REG] & CLMUL_FLAG) != 0;
602
603 // AVX is similar to SSE. Check if AVX is available on the cpu, then
604 // check if the OS enabled XSAVE/XRESTORE for the extended registers.
605 // https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
606 if ((cpuid1[ECX_REG] & AVX_FLAG) == AVX_FLAG)
607 {
608 word64 xcr0 = XGetBV(0);
609 g_hasAVX = (xcr0 & YMM_FLAG) == YMM_FLAG;
610 }
611
612 if (IsIntel(cpuid0))
613 {
614 CRYPTOPP_CONSTANT(RDRAND_FLAG = (1 << 30));
615 CRYPTOPP_CONSTANT(RDSEED_FLAG = (1 << 18));
616 CRYPTOPP_CONSTANT( ADX_FLAG = (1 << 19));
617 CRYPTOPP_CONSTANT( SHA_FLAG = (1 << 29));
618 CRYPTOPP_CONSTANT( AVX2_FLAG = (1 << 5));
619
620 g_isP4 = ((cpuid1[0] >> 8) & 0xf) == 0xf;
621 g_cacheLineSize = 8 * GETBYTE(cpuid1[1], 1);
622 g_hasRDRAND = (cpuid1[ECX_REG] & RDRAND_FLAG) != 0;
623
624 if (cpuid0[EAX_REG] >= 7)
625 {
626 if (CpuId(7, 0, cpuid2))
627 {
628 g_hasRDSEED = (cpuid2[EBX_REG] & RDSEED_FLAG) != 0;
629 g_hasADX = (cpuid2[EBX_REG] & ADX_FLAG) != 0;
630 g_hasSHA = (cpuid2[EBX_REG] & SHA_FLAG) != 0;
631 g_hasAVX2 = (cpuid2[EBX_REG] & AVX2_FLAG) != 0;
632 }
633 }
634 }
635 else if (IsAMD(cpuid0) || IsHygon(cpuid0))
636 {
637 CRYPTOPP_CONSTANT(RDRAND_FLAG = (1 << 30));
638 CRYPTOPP_CONSTANT(RDSEED_FLAG = (1 << 18));
639 CRYPTOPP_CONSTANT( ADX_FLAG = (1 << 19));
640 CRYPTOPP_CONSTANT( SHA_FLAG = (1 << 29));
641 CRYPTOPP_CONSTANT( AVX2_FLAG = (1 << 5));
642
643 CpuId(0x80000005, 0, cpuid2);
644 g_cacheLineSize = GETBYTE(cpuid2[ECX_REG], 0);
645 g_hasRDRAND = (cpuid1[ECX_REG] & RDRAND_FLAG) != 0;
646
647 if (cpuid0[EAX_REG] >= 7)
648 {
649 if (CpuId(7, 0, cpuid2))
650 {
651 g_hasRDSEED = (cpuid2[EBX_REG] & RDSEED_FLAG) != 0;
652 g_hasADX = (cpuid2[EBX_REG] & ADX_FLAG) != 0;
653 g_hasSHA = (cpuid2[EBX_REG] & SHA_FLAG) != 0;
654 g_hasAVX2 = (cpuid2[EBX_REG] & AVX2_FLAG) != 0;
655 }
656 }
657
658 // Unconditionally disable RDRAND and RDSEED on AMD cpu's with family 15h or 16h.
659 // See Crypto++ Issue 924, https://github.com/weidai11/cryptopp/issues/924,
660 // Clear RDRAND CPUID bit on AMD family 15h/16h, https://lore.kernel.org/patchwork/patch/1115413/,
661 // and AMD CPUID Specification, https://www.amd.com/system/files/TechDocs/25481.pdf
662 {
663 CRYPTOPP_CONSTANT(FAMILY_BASE_FLAG = (0x0f << 8));
664 CRYPTOPP_CONSTANT(FAMILY_EXT_FLAG = (0xff << 20));
665
666 word32 family = (cpuid1[0] & FAMILY_BASE_FLAG) >> 8;
667 if (family == 0xf)
668 family += (cpuid1[0] & FAMILY_EXT_FLAG) >> 20;
669 if (family == 0x15 || family == 0x16)
670 {
671 g_hasRDRAND = false;
672 g_hasRDSEED = false;
673 }
674 }
675 }
676 else if (IsVIA(cpuid0))
677 {
678 // Two bits: available and enabled
679 CRYPTOPP_CONSTANT( RNG_FLAGS = (0x3 << 2));
680 CRYPTOPP_CONSTANT( ACE_FLAGS = (0x3 << 6));
681 CRYPTOPP_CONSTANT(ACE2_FLAGS = (0x3 << 8));
682 CRYPTOPP_CONSTANT( PHE_FLAGS = (0x3 << 10));
683 CRYPTOPP_CONSTANT( PMM_FLAGS = (0x3 << 12));
684
685 CpuId(0xC0000000, 0, cpuid2);
686 word32 extendedFeatures = cpuid2[0];
687
688 if (extendedFeatures >= 0xC0000001)
689 {
690 CpuId(0xC0000001, 0, cpuid2);
691 g_hasPadlockRNG = (cpuid2[EDX_REG] & RNG_FLAGS) != 0;
692 g_hasPadlockACE = (cpuid2[EDX_REG] & ACE_FLAGS) != 0;
693 g_hasPadlockACE2 = (cpuid2[EDX_REG] & ACE2_FLAGS) != 0;
694 g_hasPadlockPHE = (cpuid2[EDX_REG] & PHE_FLAGS) != 0;
695 g_hasPadlockPMM = (cpuid2[EDX_REG] & PMM_FLAGS) != 0;
696 }
697
698 if (extendedFeatures >= 0xC0000005)
699 {
700 CpuId(0xC0000005, 0, cpuid2);
701 g_cacheLineSize = GETBYTE(cpuid2[ECX_REG], 0);
702 }
703 }
704
705 // Keep AVX2 in sync with OS support for AVX. AVX tests both
706 // cpu support and OS support, while AVX2 only tests cpu support.
707 g_hasAVX2 &= g_hasAVX;
708
709 done:
710
711 #if defined(_SC_LEVEL1_DCACHE_LINESIZE)
712 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
713 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
714 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
715 if (g_cacheLineSize == 0 && cacheLineSize > 0)
716 g_cacheLineSize = cacheLineSize;
717 #endif
718
719 if (g_cacheLineSize == 0)
720 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
721
722 *const_cast<volatile bool*>(&g_x86DetectionDone) = true;
723 }
724
725 // *************************** ARM-32, Aarch32 and Aarch64 ***************************
726
727 #elif (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
728
729 bool CRYPTOPP_SECTION_INIT g_ArmDetectionDone = false;
730 bool CRYPTOPP_SECTION_INIT g_hasARMv7 = false;
731 bool CRYPTOPP_SECTION_INIT g_hasNEON = false;
732 bool CRYPTOPP_SECTION_INIT g_hasPMULL = false;
733 bool CRYPTOPP_SECTION_INIT g_hasCRC32 = false;
734 bool CRYPTOPP_SECTION_INIT g_hasAES = false;
735 bool CRYPTOPP_SECTION_INIT g_hasSHA1 = false;
736 bool CRYPTOPP_SECTION_INIT g_hasSHA2 = false;
737 bool CRYPTOPP_SECTION_INIT g_hasSHA512 = false;
738 bool CRYPTOPP_SECTION_INIT g_hasSHA3 = false;
739 bool CRYPTOPP_SECTION_INIT g_hasSM3 = false;
740 bool CRYPTOPP_SECTION_INIT g_hasSM4 = false;
741 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
742
743 // ARM does not have an unprivileged equivalent to CPUID on IA-32. We have to
744 // jump through some hoops to detect features on a wide array of platforms.
745 // Our strategy is two part. First, attempt to *Query* the OS for a feature,
746 // like using getauxval on Linux. If that fails, then *Probe* the cpu
747 // executing an instruction and an observe a SIGILL if unsupported. The probes
748 // are in source files where compilation options like -march=armv8-a+crc make
749 // intrinsics available. They are expensive when compared to a standard OS
750 // feature query. Always perform the feature query first. For Linux see
751 // http://sourceware.org/ml/libc-help/2017-08/msg00012.html
752 // Avoid probes on Apple platforms because Apple's signal handling for SIGILLs
753 // appears broken. We are trying to figure out a way to feature test without
754 // probes. Also see http://stackoverflow.com/a/11197770/608639 and
755 // http://gist.github.com/erkanyildiz/390a480f27e86f8cd6ba.
756
757 extern bool CPU_ProbeARMv7();
758 extern bool CPU_ProbeNEON();
759 extern bool CPU_ProbeCRC32();
760 extern bool CPU_ProbeAES();
761 extern bool CPU_ProbeSHA1();
762 extern bool CPU_ProbeSHA256();
763 extern bool CPU_ProbeSHA512();
764 extern bool CPU_ProbeSHA3();
765 extern bool CPU_ProbeSM3();
766 extern bool CPU_ProbeSM4();
767 extern bool CPU_ProbePMULL();
768
769 // https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h
770 // https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h
771 #ifndef HWCAP_ARMv7
772 # define HWCAP_ARMv7 (1 << 29)
773 #endif
774 #ifndef HWCAP_ASIMD
775 # define HWCAP_ASIMD (1 << 1)
776 #endif
777 #ifndef HWCAP_NEON
778 # define HWCAP_NEON (1 << 12)
779 #endif
780 #ifndef HWCAP_CRC32
781 # define HWCAP_CRC32 (1 << 7)
782 #endif
783 #ifndef HWCAP2_CRC32
784 # define HWCAP2_CRC32 (1 << 4)
785 #endif
786 #ifndef HWCAP_PMULL
787 # define HWCAP_PMULL (1 << 4)
788 #endif
789 #ifndef HWCAP2_PMULL
790 # define HWCAP2_PMULL (1 << 1)
791 #endif
792 #ifndef HWCAP_AES
793 # define HWCAP_AES (1 << 3)
794 #endif
795 #ifndef HWCAP2_AES
796 # define HWCAP2_AES (1 << 0)
797 #endif
798 #ifndef HWCAP_SHA1
799 # define HWCAP_SHA1 (1 << 5)
800 #endif
801 #ifndef HWCAP_SHA2
802 # define HWCAP_SHA2 (1 << 6)
803 #endif
804 #ifndef HWCAP2_SHA1
805 # define HWCAP2_SHA1 (1 << 2)
806 #endif
807 #ifndef HWCAP2_SHA2
808 # define HWCAP2_SHA2 (1 << 3)
809 #endif
810 #ifndef HWCAP_SHA3
811 # define HWCAP_SHA3 (1 << 17)
812 #endif
813 #ifndef HWCAP_SM3
814 # define HWCAP_SM3 (1 << 18)
815 #endif
816 #ifndef HWCAP_SM4
817 # define HWCAP_SM4 (1 << 19)
818 #endif
819 #ifndef HWCAP_SHA512
820 # define HWCAP_SHA512 (1 << 21)
821 #endif
822
823 inline bool CPU_QueryARMv7()
824 {
825 #if defined(__ANDROID__) && defined(__arm__)
826 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
827 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_ARMv7) != 0))
828 return true;
829 #elif defined(__linux__) && defined(__arm__)
830 if ((getauxval(AT_HWCAP) & HWCAP_ARMv7) != 0 ||
831 (getauxval(AT_HWCAP) & HWCAP_NEON) != 0)
832 return true;
833 #elif defined(__APPLE__) && defined(__arm__)
834 // Apple hardware is ARMv7 or above.
835 return true;
836 #endif
837 return false;
838 }
839
840 inline bool CPU_QueryNEON()
841 {
842 #if defined(__ANDROID__) && defined(__aarch64__)
843 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
844 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_ASIMD) != 0))
845 return true;
846 #elif defined(__ANDROID__) && defined(__arm__)
847 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
848 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0))
849 return true;
850 #elif defined(__linux__) && defined(__aarch64__)
851 if ((getauxval(AT_HWCAP) & HWCAP_ASIMD) != 0)
852 return true;
853 #elif defined(__linux__) && defined(__aarch32__)
854 if ((getauxval(AT_HWCAP2) & HWCAP2_ASIMD) != 0)
855 return true;
856 #elif defined(__linux__) && defined(__arm__)
857 if ((getauxval(AT_HWCAP) & HWCAP_NEON) != 0)
858 return true;
859 #elif defined(__APPLE__) && defined(__aarch64__)
860 // Core feature set for Aarch32 and Aarch64.
861 return true;
862 #endif
863 return false;
864 }
865
866 inline bool CPU_QueryCRC32()
867 {
868 #if defined(__ANDROID__) && defined(__aarch64__)
869 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
870 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_CRC32) != 0))
871 return true;
872 #elif defined(__ANDROID__) && defined(__aarch32__)
873 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
874 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_CRC32) != 0))
875 return true;
876 #elif defined(__linux__) && defined(__aarch64__)
877 if ((getauxval(AT_HWCAP) & HWCAP_CRC32) != 0)
878 return true;
879 #elif defined(__linux__) && defined(__aarch32__)
880 if ((getauxval(AT_HWCAP2) & HWCAP2_CRC32) != 0)
881 return true;
882 #elif defined(__APPLE__) && defined(__aarch64__)
883 // M1 processor
884 if (IsAppleMachineARMv82())
885 return true;
886 #endif
887 return false;
888 }
889
890 inline bool CPU_QueryPMULL()
891 {
892 #if defined(__ANDROID__) && defined(__aarch64__)
893 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
894 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_PMULL) != 0))
895 return true;
896 #elif defined(__ANDROID__) && defined(__aarch32__)
897 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
898 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_PMULL) != 0))
899 return true;
900 #elif defined(__linux__) && defined(__aarch64__)
901 if ((getauxval(AT_HWCAP) & HWCAP_PMULL) != 0)
902 return true;
903 #elif defined(__linux__) && defined(__aarch32__)
904 if ((getauxval(AT_HWCAP2) & HWCAP2_PMULL) != 0)
905 return true;
906 #elif defined(__APPLE__) && defined(__aarch64__)
907 // M1 processor
908 if (IsAppleMachineARMv82())
909 return true;
910 #endif
911 return false;
912 }
913
914 inline bool CPU_QueryAES()
915 {
916 #if defined(__ANDROID__) && defined(__aarch64__)
917 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
918 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_AES) != 0))
919 return true;
920 #elif defined(__ANDROID__) && defined(__aarch32__)
921 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
922 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_AES) != 0))
923 return true;
924 #elif defined(__linux__) && defined(__aarch64__)
925 if ((getauxval(AT_HWCAP) & HWCAP_AES) != 0)
926 return true;
927 #elif defined(__linux__) && defined(__aarch32__)
928 if ((getauxval(AT_HWCAP2) & HWCAP2_AES) != 0)
929 return true;
930 #elif defined(__APPLE__) && defined(__aarch64__)
931 return IsAppleMachineARMv8();
932 #endif
933 return false;
934 }
935
936 inline bool CPU_QuerySHA1()
937 {
938 #if defined(__ANDROID__) && defined(__aarch64__)
939 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
940 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA1) != 0))
941 return true;
942 #elif defined(__ANDROID__) && defined(__aarch32__)
943 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
944 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA1) != 0))
945 return true;
946 #elif defined(__linux__) && defined(__aarch64__)
947 if ((getauxval(AT_HWCAP) & HWCAP_SHA1) != 0)
948 return true;
949 #elif defined(__linux__) && defined(__aarch32__)
950 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA1) != 0)
951 return true;
952 #elif defined(__APPLE__) && defined(__aarch64__)
953 return IsAppleMachineARMv8();
954 #endif
955 return false;
956 }
957
958 inline bool CPU_QuerySHA256()
959 {
960 #if defined(__ANDROID__) && defined(__aarch64__)
961 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
962 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA2) != 0))
963 return true;
964 #elif defined(__ANDROID__) && defined(__aarch32__)
965 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
966 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA2) != 0))
967 return true;
968 #elif defined(__linux__) && defined(__aarch64__)
969 if ((getauxval(AT_HWCAP) & HWCAP_SHA2) != 0)
970 return true;
971 #elif defined(__linux__) && defined(__aarch32__)
972 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA2) != 0)
973 return true;
974 #elif defined(__APPLE__) && defined(__aarch64__)
975 return IsAppleMachineARMv8();
976 #endif
977 return false;
978 }
979
980 // Some ARMv8.2 features are disabled at the moment
981 inline bool CPU_QuerySHA3()
982 {
983 // According to the ARM manual, SHA3 depends upon SHA1 and SHA2.
984 // If SHA1 and SHA2 are not present, then SHA3 and SHA512 are
985 // not present. Also see Arm A64 Instruction Set Architecture,
986 // https://developer.arm.com/documentation/ddi0596/2020-12/
987 if (!g_hasSHA1 || !g_hasSHA2) { return false; }
988
989 #if defined(__ANDROID__) && defined(__aarch64__) && 0
990 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
991 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA3) != 0))
992 return true;
993 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
994 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
995 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA3) != 0))
996 return true;
997 #elif defined(__linux__) && defined(__aarch64__)
998 if ((getauxval(AT_HWCAP) & HWCAP_SHA3) != 0)
999 return true;
1000 #elif defined(__linux__) && defined(__aarch32__)
1001 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA3) != 0)
1002 return true;
1003 #elif defined(__APPLE__) && defined(__aarch64__)
1004 // M1 processor
1005 if (IsAppleMachineARMv82())
1006 return true;
1007 #endif
1008 return false;
1009 }
1010
1011 // Some ARMv8.2 features are disabled at the moment
1012 inline bool CPU_QuerySHA512()
1013 {
1014 // According to the ARM manual, SHA512 depends upon SHA1 and SHA2.
1015 // If SHA1 and SHA2 are not present, then SHA3 and SHA512 are
1016 // not present. Also see Arm A64 Instruction Set Architecture,
1017 // https://developer.arm.com/documentation/ddi0596/2020-12/
1018 if (!g_hasSHA1 || !g_hasSHA2) { return false; }
1019
1020 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1021 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1022 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA512) != 0))
1023 return true;
1024 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1025 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1026 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA512) != 0))
1027 return true;
1028 #elif defined(__linux__) && defined(__aarch64__)
1029 if ((getauxval(AT_HWCAP) & HWCAP_SHA512) != 0)
1030 return true;
1031 #elif defined(__linux__) && defined(__aarch32__)
1032 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA512) != 0)
1033 return true;
1034 #elif defined(__APPLE__) && defined(__aarch64__)
1035 // M1 processor
1036 if (IsAppleMachineARMv82())
1037 return true;
1038 #endif
1039 return false;
1040 }
1041
1042 // Some ARMv8.2 features are disabled at the moment
1043 inline bool CPU_QuerySM3()
1044 {
1045 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1046 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1047 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SM3) != 0))
1048 return true;
1049 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1050 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1051 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SM3) != 0))
1052 return true;
1053 #elif defined(__linux__) && defined(__aarch64__)
1054 if ((getauxval(AT_HWCAP) & HWCAP_SM3) != 0)
1055 return true;
1056 #elif defined(__linux__) && defined(__aarch32__)
1057 if ((getauxval(AT_HWCAP2) & HWCAP2_SM3) != 0)
1058 return true;
1059 #elif defined(__APPLE__) && defined(__aarch64__) && 0
1060 // No Apple support yet.
1061 #endif
1062 return false;
1063 }
1064
1065 // Some ARMv8.2 features are disabled at the moment
1066 inline bool CPU_QuerySM4()
1067 {
1068 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1069 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1070 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SM4) != 0))
1071 return true;
1072 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1073 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1074 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SM4) != 0))
1075 return true;
1076 #elif defined(__linux__) && defined(__aarch64__)
1077 if ((getauxval(AT_HWCAP) & HWCAP_SM4) != 0)
1078 return true;
1079 #elif defined(__linux__) && defined(__aarch32__)
1080 if ((getauxval(AT_HWCAP2) & HWCAP2_SM4) != 0)
1081 return true;
1082 #elif defined(__APPLE__) && defined(__aarch64__) && 0
1083 // No Apple support yet.
1084 #endif
1085 return false;
1086 }
1087
1088 void DetectArmFeatures()
1089 {
1090 // The CPU_ProbeXXX's return false for OSes which
1091 // can't tolerate SIGILL-based probes
1092 g_hasARMv7 = CPU_QueryARMv7() || CPU_ProbeARMv7();
1093 g_hasNEON = CPU_QueryNEON() || CPU_ProbeNEON();
1094 g_hasCRC32 = CPU_QueryCRC32() || CPU_ProbeCRC32();
1095 g_hasPMULL = CPU_QueryPMULL() || CPU_ProbePMULL();
1096 g_hasAES = CPU_QueryAES() || CPU_ProbeAES();
1097 g_hasSHA1 = CPU_QuerySHA1() || CPU_ProbeSHA1();
1098 g_hasSHA2 = CPU_QuerySHA256() || CPU_ProbeSHA256();
1099 g_hasSHA512 = CPU_QuerySHA512(); // || CPU_ProbeSHA512();
1100 g_hasSHA3 = CPU_QuerySHA3(); // || CPU_ProbeSHA3();
1101 g_hasSM3 = CPU_QuerySM3(); // || CPU_ProbeSM3();
1102 g_hasSM4 = CPU_QuerySM4(); // || CPU_ProbeSM4();
1103
1104 #if defined(_SC_LEVEL1_DCACHE_LINESIZE)
1105 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
1106 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
1107 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
1108 if (cacheLineSize > 0)
1109 g_cacheLineSize = cacheLineSize;
1110 #endif
1111
1112 if (g_cacheLineSize == 0)
1113 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1114
1115 *const_cast<volatile bool*>(&g_ArmDetectionDone) = true;
1116 }
1117
1118 // *************************** PowerPC and PowerPC64 ***************************
1119
1120 #elif (CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64)
1121
1122 bool CRYPTOPP_SECTION_INIT g_PowerpcDetectionDone = false;
1123 bool CRYPTOPP_SECTION_INIT g_hasAltivec = false;
1124 bool CRYPTOPP_SECTION_INIT g_hasPower7 = false;
1125 bool CRYPTOPP_SECTION_INIT g_hasPower8 = false;
1126 bool CRYPTOPP_SECTION_INIT g_hasPower9 = false;
1127 bool CRYPTOPP_SECTION_INIT g_hasAES = false;
1128 bool CRYPTOPP_SECTION_INIT g_hasPMULL = false;
1129 bool CRYPTOPP_SECTION_INIT g_hasSHA256 = false;
1130 bool CRYPTOPP_SECTION_INIT g_hasSHA512 = false;
1131 bool CRYPTOPP_SECTION_INIT g_hasDARN = false;
1132 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1133
1134 extern bool CPU_ProbeAltivec();
1135 extern bool CPU_ProbePower7();
1136 extern bool CPU_ProbePower8();
1137 extern bool CPU_ProbePower9();
1138 extern bool CPU_ProbeAES();
1139 extern bool CPU_ProbePMULL();
1140 extern bool CPU_ProbeSHA256();
1141 extern bool CPU_ProbeSHA512();
1142 extern bool CPU_ProbeDARN();
1143
1144 // AIX defines. We used to just call __power_7_andup()
1145 // and friends but at Power9, too many compilers were
1146 // missing __power_9_andup(). Instead we switched to
1147 // a pattern similar to OpenSSL caps testing.
1148 #ifndef __power_6_andup
1149 # define __power_6_andup() __power_set(0xffffffffU<<14)
1150 #endif
1151 #ifndef __power_7_andup
1152 # define __power_7_andup() __power_set(0xffffffffU<<15)
1153 #endif
1154 #ifndef __power_8_andup
1155 # define __power_8_andup() __power_set(0xffffffffU<<16)
1156 #endif
1157 #ifndef __power_9_andup
1158 # define __power_9_andup() __power_set(0xffffffffU<<17)
1159 #endif
1160
1161 // AIX first supported Altivec at Power6, though it
1162 // was available much earlier for other vendors.
1163 inline bool CPU_QueryAltivec()
1164 {
1165 #if defined(__linux__) && defined(PPC_FEATURE_HAS_ALTIVEC)
1166 if ((getauxval(AT_HWCAP) & PPC_FEATURE_HAS_ALTIVEC) != 0)
1167 return true;
1168 #elif defined(_AIX)
1169 if (__power_6_andup() != 0)
1170 return true;
1171 #elif defined(__APPLE__) && defined(__POWERPC__)
1172 unsigned int unused, arch;
1173 GetAppleMachineInfo(unused, unused, arch);
1174 return arch == AppleMachineInfo::PowerMac;
1175 #endif
1176 return false;
1177 }
1178
1179 inline bool CPU_QueryPower7()
1180 {
1181 // Power7 and ISA 2.06
1182 #if defined(__linux__) && defined(PPC_FEATURE_ARCH_2_06)
1183 if ((getauxval(AT_HWCAP) & PPC_FEATURE_ARCH_2_06) != 0)
1184 return true;
1185 #elif defined(_AIX)
1186 if (__power_7_andup() != 0)
1187 return true;
1188 #endif
1189 return false;
1190 }
1191
1192 inline bool CPU_QueryPower8()
1193 {
1194 // Power8 and ISA 2.07 provide in-core crypto.
1195 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_2_07)
1196 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_2_07) != 0)
1197 return true;
1198 #elif defined(_AIX)
1199 if (__power_8_andup() != 0)
1200 return true;
1201 #endif
1202 return false;
1203 }
1204
1205 inline bool CPU_QueryPower9()
1206 {
1207 // Power9 and ISA 3.0.
1208 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_3_00)
1209 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_3_00) != 0)
1210 return true;
1211 #elif defined(_AIX)
1212 if (__power_9_andup() != 0)
1213 return true;
1214 #endif
1215 return false;
1216 }
1217
1218 inline bool CPU_QueryAES()
1219 {
1220 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1221 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1222 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1223 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1224 return true;
1225 #elif defined(_AIX)
1226 if (__power_8_andup() != 0)
1227 return true;
1228 #endif
1229 return false;
1230 }
1231
1232 inline bool CPU_QueryPMULL()
1233 {
1234 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1235 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1236 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1237 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1238 return true;
1239 #elif defined(_AIX)
1240 if (__power_8_andup() != 0)
1241 return true;
1242 #endif
1243 return false;
1244 }
1245
1246 inline bool CPU_QuerySHA256()
1247 {
1248 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1249 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1250 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1251 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1252 return true;
1253 #elif defined(_AIX)
1254 if (__power_8_andup() != 0)
1255 return true;
1256 #endif
1257 return false;
1258 }
1259 inline bool CPU_QuerySHA512()
1260 {
1261 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1262 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1263 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1264 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1265 return true;
1266 #elif defined(_AIX)
1267 if (__power_8_andup() != 0)
1268 return true;
1269 #endif
1270 return false;
1271 }
1272
1273 // Power9 random number generator
1274 inline bool CPU_QueryDARN()
1275 {
1276 // Power9 and ISA 3.0 provide DARN.
1277 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_3_00)
1278 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_3_00) != 0)
1279 return true;
1280 #elif defined(_AIX)
1281 if (__power_9_andup() != 0)
1282 return true;
1283 #endif
1284 return false;
1285 }
1286
1287 void DetectPowerpcFeatures()
1288 {
1289 // GCC 10 is giving us trouble in CPU_ProbePower9() and
1290 // CPU_ProbeDARN(). GCC is generating POWER9 instructions
1291 // on POWER8 for ppc_power9.cpp. The compiler idiots did
1292 // not think through the consequences of requiring us to
1293 // use -mcpu=power9 to unlock the ISA. Epic fail.
1294 // https://github.com/weidai11/cryptopp/issues/986
1295
1296 // The CPU_ProbeXXX's return false for OSes which
1297 // can't tolerate SIGILL-based probes, like Apple
1298 g_hasAltivec = CPU_QueryAltivec() || CPU_ProbeAltivec();
1299 g_hasPower7 = CPU_QueryPower7() || CPU_ProbePower7();
1300 g_hasPower8 = CPU_QueryPower8() || CPU_ProbePower8();
1301 g_hasPower9 = CPU_QueryPower9() || CPU_ProbePower9();
1302 g_hasPMULL = CPU_QueryPMULL() || CPU_ProbePMULL();
1303 g_hasAES = CPU_QueryAES() || CPU_ProbeAES();
1304 g_hasSHA256 = CPU_QuerySHA256() || CPU_ProbeSHA256();
1305 g_hasSHA512 = CPU_QuerySHA512() || CPU_ProbeSHA512();
1306 g_hasDARN = CPU_QueryDARN() || CPU_ProbeDARN();
1307
1308 #if defined(_AIX) && defined(SC_L1C_DLS)
1309 // /usr/include/sys/systemcfg.h
1310 int cacheLineSize = getsystemcfg(SC_L1C_DLS);
1311 if (cacheLineSize > 0)
1312 g_cacheLineSize = cacheLineSize;
1313 #elif defined(_SC_LEVEL1_DCACHE_LINESIZE)
1314 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
1315 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
1316 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
1317 if (cacheLineSize > 0)
1318 g_cacheLineSize = cacheLineSize;
1319 #endif
1320
1321 if (g_cacheLineSize == 0)
1322 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1323
1324 *const_cast<volatile bool*>(&g_PowerpcDetectionDone) = true;
1325 }
1326
1327 #endif
1328 NAMESPACE_END
1329
1330 // *************************** C++ Static Initialization ***************************
1331
1332 ANONYMOUS_NAMESPACE_BEGIN
1333
1334 class InitCpu
1335 {
1336 public:
InitCpu()1337 InitCpu()
1338 {
1339 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
1340 CryptoPP::DetectX86Features();
1341 #elif CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8
1342 CryptoPP::DetectArmFeatures();
1343 #elif CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64
1344 CryptoPP::DetectPowerpcFeatures();
1345 #endif
1346 }
1347 };
1348
1349 // This is not really needed because HasSSE() and friends can dynamically initialize.
1350 // Everything depends on CPU features so we initialize it once at load time.
1351 // Dynamic initialization will be used if init priorities are not available.
1352
1353 #if HAVE_GCC_INIT_PRIORITY
1354 const InitCpu s_init __attribute__ ((init_priority (CRYPTOPP_INIT_PRIORITY + 10))) = InitCpu();
1355 #elif HAVE_MSC_INIT_PRIORITY
1356 #pragma warning(disable: 4075)
1357 #pragma init_seg(".CRT$XCU")
1358 const InitCpu s_init;
1359 #pragma warning(default: 4075)
1360 #elif HAVE_XLC_INIT_PRIORITY
1361 // XLC needs constant, not a define
1362 #pragma priority(270)
1363 const InitCpu s_init;
1364 #else
1365 const InitCpu s_init;
1366 #endif
1367
1368 ANONYMOUS_NAMESPACE_END
1369
1370 #endif // CRYPTOPP_IMPORTS
1371