14ec2eca9Schristos /*
2*5b10f583Schristos * Copyright 2011-2022 The OpenSSL Project Authors. All Rights Reserved.
34ec2eca9Schristos *
4*5b10f583Schristos * Licensed under the Apache License 2.0 (the "License"). You may not use
54ec2eca9Schristos * this file except in compliance with the License. You can obtain a copy
64ec2eca9Schristos * in the file LICENSE in the source distribution or at
74ec2eca9Schristos * https://www.openssl.org/source/license.html
84ec2eca9Schristos */
94ec2eca9Schristos
1013ce5681Schristos #include <stdio.h>
1113ce5681Schristos #include <stdlib.h>
1213ce5681Schristos #include <string.h>
1313ce5681Schristos #include <setjmp.h>
1413ce5681Schristos #include <signal.h>
154ec2eca9Schristos #include <openssl/crypto.h>
16*5b10f583Schristos #ifdef __APPLE__
17*5b10f583Schristos #include <sys/sysctl.h>
18*5b10f583Schristos #endif
19e7ccb6d1Schristos #include "internal/cryptlib.h"
2013ce5681Schristos
2113ce5681Schristos #include "arm_arch.h"
2213ce5681Schristos
237fa80bafSspz unsigned int OPENSSL_armcap_P = 0;
24*5b10f583Schristos unsigned int OPENSSL_arm_midr = 0;
25*5b10f583Schristos unsigned int OPENSSL_armv8_rsa_neonized = 0;
2613ce5681Schristos
277fa80bafSspz #if __ARM_MAX_ARCH__<7
OPENSSL_cpuid_setup(void)287fa80bafSspz void OPENSSL_cpuid_setup(void)
297fa80bafSspz {
307fa80bafSspz }
317fa80bafSspz
OPENSSL_rdtsc(void)32e7ccb6d1Schristos uint32_t OPENSSL_rdtsc(void)
337fa80bafSspz {
347fa80bafSspz return 0;
357fa80bafSspz }
367fa80bafSspz #else
3713ce5681Schristos static sigset_t all_masked;
3813ce5681Schristos
3913ce5681Schristos static sigjmp_buf ill_jmp;
ill_handler(int sig)40d572d25fSspz static void ill_handler(int sig)
41d572d25fSspz {
42d572d25fSspz siglongjmp(ill_jmp, sig);
43d572d25fSspz }
4413ce5681Schristos
4513ce5681Schristos /*
4613ce5681Schristos * Following subroutines could have been inlined, but it's not all
4713ce5681Schristos * ARM compilers support inline assembler...
4813ce5681Schristos */
4913ce5681Schristos void _armv7_neon_probe(void);
507fa80bafSspz void _armv8_aes_probe(void);
517fa80bafSspz void _armv8_sha1_probe(void);
527fa80bafSspz void _armv8_sha256_probe(void);
537fa80bafSspz void _armv8_pmull_probe(void);
54e7ccb6d1Schristos # ifdef __aarch64__
55e7ccb6d1Schristos void _armv8_sha512_probe(void);
56*5b10f583Schristos unsigned int _armv8_cpuid_probe(void);
57e7ccb6d1Schristos # endif
58e7ccb6d1Schristos uint32_t _armv7_tick(void);
5913ce5681Schristos
OPENSSL_rdtsc(void)60e7ccb6d1Schristos uint32_t OPENSSL_rdtsc(void)
6113ce5681Schristos {
6267270043Schristos if (OPENSSL_armcap_P & ARMV7_TICK)
6313ce5681Schristos return _armv7_tick();
6413ce5681Schristos else
6513ce5681Schristos return 0;
6613ce5681Schristos }
6713ce5681Schristos
684ec2eca9Schristos # if defined(__GNUC__) && __GNUC__>=2
694ec2eca9Schristos void OPENSSL_cpuid_setup(void) __attribute__ ((constructor));
704ec2eca9Schristos # endif
7192d83a12Schristos
7292d83a12Schristos # if defined(__GLIBC__) && defined(__GLIBC_PREREQ)
7392d83a12Schristos # if __GLIBC_PREREQ(2, 16)
7492d83a12Schristos # include <sys/auxv.h>
7592d83a12Schristos # define OSSL_IMPLEMENT_GETAUXVAL
7692d83a12Schristos # endif
77e1f0537eSchristos # elif defined(__ANDROID_API__)
78e1f0537eSchristos /* see https://developer.android.google.cn/ndk/guides/cpu-features */
79e1f0537eSchristos # if __ANDROID_API__ >= 18
80e1f0537eSchristos # include <sys/auxv.h>
81e1f0537eSchristos # define OSSL_IMPLEMENT_GETAUXVAL
82e1f0537eSchristos # endif
8313ce5681Schristos # endif
84856fa3afSchristos # if defined(__FreeBSD__)
85856fa3afSchristos # include <sys/param.h>
86856fa3afSchristos # if __FreeBSD_version >= 1200000
87856fa3afSchristos # include <sys/auxv.h>
88856fa3afSchristos # define OSSL_IMPLEMENT_GETAUXVAL
89856fa3afSchristos
getauxval(unsigned long key)90856fa3afSchristos static unsigned long getauxval(unsigned long key)
91856fa3afSchristos {
92856fa3afSchristos unsigned long val = 0ul;
93856fa3afSchristos
94856fa3afSchristos if (elf_aux_info((int)key, &val, sizeof(val)) != 0)
95856fa3afSchristos return 0ul;
96856fa3afSchristos
97856fa3afSchristos return val;
98856fa3afSchristos }
99856fa3afSchristos # endif
100856fa3afSchristos # endif
1017fa80bafSspz
1027fa80bafSspz /*
103e1f0537eSchristos * Android: according to https://developer.android.com/ndk/guides/cpu-features,
104e1f0537eSchristos * getauxval is supported starting with API level 18
105e1f0537eSchristos */
106e1f0537eSchristos # if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 18
107e1f0537eSchristos # include <sys/auxv.h>
108e1f0537eSchristos # define OSSL_IMPLEMENT_GETAUXVAL
109e1f0537eSchristos # endif
110e1f0537eSchristos
111e1f0537eSchristos /*
1124ec2eca9Schristos * ARM puts the feature bits for Crypto Extensions in AT_HWCAP2, whereas
1137fa80bafSspz * AArch64 used AT_HWCAP.
1147fa80bafSspz */
115e1f0537eSchristos # ifndef AT_HWCAP
116e1f0537eSchristos # define AT_HWCAP 16
117e1f0537eSchristos # endif
118e1f0537eSchristos # ifndef AT_HWCAP2
119e1f0537eSchristos # define AT_HWCAP2 26
120e1f0537eSchristos # endif
1217fa80bafSspz # if defined(__arm__) || defined (__arm)
122e1f0537eSchristos # define HWCAP AT_HWCAP
1237fa80bafSspz # define HWCAP_NEON (1 << 12)
1247fa80bafSspz
125e1f0537eSchristos # define HWCAP_CE AT_HWCAP2
1267fa80bafSspz # define HWCAP_CE_AES (1 << 0)
1277fa80bafSspz # define HWCAP_CE_PMULL (1 << 1)
1287fa80bafSspz # define HWCAP_CE_SHA1 (1 << 2)
1297fa80bafSspz # define HWCAP_CE_SHA256 (1 << 3)
1307fa80bafSspz # elif defined(__aarch64__)
131e1f0537eSchristos # define HWCAP AT_HWCAP
1327fa80bafSspz # define HWCAP_NEON (1 << 1)
1337fa80bafSspz
1347fa80bafSspz # define HWCAP_CE HWCAP
1357fa80bafSspz # define HWCAP_CE_AES (1 << 3)
1367fa80bafSspz # define HWCAP_CE_PMULL (1 << 4)
1377fa80bafSspz # define HWCAP_CE_SHA1 (1 << 5)
1387fa80bafSspz # define HWCAP_CE_SHA256 (1 << 6)
139*5b10f583Schristos # define HWCAP_CPUID (1 << 11)
140e7ccb6d1Schristos # define HWCAP_CE_SHA512 (1 << 21)
1417fa80bafSspz # endif
1427fa80bafSspz
OPENSSL_cpuid_setup(void)14313ce5681Schristos void OPENSSL_cpuid_setup(void)
14413ce5681Schristos {
145e7ccb6d1Schristos const char *e;
14613ce5681Schristos struct sigaction ill_oact, ill_act;
14713ce5681Schristos sigset_t oset;
14813ce5681Schristos static int trigger = 0;
14913ce5681Schristos
150d572d25fSspz if (trigger)
151d572d25fSspz return;
15213ce5681Schristos trigger = 1;
15313ce5681Schristos
154*5b10f583Schristos OPENSSL_armcap_P = 0;
155*5b10f583Schristos
156d572d25fSspz if ((e = getenv("OPENSSL_armcap"))) {
1577fa80bafSspz OPENSSL_armcap_P = (unsigned int)strtoul(e, NULL, 0);
15813ce5681Schristos return;
15913ce5681Schristos }
16013ce5681Schristos
161*5b10f583Schristos # if defined(__APPLE__)
162*5b10f583Schristos # if !defined(__aarch64__)
1634ec2eca9Schristos /*
1644ec2eca9Schristos * Capability probing by catching SIGILL appears to be problematic
1654ec2eca9Schristos * on iOS. But since Apple universe is "monocultural", it's actually
1664ec2eca9Schristos * possible to simply set pre-defined processor capability mask.
1674ec2eca9Schristos */
1684ec2eca9Schristos if (1) {
1694ec2eca9Schristos OPENSSL_armcap_P = ARMV7_NEON;
1704ec2eca9Schristos return;
1714ec2eca9Schristos }
1724ec2eca9Schristos /*
1734ec2eca9Schristos * One could do same even for __aarch64__ iOS builds. It's not done
1744ec2eca9Schristos * exclusively for reasons of keeping code unified across platforms.
1754ec2eca9Schristos * Unified code works because it never triggers SIGILL on Apple
1764ec2eca9Schristos * devices...
1774ec2eca9Schristos */
178*5b10f583Schristos # else
179*5b10f583Schristos {
180*5b10f583Schristos unsigned int sha512;
181*5b10f583Schristos size_t len = sizeof(sha512);
1824ec2eca9Schristos
183*5b10f583Schristos if (sysctlbyname("hw.optional.armv8_2_sha512", &sha512, &len, NULL, 0) == 0 && sha512 == 1)
184*5b10f583Schristos OPENSSL_armcap_P |= ARMV8_SHA512;
185*5b10f583Schristos }
186*5b10f583Schristos # endif
187*5b10f583Schristos # endif
18813ce5681Schristos
18992d83a12Schristos # ifdef OSSL_IMPLEMENT_GETAUXVAL
1907fa80bafSspz if (getauxval(HWCAP) & HWCAP_NEON) {
1917fa80bafSspz unsigned long hwcap = getauxval(HWCAP_CE);
1927fa80bafSspz
1937fa80bafSspz OPENSSL_armcap_P |= ARMV7_NEON;
1947fa80bafSspz
1957fa80bafSspz if (hwcap & HWCAP_CE_AES)
1967fa80bafSspz OPENSSL_armcap_P |= ARMV8_AES;
1977fa80bafSspz
1987fa80bafSspz if (hwcap & HWCAP_CE_PMULL)
1997fa80bafSspz OPENSSL_armcap_P |= ARMV8_PMULL;
2007fa80bafSspz
2017fa80bafSspz if (hwcap & HWCAP_CE_SHA1)
2027fa80bafSspz OPENSSL_armcap_P |= ARMV8_SHA1;
2037fa80bafSspz
2047fa80bafSspz if (hwcap & HWCAP_CE_SHA256)
2057fa80bafSspz OPENSSL_armcap_P |= ARMV8_SHA256;
206e7ccb6d1Schristos
207e7ccb6d1Schristos # ifdef __aarch64__
208e7ccb6d1Schristos if (hwcap & HWCAP_CE_SHA512)
209e7ccb6d1Schristos OPENSSL_armcap_P |= ARMV8_SHA512;
210*5b10f583Schristos
211*5b10f583Schristos if (hwcap & HWCAP_CPUID)
212*5b10f583Schristos OPENSSL_armcap_P |= ARMV8_CPUID;
213e7ccb6d1Schristos # endif
2147fa80bafSspz }
21592d83a12Schristos # endif
21692d83a12Schristos
21792d83a12Schristos sigfillset(&all_masked);
21892d83a12Schristos sigdelset(&all_masked, SIGILL);
21992d83a12Schristos sigdelset(&all_masked, SIGTRAP);
22092d83a12Schristos sigdelset(&all_masked, SIGFPE);
22192d83a12Schristos sigdelset(&all_masked, SIGBUS);
22292d83a12Schristos sigdelset(&all_masked, SIGSEGV);
22392d83a12Schristos
22492d83a12Schristos memset(&ill_act, 0, sizeof(ill_act));
22592d83a12Schristos ill_act.sa_handler = ill_handler;
22692d83a12Schristos ill_act.sa_mask = all_masked;
22792d83a12Schristos
22892d83a12Schristos sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset);
22992d83a12Schristos sigaction(SIGILL, &ill_act, &ill_oact);
23092d83a12Schristos
23192d83a12Schristos /* If we used getauxval, we already have all the values */
23292d83a12Schristos # ifndef OSSL_IMPLEMENT_GETAUXVAL
23392d83a12Schristos if (sigsetjmp(ill_jmp, 1) == 0) {
23413ce5681Schristos _armv7_neon_probe();
23513ce5681Schristos OPENSSL_armcap_P |= ARMV7_NEON;
2367fa80bafSspz if (sigsetjmp(ill_jmp, 1) == 0) {
2377fa80bafSspz _armv8_pmull_probe();
2387fa80bafSspz OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES;
2397fa80bafSspz } else if (sigsetjmp(ill_jmp, 1) == 0) {
2407fa80bafSspz _armv8_aes_probe();
2417fa80bafSspz OPENSSL_armcap_P |= ARMV8_AES;
2427fa80bafSspz }
2437fa80bafSspz if (sigsetjmp(ill_jmp, 1) == 0) {
2447fa80bafSspz _armv8_sha1_probe();
2457fa80bafSspz OPENSSL_armcap_P |= ARMV8_SHA1;
2467fa80bafSspz }
2477fa80bafSspz if (sigsetjmp(ill_jmp, 1) == 0) {
2487fa80bafSspz _armv8_sha256_probe();
2497fa80bafSspz OPENSSL_armcap_P |= ARMV8_SHA256;
2507fa80bafSspz }
251e7ccb6d1Schristos # if defined(__aarch64__) && !defined(__APPLE__)
252e7ccb6d1Schristos if (sigsetjmp(ill_jmp, 1) == 0) {
253e7ccb6d1Schristos _armv8_sha512_probe();
254e7ccb6d1Schristos OPENSSL_armcap_P |= ARMV8_SHA512;
255e7ccb6d1Schristos }
256e7ccb6d1Schristos # endif
25713ce5681Schristos }
25892d83a12Schristos # endif
25992d83a12Schristos
260*5b10f583Schristos /*
261*5b10f583Schristos * Probing for ARMV7_TICK is known to produce unreliable results,
262*5b10f583Schristos * so we will only use the feature when the user explicitly enables
263*5b10f583Schristos * it with OPENSSL_armcap.
264*5b10f583Schristos */
26513ce5681Schristos
26613ce5681Schristos sigaction(SIGILL, &ill_oact, NULL);
26713ce5681Schristos sigprocmask(SIG_SETMASK, &oset, NULL);
268*5b10f583Schristos
269*5b10f583Schristos # ifdef __aarch64__
270*5b10f583Schristos if (OPENSSL_armcap_P & ARMV8_CPUID)
271*5b10f583Schristos OPENSSL_arm_midr = _armv8_cpuid_probe();
272*5b10f583Schristos
273*5b10f583Schristos if ((MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) ||
274*5b10f583Schristos MIDR_IS_CPU_MODEL(OPENSSL_arm_midr, ARM_CPU_IMP_ARM, ARM_CPU_PART_N1)) &&
275*5b10f583Schristos (OPENSSL_armcap_P & ARMV7_NEON)) {
276*5b10f583Schristos OPENSSL_armv8_rsa_neonized = 1;
277*5b10f583Schristos }
278*5b10f583Schristos # endif
27913ce5681Schristos }
2807fa80bafSspz #endif
281