1 /* 2 * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the OpenSSL license (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <string.h> 13 #include <setjmp.h> 14 #include <signal.h> 15 #include <unistd.h> 16 #if defined(__linux) || defined(_AIX) 17 # include <sys/utsname.h> 18 #endif 19 #if defined(_AIX53) /* defined even on post-5.3 */ 20 # include <sys/systemcfg.h> 21 # if !defined(__power_set) 22 # define __power_set(a) (_system_configuration.implementation & (a)) 23 # endif 24 #endif 25 #if defined(__APPLE__) && defined(__MACH__) 26 # include <sys/types.h> 27 # include <sys/sysctl.h> 28 #endif 29 #include <openssl/crypto.h> 30 #include <openssl/bn.h> 31 #include <internal/cryptlib.h> 32 #include <crypto/chacha.h> 33 #include "bn/bn_local.h" 34 35 #include "ppc_arch.h" 36 37 unsigned int OPENSSL_ppccap_P = 0; 38 39 static sigset_t all_masked; 40 41 #ifdef OPENSSL_BN_ASM_MONT 42 int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 43 const BN_ULONG *np, const BN_ULONG *n0, int num) 44 { 45 int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 46 const BN_ULONG *np, const BN_ULONG *n0, int num); 47 int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 48 const BN_ULONG *np, const BN_ULONG *n0, int num); 49 50 if (num < 4) 51 return 0; 52 53 if ((num & 3) == 0) 54 return bn_mul4x_mont_int(rp, ap, bp, np, n0, num); 55 56 /* 57 * There used to be [optional] call to bn_mul_mont_fpu64 here, 58 * but above subroutine is faster on contemporary processors. 59 * Formulation means that there might be old processors where 60 * FPU code path would be faster, POWER6 perhaps, but there was 61 * no opportunity to figure it out... 62 */ 63 64 return bn_mul_mont_int(rp, ap, bp, np, n0, num); 65 } 66 #endif 67 68 void sha256_block_p8(void *ctx, const void *inp, size_t len); 69 void sha256_block_ppc(void *ctx, const void *inp, size_t len); 70 void sha256_block_data_order(void *ctx, const void *inp, size_t len); 71 void sha256_block_data_order(void *ctx, const void *inp, size_t len) 72 { 73 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) : 74 sha256_block_ppc(ctx, inp, len); 75 } 76 77 void sha512_block_p8(void *ctx, const void *inp, size_t len); 78 void sha512_block_ppc(void *ctx, const void *inp, size_t len); 79 void sha512_block_data_order(void *ctx, const void *inp, size_t len); 80 void sha512_block_data_order(void *ctx, const void *inp, size_t len) 81 { 82 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) : 83 sha512_block_ppc(ctx, inp, len); 84 } 85 86 #ifndef OPENSSL_NO_CHACHA 87 void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp, 88 size_t len, const unsigned int key[8], 89 const unsigned int counter[4]); 90 void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp, 91 size_t len, const unsigned int key[8], 92 const unsigned int counter[4]); 93 void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp, 94 size_t len, const unsigned int key[8], 95 const unsigned int counter[4]); 96 void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, 97 size_t len, const unsigned int key[8], 98 const unsigned int counter[4]) 99 { 100 OPENSSL_ppccap_P & PPC_CRYPTO207 101 ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) 102 : OPENSSL_ppccap_P & PPC_ALTIVEC 103 ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) 104 : ChaCha20_ctr32_int(out, inp, len, key, counter); 105 } 106 #endif 107 108 #ifndef OPENSSL_NO_POLY1305 109 void poly1305_init_int(void *ctx, const unsigned char key[16]); 110 void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, 111 unsigned int padbit); 112 void poly1305_emit(void *ctx, unsigned char mac[16], 113 const unsigned int nonce[4]); 114 void poly1305_init_fpu(void *ctx, const unsigned char key[16]); 115 void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len, 116 unsigned int padbit); 117 void poly1305_emit_fpu(void *ctx, unsigned char mac[16], 118 const unsigned int nonce[4]); 119 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]); 120 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]) 121 { 122 if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) { 123 poly1305_init_fpu(ctx, key); 124 func[0] = (void*)(uintptr_t)poly1305_blocks_fpu; 125 func[1] = (void*)(uintptr_t)poly1305_emit_fpu; 126 } else { 127 poly1305_init_int(ctx, key); 128 func[0] = (void*)(uintptr_t)poly1305_blocks; 129 func[1] = (void*)(uintptr_t)poly1305_emit; 130 } 131 return 1; 132 } 133 #endif 134 135 #ifdef ECP_NISTZ256_ASM 136 void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4], 137 const unsigned long b[4]); 138 139 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]); 140 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]) 141 { 142 static const unsigned long RR[] = { 0x0000000000000003U, 143 0xfffffffbffffffffU, 144 0xfffffffffffffffeU, 145 0x00000004fffffffdU }; 146 147 ecp_nistz256_mul_mont(res, in, RR); 148 } 149 150 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]); 151 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]) 152 { 153 static const unsigned long one[] = { 1, 0, 0, 0 }; 154 155 ecp_nistz256_mul_mont(res, in, one); 156 } 157 #endif 158 159 static sigjmp_buf ill_jmp; 160 static void ill_handler(int sig) 161 { 162 siglongjmp(ill_jmp, sig); 163 } 164 165 void OPENSSL_fpu_probe(void); 166 void OPENSSL_ppc64_probe(void); 167 void OPENSSL_altivec_probe(void); 168 void OPENSSL_crypto207_probe(void); 169 void OPENSSL_madd300_probe(void); 170 171 long OPENSSL_rdtsc_mftb(void); 172 long OPENSSL_rdtsc_mfspr268(void); 173 174 uint32_t OPENSSL_rdtsc(void) 175 { 176 if (OPENSSL_ppccap_P & PPC_MFTB) 177 return OPENSSL_rdtsc_mftb(); 178 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 179 return OPENSSL_rdtsc_mfspr268(); 180 else 181 return 0; 182 } 183 184 size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t); 185 size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t); 186 187 size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) 188 { 189 if (OPENSSL_ppccap_P & PPC_MFTB) 190 return OPENSSL_instrument_bus_mftb(out, cnt); 191 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 192 return OPENSSL_instrument_bus_mfspr268(out, cnt); 193 else 194 return 0; 195 } 196 197 size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t); 198 size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t); 199 200 size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) 201 { 202 if (OPENSSL_ppccap_P & PPC_MFTB) 203 return OPENSSL_instrument_bus2_mftb(out, cnt, max); 204 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 205 return OPENSSL_instrument_bus2_mfspr268(out, cnt, max); 206 else 207 return 0; 208 } 209 210 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) 211 # if __GLIBC_PREREQ(2, 16) 212 # include <sys/auxv.h> 213 # define OSSL_IMPLEMENT_GETAUXVAL 214 # elif defined(__ANDROID_API__) 215 /* see https://developer.android.google.cn/ndk/guides/cpu-features */ 216 # if __ANDROID_API__ >= 18 217 # include <sys/auxv.h> 218 # define OSSL_IMPLEMENT_GETAUXVAL 219 # endif 220 # endif 221 #endif 222 223 #if defined(__FreeBSD__) 224 # include <sys/param.h> 225 # if __FreeBSD_version >= 1200000 226 # include <sys/auxv.h> 227 # define OSSL_IMPLEMENT_GETAUXVAL 228 229 static unsigned long getauxval(unsigned long key) 230 { 231 unsigned long val = 0ul; 232 233 if (elf_aux_info((int)key, &val, sizeof(val)) != 0) 234 return 0ul; 235 236 return val; 237 } 238 # endif 239 #endif 240 241 /* I wish <sys/auxv.h> was universally available */ 242 #define HWCAP 16 /* AT_HWCAP */ 243 #define HWCAP_PPC64 (1U << 30) 244 #define HWCAP_ALTIVEC (1U << 28) 245 #define HWCAP_FPU (1U << 27) 246 #define HWCAP_POWER6_EXT (1U << 9) 247 #define HWCAP_VSX (1U << 7) 248 249 #define HWCAP2 26 /* AT_HWCAP2 */ 250 #define HWCAP_VEC_CRYPTO (1U << 25) 251 #define HWCAP_ARCH_3_00 (1U << 23) 252 253 # if defined(__GNUC__) && __GNUC__>=2 254 __attribute__ ((constructor)) 255 # endif 256 void OPENSSL_cpuid_setup(void) 257 { 258 char *e; 259 struct sigaction ill_oact, ill_act; 260 sigset_t oset; 261 static int trigger = 0; 262 263 if (trigger) 264 return; 265 trigger = 1; 266 267 if ((e = getenv("OPENSSL_ppccap"))) { 268 OPENSSL_ppccap_P = strtoul(e, NULL, 0); 269 return; 270 } 271 272 OPENSSL_ppccap_P = 0; 273 274 #if defined(_AIX) 275 OPENSSL_ppccap_P |= PPC_FPU; 276 277 if (sizeof(size_t) == 4) { 278 struct utsname uts; 279 # if defined(_SC_AIX_KERNEL_BITMODE) 280 if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64) 281 return; 282 # endif 283 if (uname(&uts) != 0 || atoi(uts.version) < 6) 284 return; 285 } 286 287 # if defined(__power_set) 288 /* 289 * Value used in __power_set is a single-bit 1<<n one denoting 290 * specific processor class. Incidentally 0xffffffff<<n can be 291 * used to denote specific processor and its successors. 292 */ 293 if (sizeof(size_t) == 4) { 294 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 295 if (__power_set(0xffffffffU<<13)) /* POWER5 and later */ 296 OPENSSL_ppccap_P |= PPC_FPU64; 297 } else { 298 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 299 if (__power_set(0x1U<<14)) /* POWER6 */ 300 OPENSSL_ppccap_P |= PPC_FPU64; 301 } 302 303 if (__power_set(0xffffffffU<<14)) /* POWER6 and later */ 304 OPENSSL_ppccap_P |= PPC_ALTIVEC; 305 306 if (__power_set(0xffffffffU<<16)) /* POWER8 and later */ 307 OPENSSL_ppccap_P |= PPC_CRYPTO207; 308 309 if (__power_set(0xffffffffU<<17)) /* POWER9 and later */ 310 OPENSSL_ppccap_P |= PPC_MADD300; 311 312 return; 313 # endif 314 #endif 315 316 #if defined(__APPLE__) && defined(__MACH__) 317 OPENSSL_ppccap_P |= PPC_FPU; 318 319 { 320 int val; 321 size_t len = sizeof(val); 322 323 if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) { 324 if (val) 325 OPENSSL_ppccap_P |= PPC_FPU64; 326 } 327 328 len = sizeof(val); 329 if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) { 330 if (val) 331 OPENSSL_ppccap_P |= PPC_ALTIVEC; 332 } 333 334 return; 335 } 336 #endif 337 338 #ifdef OSSL_IMPLEMENT_GETAUXVAL 339 { 340 unsigned long hwcap = getauxval(HWCAP); 341 unsigned long hwcap2 = getauxval(HWCAP2); 342 343 if (hwcap & HWCAP_FPU) { 344 OPENSSL_ppccap_P |= PPC_FPU; 345 346 if (sizeof(size_t) == 4) { 347 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 348 if (hwcap & HWCAP_PPC64) 349 OPENSSL_ppccap_P |= PPC_FPU64; 350 } else { 351 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 352 if (hwcap & HWCAP_POWER6_EXT) 353 OPENSSL_ppccap_P |= PPC_FPU64; 354 } 355 } 356 357 if (hwcap & HWCAP_ALTIVEC) { 358 OPENSSL_ppccap_P |= PPC_ALTIVEC; 359 360 if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO)) 361 OPENSSL_ppccap_P |= PPC_CRYPTO207; 362 } 363 364 if (hwcap2 & HWCAP_ARCH_3_00) { 365 OPENSSL_ppccap_P |= PPC_MADD300; 366 } 367 } 368 #endif 369 370 sigfillset(&all_masked); 371 sigdelset(&all_masked, SIGILL); 372 sigdelset(&all_masked, SIGTRAP); 373 #ifdef SIGEMT 374 sigdelset(&all_masked, SIGEMT); 375 #endif 376 sigdelset(&all_masked, SIGFPE); 377 sigdelset(&all_masked, SIGBUS); 378 sigdelset(&all_masked, SIGSEGV); 379 380 memset(&ill_act, 0, sizeof(ill_act)); 381 ill_act.sa_handler = ill_handler; 382 ill_act.sa_mask = all_masked; 383 384 sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); 385 sigaction(SIGILL, &ill_act, &ill_oact); 386 387 #ifndef OSSL_IMPLEMENT_GETAUXVAL 388 if (sigsetjmp(ill_jmp,1) == 0) { 389 OPENSSL_fpu_probe(); 390 OPENSSL_ppccap_P |= PPC_FPU; 391 392 if (sizeof(size_t) == 4) { 393 # ifdef __linux 394 struct utsname uts; 395 if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0) 396 # endif 397 if (sigsetjmp(ill_jmp, 1) == 0) { 398 OPENSSL_ppc64_probe(); 399 OPENSSL_ppccap_P |= PPC_FPU64; 400 } 401 } else { 402 /* 403 * Wanted code detecting POWER6 CPU and setting PPC_FPU64 404 */ 405 } 406 } 407 408 if (sigsetjmp(ill_jmp, 1) == 0) { 409 OPENSSL_altivec_probe(); 410 OPENSSL_ppccap_P |= PPC_ALTIVEC; 411 if (sigsetjmp(ill_jmp, 1) == 0) { 412 OPENSSL_crypto207_probe(); 413 OPENSSL_ppccap_P |= PPC_CRYPTO207; 414 } 415 } 416 417 if (sigsetjmp(ill_jmp, 1) == 0) { 418 OPENSSL_madd300_probe(); 419 OPENSSL_ppccap_P |= PPC_MADD300; 420 } 421 #endif 422 423 if (sigsetjmp(ill_jmp, 1) == 0) { 424 OPENSSL_rdtsc_mftb(); 425 OPENSSL_ppccap_P |= PPC_MFTB; 426 } else if (sigsetjmp(ill_jmp, 1) == 0) { 427 OPENSSL_rdtsc_mfspr268(); 428 OPENSSL_ppccap_P |= PPC_MFSPR268; 429 } 430 431 sigaction(SIGILL, &ill_oact, NULL); 432 sigprocmask(SIG_SETMASK, &oset, NULL); 433 } 434