1 /* 2 * Copyright 2009-2021 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the OpenSSL license (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <string.h> 13 #include <setjmp.h> 14 #include <signal.h> 15 #include <unistd.h> 16 #if defined(__linux) || defined(_AIX) 17 # include <sys/utsname.h> 18 #endif 19 #if defined(_AIX53) /* defined even on post-5.3 */ 20 # include <sys/systemcfg.h> 21 # if !defined(__power_set) 22 # define __power_set(a) (_system_configuration.implementation & (a)) 23 # endif 24 #endif 25 #if defined(__APPLE__) && defined(__MACH__) 26 # include <sys/types.h> 27 # include <sys/sysctl.h> 28 #endif 29 #include <openssl/crypto.h> 30 #include <openssl/bn.h> 31 #include <internal/cryptlib.h> 32 #include <crypto/chacha.h> 33 #include "bn/bn_local.h" 34 35 #include "ppc_arch.h" 36 37 unsigned int OPENSSL_ppccap_P = 0; 38 39 static sigset_t all_masked; 40 41 #ifdef OPENSSL_BN_ASM_MONT 42 int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 43 const BN_ULONG *np, const BN_ULONG *n0, int num) 44 { 45 int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 46 const BN_ULONG *np, const BN_ULONG *n0, int num); 47 int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 48 const BN_ULONG *np, const BN_ULONG *n0, int num); 49 50 if (num < 4) 51 return 0; 52 53 if ((num & 3) == 0) 54 return bn_mul4x_mont_int(rp, ap, bp, np, n0, num); 55 56 /* 57 * There used to be [optional] call to bn_mul_mont_fpu64 here, 58 * but above subroutine is faster on contemporary processors. 59 * Formulation means that there might be old processors where 60 * FPU code path would be faster, POWER6 perhaps, but there was 61 * no opportunity to figure it out... 62 */ 63 64 return bn_mul_mont_int(rp, ap, bp, np, n0, num); 65 } 66 #endif 67 68 void sha256_block_p8(void *ctx, const void *inp, size_t len); 69 void sha256_block_ppc(void *ctx, const void *inp, size_t len); 70 void sha256_block_data_order(void *ctx, const void *inp, size_t len); 71 void sha256_block_data_order(void *ctx, const void *inp, size_t len) 72 { 73 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) : 74 sha256_block_ppc(ctx, inp, len); 75 } 76 77 void sha512_block_p8(void *ctx, const void *inp, size_t len); 78 void sha512_block_ppc(void *ctx, const void *inp, size_t len); 79 void sha512_block_data_order(void *ctx, const void *inp, size_t len); 80 void sha512_block_data_order(void *ctx, const void *inp, size_t len) 81 { 82 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) : 83 sha512_block_ppc(ctx, inp, len); 84 } 85 86 #ifndef OPENSSL_NO_CHACHA 87 void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp, 88 size_t len, const unsigned int key[8], 89 const unsigned int counter[4]); 90 void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp, 91 size_t len, const unsigned int key[8], 92 const unsigned int counter[4]); 93 void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp, 94 size_t len, const unsigned int key[8], 95 const unsigned int counter[4]); 96 void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, 97 size_t len, const unsigned int key[8], 98 const unsigned int counter[4]) 99 { 100 OPENSSL_ppccap_P & PPC_CRYPTO207 101 ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) 102 : OPENSSL_ppccap_P & PPC_ALTIVEC 103 ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) 104 : ChaCha20_ctr32_int(out, inp, len, key, counter); 105 } 106 #endif 107 108 #ifndef OPENSSL_NO_POLY1305 109 void poly1305_init_int(void *ctx, const unsigned char key[16]); 110 void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, 111 unsigned int padbit); 112 void poly1305_emit(void *ctx, unsigned char mac[16], 113 const unsigned int nonce[4]); 114 void poly1305_init_fpu(void *ctx, const unsigned char key[16]); 115 void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len, 116 unsigned int padbit); 117 void poly1305_emit_fpu(void *ctx, unsigned char mac[16], 118 const unsigned int nonce[4]); 119 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]); 120 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]) 121 { 122 if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) { 123 poly1305_init_fpu(ctx, key); 124 func[0] = (void*)(uintptr_t)poly1305_blocks_fpu; 125 func[1] = (void*)(uintptr_t)poly1305_emit_fpu; 126 } else { 127 poly1305_init_int(ctx, key); 128 func[0] = (void*)(uintptr_t)poly1305_blocks; 129 func[1] = (void*)(uintptr_t)poly1305_emit; 130 } 131 return 1; 132 } 133 #endif 134 135 #ifdef ECP_NISTZ256_ASM 136 void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4], 137 const unsigned long b[4]); 138 139 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]); 140 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]) 141 { 142 static const unsigned long RR[] = { 0x0000000000000003U, 143 0xfffffffbffffffffU, 144 0xfffffffffffffffeU, 145 0x00000004fffffffdU }; 146 147 ecp_nistz256_mul_mont(res, in, RR); 148 } 149 150 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]); 151 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]) 152 { 153 static const unsigned long one[] = { 1, 0, 0, 0 }; 154 155 ecp_nistz256_mul_mont(res, in, one); 156 } 157 #endif 158 159 static sigjmp_buf ill_jmp; 160 static void ill_handler(int sig) 161 { 162 siglongjmp(ill_jmp, sig); 163 } 164 165 void OPENSSL_fpu_probe(void); 166 void OPENSSL_ppc64_probe(void); 167 void OPENSSL_altivec_probe(void); 168 void OPENSSL_crypto207_probe(void); 169 void OPENSSL_madd300_probe(void); 170 171 long OPENSSL_rdtsc_mftb(void); 172 long OPENSSL_rdtsc_mfspr268(void); 173 174 uint32_t OPENSSL_rdtsc(void) 175 { 176 if (OPENSSL_ppccap_P & PPC_MFTB) 177 return OPENSSL_rdtsc_mftb(); 178 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 179 return OPENSSL_rdtsc_mfspr268(); 180 else 181 return 0; 182 } 183 184 size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t); 185 size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t); 186 187 size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) 188 { 189 if (OPENSSL_ppccap_P & PPC_MFTB) 190 return OPENSSL_instrument_bus_mftb(out, cnt); 191 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 192 return OPENSSL_instrument_bus_mfspr268(out, cnt); 193 else 194 return 0; 195 } 196 197 size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t); 198 size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t); 199 200 size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) 201 { 202 if (OPENSSL_ppccap_P & PPC_MFTB) 203 return OPENSSL_instrument_bus2_mftb(out, cnt, max); 204 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 205 return OPENSSL_instrument_bus2_mfspr268(out, cnt, max); 206 else 207 return 0; 208 } 209 210 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) 211 # if __GLIBC_PREREQ(2, 16) 212 # include <sys/auxv.h> 213 # define OSSL_IMPLEMENT_GETAUXVAL 214 # elif defined(__ANDROID_API__) 215 /* see https://developer.android.google.cn/ndk/guides/cpu-features */ 216 # if __ANDROID_API__ >= 18 217 # include <sys/auxv.h> 218 # define OSSL_IMPLEMENT_GETAUXVAL 219 # endif 220 # endif 221 #endif 222 223 #if defined(__FreeBSD__) 224 # include <sys/param.h> 225 # if __FreeBSD_version >= 1200000 226 # include <sys/auxv.h> 227 # define OSSL_IMPLEMENT_GETAUXVAL 228 229 static unsigned long getauxval(unsigned long key) 230 { 231 unsigned long val = 0ul; 232 233 if (elf_aux_info((int)key, &val, sizeof(val)) != 0) 234 return 0ul; 235 236 return val; 237 } 238 # endif 239 #endif 240 241 /* I wish <sys/auxv.h> was universally available */ 242 #ifndef AT_HWCAP 243 # define AT_HWCAP 16 /* AT_HWCAP */ 244 #endif 245 #define HWCAP_PPC64 (1U << 30) 246 #define HWCAP_ALTIVEC (1U << 28) 247 #define HWCAP_FPU (1U << 27) 248 #define HWCAP_POWER6_EXT (1U << 9) 249 #define HWCAP_VSX (1U << 7) 250 251 #ifndef AT_HWCAP2 252 # define AT_HWCAP2 26 /* AT_HWCAP2 */ 253 #endif 254 #define HWCAP_VEC_CRYPTO (1U << 25) 255 #define HWCAP_ARCH_3_00 (1U << 23) 256 257 # if defined(__GNUC__) && __GNUC__>=2 258 __attribute__ ((constructor)) 259 # endif 260 void OPENSSL_cpuid_setup(void) 261 { 262 char *e; 263 struct sigaction ill_oact, ill_act; 264 sigset_t oset; 265 static int trigger = 0; 266 267 if (trigger) 268 return; 269 trigger = 1; 270 271 if ((e = getenv("OPENSSL_ppccap"))) { 272 OPENSSL_ppccap_P = strtoul(e, NULL, 0); 273 return; 274 } 275 276 OPENSSL_ppccap_P = 0; 277 278 #if defined(_AIX) 279 OPENSSL_ppccap_P |= PPC_FPU; 280 281 if (sizeof(size_t) == 4) { 282 struct utsname uts; 283 # if defined(_SC_AIX_KERNEL_BITMODE) 284 if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64) 285 return; 286 # endif 287 if (uname(&uts) != 0 || atoi(uts.version) < 6) 288 return; 289 } 290 291 # if defined(__power_set) 292 /* 293 * Value used in __power_set is a single-bit 1<<n one denoting 294 * specific processor class. Incidentally 0xffffffff<<n can be 295 * used to denote specific processor and its successors. 296 */ 297 if (sizeof(size_t) == 4) { 298 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 299 if (__power_set(0xffffffffU<<13)) /* POWER5 and later */ 300 OPENSSL_ppccap_P |= PPC_FPU64; 301 } else { 302 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 303 if (__power_set(0x1U<<14)) /* POWER6 */ 304 OPENSSL_ppccap_P |= PPC_FPU64; 305 } 306 307 if (__power_set(0xffffffffU<<14)) /* POWER6 and later */ 308 OPENSSL_ppccap_P |= PPC_ALTIVEC; 309 310 if (__power_set(0xffffffffU<<16)) /* POWER8 and later */ 311 OPENSSL_ppccap_P |= PPC_CRYPTO207; 312 313 if (__power_set(0xffffffffU<<17)) /* POWER9 and later */ 314 OPENSSL_ppccap_P |= PPC_MADD300; 315 316 return; 317 # endif 318 #endif 319 320 #if defined(__APPLE__) && defined(__MACH__) 321 OPENSSL_ppccap_P |= PPC_FPU; 322 323 { 324 int val; 325 size_t len = sizeof(val); 326 327 if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) { 328 if (val) 329 OPENSSL_ppccap_P |= PPC_FPU64; 330 } 331 332 len = sizeof(val); 333 if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) { 334 if (val) 335 OPENSSL_ppccap_P |= PPC_ALTIVEC; 336 } 337 338 return; 339 } 340 #endif 341 342 #ifdef OSSL_IMPLEMENT_GETAUXVAL 343 { 344 unsigned long hwcap = getauxval(AT_HWCAP); 345 unsigned long hwcap2 = getauxval(AT_HWCAP2); 346 347 if (hwcap & HWCAP_FPU) { 348 OPENSSL_ppccap_P |= PPC_FPU; 349 350 if (sizeof(size_t) == 4) { 351 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 352 if (hwcap & HWCAP_PPC64) 353 OPENSSL_ppccap_P |= PPC_FPU64; 354 } else { 355 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 356 if (hwcap & HWCAP_POWER6_EXT) 357 OPENSSL_ppccap_P |= PPC_FPU64; 358 } 359 } 360 361 if (hwcap & HWCAP_ALTIVEC) { 362 OPENSSL_ppccap_P |= PPC_ALTIVEC; 363 364 if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO)) 365 OPENSSL_ppccap_P |= PPC_CRYPTO207; 366 } 367 368 if (hwcap2 & HWCAP_ARCH_3_00) { 369 OPENSSL_ppccap_P |= PPC_MADD300; 370 } 371 } 372 #endif 373 374 sigfillset(&all_masked); 375 sigdelset(&all_masked, SIGILL); 376 sigdelset(&all_masked, SIGTRAP); 377 #ifdef SIGEMT 378 sigdelset(&all_masked, SIGEMT); 379 #endif 380 sigdelset(&all_masked, SIGFPE); 381 sigdelset(&all_masked, SIGBUS); 382 sigdelset(&all_masked, SIGSEGV); 383 384 memset(&ill_act, 0, sizeof(ill_act)); 385 ill_act.sa_handler = ill_handler; 386 ill_act.sa_mask = all_masked; 387 388 sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); 389 sigaction(SIGILL, &ill_act, &ill_oact); 390 391 #ifndef OSSL_IMPLEMENT_GETAUXVAL 392 if (sigsetjmp(ill_jmp,1) == 0) { 393 OPENSSL_fpu_probe(); 394 OPENSSL_ppccap_P |= PPC_FPU; 395 396 if (sizeof(size_t) == 4) { 397 # ifdef __linux 398 struct utsname uts; 399 if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0) 400 # endif 401 if (sigsetjmp(ill_jmp, 1) == 0) { 402 OPENSSL_ppc64_probe(); 403 OPENSSL_ppccap_P |= PPC_FPU64; 404 } 405 } else { 406 /* 407 * Wanted code detecting POWER6 CPU and setting PPC_FPU64 408 */ 409 } 410 } 411 412 if (sigsetjmp(ill_jmp, 1) == 0) { 413 OPENSSL_altivec_probe(); 414 OPENSSL_ppccap_P |= PPC_ALTIVEC; 415 if (sigsetjmp(ill_jmp, 1) == 0) { 416 OPENSSL_crypto207_probe(); 417 OPENSSL_ppccap_P |= PPC_CRYPTO207; 418 } 419 } 420 421 if (sigsetjmp(ill_jmp, 1) == 0) { 422 OPENSSL_madd300_probe(); 423 OPENSSL_ppccap_P |= PPC_MADD300; 424 } 425 #endif 426 427 if (sigsetjmp(ill_jmp, 1) == 0) { 428 OPENSSL_rdtsc_mftb(); 429 OPENSSL_ppccap_P |= PPC_MFTB; 430 } else if (sigsetjmp(ill_jmp, 1) == 0) { 431 OPENSSL_rdtsc_mfspr268(); 432 OPENSSL_ppccap_P |= PPC_MFSPR268; 433 } 434 435 sigaction(SIGILL, &ill_oact, NULL); 436 sigprocmask(SIG_SETMASK, &oset, NULL); 437 } 438