1 /* 2 * Copyright 2009-2019 The OpenSSL Project Authors. All Rights Reserved. 3 * 4 * Licensed under the OpenSSL license (the "License"). You may not use 5 * this file except in compliance with the License. You can obtain a copy 6 * in the file LICENSE in the source distribution or at 7 * https://www.openssl.org/source/license.html 8 */ 9 10 #include <stdio.h> 11 #include <stdlib.h> 12 #include <string.h> 13 #include <setjmp.h> 14 #include <signal.h> 15 #include <unistd.h> 16 #if defined(__linux) || defined(_AIX) 17 # include <sys/utsname.h> 18 #endif 19 #if defined(_AIX53) /* defined even on post-5.3 */ 20 # include <sys/systemcfg.h> 21 # if !defined(__power_set) 22 # define __power_set(a) (_system_configuration.implementation & (a)) 23 # endif 24 #endif 25 #if defined(__APPLE__) && defined(__MACH__) 26 # include <sys/types.h> 27 # include <sys/sysctl.h> 28 #endif 29 #include <openssl/crypto.h> 30 #include <openssl/bn.h> 31 #include <internal/cryptlib.h> 32 #include <crypto/chacha.h> 33 #include "bn/bn_local.h" 34 35 #include "ppc_arch.h" 36 37 unsigned int OPENSSL_ppccap_P = 0; 38 39 static sigset_t all_masked; 40 41 #ifdef OPENSSL_BN_ASM_MONT 42 int bn_mul_mont(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 43 const BN_ULONG *np, const BN_ULONG *n0, int num) 44 { 45 int bn_mul_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 46 const BN_ULONG *np, const BN_ULONG *n0, int num); 47 int bn_mul4x_mont_int(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp, 48 const BN_ULONG *np, const BN_ULONG *n0, int num); 49 50 if (num < 4) 51 return 0; 52 53 if ((num & 3) == 0) 54 return bn_mul4x_mont_int(rp, ap, bp, np, n0, num); 55 56 /* 57 * There used to be [optional] call to bn_mul_mont_fpu64 here, 58 * but above subroutine is faster on contemporary processors. 59 * Formulation means that there might be old processors where 60 * FPU code path would be faster, POWER6 perhaps, but there was 61 * no opportunity to figure it out... 62 */ 63 64 return bn_mul_mont_int(rp, ap, bp, np, n0, num); 65 } 66 #endif 67 68 void sha256_block_p8(void *ctx, const void *inp, size_t len); 69 void sha256_block_ppc(void *ctx, const void *inp, size_t len); 70 void sha256_block_data_order(void *ctx, const void *inp, size_t len); 71 void sha256_block_data_order(void *ctx, const void *inp, size_t len) 72 { 73 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha256_block_p8(ctx, inp, len) : 74 sha256_block_ppc(ctx, inp, len); 75 } 76 77 void sha512_block_p8(void *ctx, const void *inp, size_t len); 78 void sha512_block_ppc(void *ctx, const void *inp, size_t len); 79 void sha512_block_data_order(void *ctx, const void *inp, size_t len); 80 void sha512_block_data_order(void *ctx, const void *inp, size_t len) 81 { 82 OPENSSL_ppccap_P & PPC_CRYPTO207 ? sha512_block_p8(ctx, inp, len) : 83 sha512_block_ppc(ctx, inp, len); 84 } 85 86 #ifndef OPENSSL_NO_CHACHA 87 void ChaCha20_ctr32_int(unsigned char *out, const unsigned char *inp, 88 size_t len, const unsigned int key[8], 89 const unsigned int counter[4]); 90 void ChaCha20_ctr32_vmx(unsigned char *out, const unsigned char *inp, 91 size_t len, const unsigned int key[8], 92 const unsigned int counter[4]); 93 void ChaCha20_ctr32_vsx(unsigned char *out, const unsigned char *inp, 94 size_t len, const unsigned int key[8], 95 const unsigned int counter[4]); 96 void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, 97 size_t len, const unsigned int key[8], 98 const unsigned int counter[4]) 99 { 100 OPENSSL_ppccap_P & PPC_CRYPTO207 101 ? ChaCha20_ctr32_vsx(out, inp, len, key, counter) 102 : OPENSSL_ppccap_P & PPC_ALTIVEC 103 ? ChaCha20_ctr32_vmx(out, inp, len, key, counter) 104 : ChaCha20_ctr32_int(out, inp, len, key, counter); 105 } 106 #endif 107 108 #ifndef OPENSSL_NO_POLY1305 109 void poly1305_init_int(void *ctx, const unsigned char key[16]); 110 void poly1305_blocks(void *ctx, const unsigned char *inp, size_t len, 111 unsigned int padbit); 112 void poly1305_emit(void *ctx, unsigned char mac[16], 113 const unsigned int nonce[4]); 114 void poly1305_init_fpu(void *ctx, const unsigned char key[16]); 115 void poly1305_blocks_fpu(void *ctx, const unsigned char *inp, size_t len, 116 unsigned int padbit); 117 void poly1305_emit_fpu(void *ctx, unsigned char mac[16], 118 const unsigned int nonce[4]); 119 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]); 120 int poly1305_init(void *ctx, const unsigned char key[16], void *func[2]) 121 { 122 if (sizeof(size_t) == 4 && (OPENSSL_ppccap_P & PPC_FPU)) { 123 poly1305_init_fpu(ctx, key); 124 func[0] = (void*)(uintptr_t)poly1305_blocks_fpu; 125 func[1] = (void*)(uintptr_t)poly1305_emit_fpu; 126 } else { 127 poly1305_init_int(ctx, key); 128 func[0] = (void*)(uintptr_t)poly1305_blocks; 129 func[1] = (void*)(uintptr_t)poly1305_emit; 130 } 131 return 1; 132 } 133 #endif 134 135 #ifdef ECP_NISTZ256_ASM 136 void ecp_nistz256_mul_mont(unsigned long res[4], const unsigned long a[4], 137 const unsigned long b[4]); 138 139 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]); 140 void ecp_nistz256_to_mont(unsigned long res[4], const unsigned long in[4]) 141 { 142 static const unsigned long RR[] = { 0x0000000000000003U, 143 0xfffffffbffffffffU, 144 0xfffffffffffffffeU, 145 0x00000004fffffffdU }; 146 147 ecp_nistz256_mul_mont(res, in, RR); 148 } 149 150 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]); 151 void ecp_nistz256_from_mont(unsigned long res[4], const unsigned long in[4]) 152 { 153 static const unsigned long one[] = { 1, 0, 0, 0 }; 154 155 ecp_nistz256_mul_mont(res, in, one); 156 } 157 #endif 158 159 static sigjmp_buf ill_jmp; 160 static void ill_handler(int sig) 161 { 162 siglongjmp(ill_jmp, sig); 163 } 164 165 void OPENSSL_fpu_probe(void); 166 void OPENSSL_ppc64_probe(void); 167 void OPENSSL_altivec_probe(void); 168 void OPENSSL_crypto207_probe(void); 169 void OPENSSL_madd300_probe(void); 170 171 long OPENSSL_rdtsc_mftb(void); 172 long OPENSSL_rdtsc_mfspr268(void); 173 174 uint32_t OPENSSL_rdtsc(void) 175 { 176 if (OPENSSL_ppccap_P & PPC_MFTB) 177 return OPENSSL_rdtsc_mftb(); 178 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 179 return OPENSSL_rdtsc_mfspr268(); 180 else 181 return 0; 182 } 183 184 size_t OPENSSL_instrument_bus_mftb(unsigned int *, size_t); 185 size_t OPENSSL_instrument_bus_mfspr268(unsigned int *, size_t); 186 187 size_t OPENSSL_instrument_bus(unsigned int *out, size_t cnt) 188 { 189 if (OPENSSL_ppccap_P & PPC_MFTB) 190 return OPENSSL_instrument_bus_mftb(out, cnt); 191 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 192 return OPENSSL_instrument_bus_mfspr268(out, cnt); 193 else 194 return 0; 195 } 196 197 size_t OPENSSL_instrument_bus2_mftb(unsigned int *, size_t, size_t); 198 size_t OPENSSL_instrument_bus2_mfspr268(unsigned int *, size_t, size_t); 199 200 size_t OPENSSL_instrument_bus2(unsigned int *out, size_t cnt, size_t max) 201 { 202 if (OPENSSL_ppccap_P & PPC_MFTB) 203 return OPENSSL_instrument_bus2_mftb(out, cnt, max); 204 else if (OPENSSL_ppccap_P & PPC_MFSPR268) 205 return OPENSSL_instrument_bus2_mfspr268(out, cnt, max); 206 else 207 return 0; 208 } 209 210 #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) 211 # if __GLIBC_PREREQ(2, 16) 212 # include <sys/auxv.h> 213 # define OSSL_IMPLEMENT_GETAUXVAL 214 # endif 215 #endif 216 217 /* I wish <sys/auxv.h> was universally available */ 218 #define HWCAP 16 /* AT_HWCAP */ 219 #define HWCAP_PPC64 (1U << 30) 220 #define HWCAP_ALTIVEC (1U << 28) 221 #define HWCAP_FPU (1U << 27) 222 #define HWCAP_POWER6_EXT (1U << 9) 223 #define HWCAP_VSX (1U << 7) 224 225 #define HWCAP2 26 /* AT_HWCAP2 */ 226 #define HWCAP_VEC_CRYPTO (1U << 25) 227 #define HWCAP_ARCH_3_00 (1U << 23) 228 229 # if defined(__GNUC__) && __GNUC__>=2 230 __attribute__ ((constructor)) 231 # endif 232 void OPENSSL_cpuid_setup(void) 233 { 234 char *e; 235 struct sigaction ill_oact, ill_act; 236 sigset_t oset; 237 static int trigger = 0; 238 239 if (trigger) 240 return; 241 trigger = 1; 242 243 if ((e = getenv("OPENSSL_ppccap"))) { 244 OPENSSL_ppccap_P = strtoul(e, NULL, 0); 245 return; 246 } 247 248 OPENSSL_ppccap_P = 0; 249 250 #if defined(_AIX) 251 OPENSSL_ppccap_P |= PPC_FPU; 252 253 if (sizeof(size_t) == 4) { 254 struct utsname uts; 255 # if defined(_SC_AIX_KERNEL_BITMODE) 256 if (sysconf(_SC_AIX_KERNEL_BITMODE) != 64) 257 return; 258 # endif 259 if (uname(&uts) != 0 || atoi(uts.version) < 6) 260 return; 261 } 262 263 # if defined(__power_set) 264 /* 265 * Value used in __power_set is a single-bit 1<<n one denoting 266 * specific processor class. Incidentally 0xffffffff<<n can be 267 * used to denote specific processor and its successors. 268 */ 269 if (sizeof(size_t) == 4) { 270 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 271 if (__power_set(0xffffffffU<<13)) /* POWER5 and later */ 272 OPENSSL_ppccap_P |= PPC_FPU64; 273 } else { 274 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 275 if (__power_set(0x1U<<14)) /* POWER6 */ 276 OPENSSL_ppccap_P |= PPC_FPU64; 277 } 278 279 if (__power_set(0xffffffffU<<14)) /* POWER6 and later */ 280 OPENSSL_ppccap_P |= PPC_ALTIVEC; 281 282 if (__power_set(0xffffffffU<<16)) /* POWER8 and later */ 283 OPENSSL_ppccap_P |= PPC_CRYPTO207; 284 285 if (__power_set(0xffffffffU<<17)) /* POWER9 and later */ 286 OPENSSL_ppccap_P |= PPC_MADD300; 287 288 return; 289 # endif 290 #endif 291 292 #if defined(__APPLE__) && defined(__MACH__) 293 OPENSSL_ppccap_P |= PPC_FPU; 294 295 { 296 int val; 297 size_t len = sizeof(val); 298 299 if (sysctlbyname("hw.optional.64bitops", &val, &len, NULL, 0) == 0) { 300 if (val) 301 OPENSSL_ppccap_P |= PPC_FPU64; 302 } 303 304 len = sizeof(val); 305 if (sysctlbyname("hw.optional.altivec", &val, &len, NULL, 0) == 0) { 306 if (val) 307 OPENSSL_ppccap_P |= PPC_ALTIVEC; 308 } 309 310 return; 311 } 312 #endif 313 314 #ifdef OSSL_IMPLEMENT_GETAUXVAL 315 { 316 unsigned long hwcap = getauxval(HWCAP); 317 unsigned long hwcap2 = getauxval(HWCAP2); 318 319 if (hwcap & HWCAP_FPU) { 320 OPENSSL_ppccap_P |= PPC_FPU; 321 322 if (sizeof(size_t) == 4) { 323 /* In 32-bit case PPC_FPU64 is always fastest [if option] */ 324 if (hwcap & HWCAP_PPC64) 325 OPENSSL_ppccap_P |= PPC_FPU64; 326 } else { 327 /* In 64-bit case PPC_FPU64 is fastest only on POWER6 */ 328 if (hwcap & HWCAP_POWER6_EXT) 329 OPENSSL_ppccap_P |= PPC_FPU64; 330 } 331 } 332 333 if (hwcap & HWCAP_ALTIVEC) { 334 OPENSSL_ppccap_P |= PPC_ALTIVEC; 335 336 if ((hwcap & HWCAP_VSX) && (hwcap2 & HWCAP_VEC_CRYPTO)) 337 OPENSSL_ppccap_P |= PPC_CRYPTO207; 338 } 339 340 if (hwcap2 & HWCAP_ARCH_3_00) { 341 OPENSSL_ppccap_P |= PPC_MADD300; 342 } 343 } 344 #endif 345 346 sigfillset(&all_masked); 347 sigdelset(&all_masked, SIGILL); 348 sigdelset(&all_masked, SIGTRAP); 349 #ifdef SIGEMT 350 sigdelset(&all_masked, SIGEMT); 351 #endif 352 sigdelset(&all_masked, SIGFPE); 353 sigdelset(&all_masked, SIGBUS); 354 sigdelset(&all_masked, SIGSEGV); 355 356 memset(&ill_act, 0, sizeof(ill_act)); 357 ill_act.sa_handler = ill_handler; 358 ill_act.sa_mask = all_masked; 359 360 sigprocmask(SIG_SETMASK, &ill_act.sa_mask, &oset); 361 sigaction(SIGILL, &ill_act, &ill_oact); 362 363 #ifndef OSSL_IMPLEMENT_GETAUXVAL 364 if (sigsetjmp(ill_jmp,1) == 0) { 365 OPENSSL_fpu_probe(); 366 OPENSSL_ppccap_P |= PPC_FPU; 367 368 if (sizeof(size_t) == 4) { 369 # ifdef __linux 370 struct utsname uts; 371 if (uname(&uts) == 0 && strcmp(uts.machine, "ppc64") == 0) 372 # endif 373 if (sigsetjmp(ill_jmp, 1) == 0) { 374 OPENSSL_ppc64_probe(); 375 OPENSSL_ppccap_P |= PPC_FPU64; 376 } 377 } else { 378 /* 379 * Wanted code detecting POWER6 CPU and setting PPC_FPU64 380 */ 381 } 382 } 383 384 if (sigsetjmp(ill_jmp, 1) == 0) { 385 OPENSSL_altivec_probe(); 386 OPENSSL_ppccap_P |= PPC_ALTIVEC; 387 if (sigsetjmp(ill_jmp, 1) == 0) { 388 OPENSSL_crypto207_probe(); 389 OPENSSL_ppccap_P |= PPC_CRYPTO207; 390 } 391 } 392 393 if (sigsetjmp(ill_jmp, 1) == 0) { 394 OPENSSL_madd300_probe(); 395 OPENSSL_ppccap_P |= PPC_MADD300; 396 } 397 #endif 398 399 if (sigsetjmp(ill_jmp, 1) == 0) { 400 OPENSSL_rdtsc_mftb(); 401 OPENSSL_ppccap_P |= PPC_MFTB; 402 } else if (sigsetjmp(ill_jmp, 1) == 0) { 403 OPENSSL_rdtsc_mfspr268(); 404 OPENSSL_ppccap_P |= PPC_MFSPR268; 405 } 406 407 sigaction(SIGILL, &ill_oact, NULL); 408 sigprocmask(SIG_SETMASK, &oset, NULL); 409 } 410