xref: /dragonfly/crypto/libressl/crypto/bn/bn_prime.c (revision 6f5ec8b5)
1 /* $OpenBSD: bn_prime.c,v 1.22 2022/07/19 16:19:19 tb Exp $ */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 /* ====================================================================
59  * Copyright (c) 1998-2001 The OpenSSL Project.  All rights reserved.
60  *
61  * Redistribution and use in source and binary forms, with or without
62  * modification, are permitted provided that the following conditions
63  * are met:
64  *
65  * 1. Redistributions of source code must retain the above copyright
66  *    notice, this list of conditions and the following disclaimer.
67  *
68  * 2. Redistributions in binary form must reproduce the above copyright
69  *    notice, this list of conditions and the following disclaimer in
70  *    the documentation and/or other materials provided with the
71  *    distribution.
72  *
73  * 3. All advertising materials mentioning features or use of this
74  *    software must display the following acknowledgment:
75  *    "This product includes software developed by the OpenSSL Project
76  *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
77  *
78  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
79  *    endorse or promote products derived from this software without
80  *    prior written permission. For written permission, please contact
81  *    openssl-core@openssl.org.
82  *
83  * 5. Products derived from this software may not be called "OpenSSL"
84  *    nor may "OpenSSL" appear in their names without prior written
85  *    permission of the OpenSSL Project.
86  *
87  * 6. Redistributions of any form whatsoever must retain the following
88  *    acknowledgment:
89  *    "This product includes software developed by the OpenSSL Project
90  *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
91  *
92  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
93  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
94  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
95  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
96  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
97  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
98  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
99  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
100  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
101  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
102  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
103  * OF THE POSSIBILITY OF SUCH DAMAGE.
104  * ====================================================================
105  *
106  * This product includes cryptographic software written by Eric Young
107  * (eay@cryptsoft.com).  This product includes software written by Tim
108  * Hudson (tjh@cryptsoft.com).
109  *
110  */
111 
112 #include <stdio.h>
113 #include <time.h>
114 
115 #include <openssl/err.h>
116 
117 #include "bn_lcl.h"
118 
119 #define LIBRESSL_HAS_BPSW
120 
121 /* NB: these functions have been "upgraded", the deprecated versions (which are
122  * compatibility wrappers using these functions) are in bn_depr.c.
123  * - Geoff
124  */
125 
126 /* The quick sieve algorithm approach to weeding out primes is
127  * Philip Zimmermann's, as implemented in PGP.  I have had a read of
128  * his comments and implemented my own version.
129  */
130 #include "bn_prime.h"
131 
132 static int witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1,
133     const BIGNUM *a1_odd, int k, BN_CTX *ctx, BN_MONT_CTX *mont);
134 static int probable_prime(BIGNUM *rnd, int bits);
135 static int probable_prime_dh(BIGNUM *rnd, int bits,
136     const BIGNUM *add, const BIGNUM *rem, BN_CTX *ctx);
137 static int probable_prime_dh_safe(BIGNUM *rnd, int bits,
138     const BIGNUM *add, const BIGNUM *rem, BN_CTX *ctx);
139 
140 int
141 BN_GENCB_call(BN_GENCB *cb, int a, int b)
142 {
143 	/* No callback means continue */
144 	if (!cb)
145 		return 1;
146 	switch (cb->ver) {
147 	case 1:
148 		/* Deprecated-style callbacks */
149 		if (!cb->cb.cb_1)
150 			return 1;
151 		cb->cb.cb_1(a, b, cb->arg);
152 		return 1;
153 	case 2:
154 		/* New-style callbacks */
155 		return cb->cb.cb_2(a, b, cb);
156 	default:
157 		break;
158 	}
159 	/* Unrecognised callback type */
160 	return 0;
161 }
162 
163 int
164 BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
165     const BIGNUM *rem, BN_GENCB *cb)
166 {
167 	BIGNUM *t;
168 	int found = 0;
169 	int i, j, c1 = 0;
170 	BN_CTX *ctx;
171 	int checks = 1;
172 
173 	if (bits < 2 || (bits == 2 && safe)) {
174 		/*
175 		 * There are no prime numbers smaller than 2, and the smallest
176 		 * safe prime (7) spans three bits.
177 		 */
178 		BNerror(BN_R_BITS_TOO_SMALL);
179 		return 0;
180 	}
181 
182 	ctx = BN_CTX_new();
183 	if (ctx == NULL)
184 		goto err;
185 	BN_CTX_start(ctx);
186 	if ((t = BN_CTX_get(ctx)) == NULL)
187 		goto err;
188 
189 #ifndef LIBRESSL_HAS_BPSW
190 	checks = BN_prime_checks_for_size(bits);
191 #endif
192 
193 loop:
194 	/* make a random number and set the top and bottom bits */
195 	if (add == NULL) {
196 		if (!probable_prime(ret, bits))
197 			goto err;
198 	} else {
199 		if (safe) {
200 			if (!probable_prime_dh_safe(ret, bits, add, rem, ctx))
201 				goto err;
202 		} else {
203 			if (!probable_prime_dh(ret, bits, add, rem, ctx))
204 				goto err;
205 		}
206 	}
207 	/* if (BN_mod_word(ret,(BN_ULONG)3) == 1) goto loop; */
208 	if (!BN_GENCB_call(cb, 0, c1++))
209 		/* aborted */
210 		goto err;
211 
212 	if (!safe) {
213 		i = BN_is_prime_fasttest_ex(ret, checks, ctx, 0, cb);
214 		if (i == -1)
215 			goto err;
216 		if (i == 0)
217 			goto loop;
218 	} else {
219 		/* for "safe prime" generation,
220 		 * check that (p-1)/2 is prime.
221 		 * Since a prime is odd, We just
222 		 * need to divide by 2 */
223 		if (!BN_rshift1(t, ret))
224 			goto err;
225 
226 		for (i = 0; i < checks; i++) {
227 			j = BN_is_prime_fasttest_ex(ret, 1, ctx, 0, cb);
228 			if (j == -1)
229 				goto err;
230 			if (j == 0)
231 				goto loop;
232 
233 			j = BN_is_prime_fasttest_ex(t, 1, ctx, 0, cb);
234 			if (j == -1)
235 				goto err;
236 			if (j == 0)
237 				goto loop;
238 
239 			if (!BN_GENCB_call(cb, 2, c1 - 1))
240 				goto err;
241 			/* We have a safe prime test pass */
242 		}
243 	}
244 	/* we have a prime :-) */
245 	found = 1;
246 
247 err:
248 	if (ctx != NULL) {
249 		BN_CTX_end(ctx);
250 		BN_CTX_free(ctx);
251 	}
252 	bn_check_top(ret);
253 	return found;
254 }
255 
256 int
257 BN_is_prime_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed, BN_GENCB *cb)
258 {
259 	return BN_is_prime_fasttest_ex(a, checks, ctx_passed, 0, cb);
260 }
261 
262 int
263 BN_is_prime_fasttest_ex(const BIGNUM *a, int checks, BN_CTX *ctx_passed,
264     int do_trial_division, BN_GENCB *cb)
265 {
266 	BN_CTX *ctx = NULL;
267 	BIGNUM *A1, *A1_odd, *check; /* taken from ctx */
268 	BN_MONT_CTX *mont = NULL;
269 	const BIGNUM *A = NULL;
270 	int i, j, k;
271 	int ret = -1;
272 
273 #ifdef LIBRESSL_HAS_BPSW
274 	int is_prime;
275 
276 	/* XXX - tickle BN_GENCB in bn_is_prime_bpsw(). */
277 	if (!bn_is_prime_bpsw(&is_prime, a, ctx_passed))
278 		return -1;
279 
280 	return is_prime;
281 #endif
282 
283 	if (BN_cmp(a, BN_value_one()) <= 0)
284 		return 0;
285 
286 	if (checks == BN_prime_checks)
287 		checks = BN_prime_checks_for_size(BN_num_bits(a));
288 
289 	/* first look for small factors */
290 	if (!BN_is_odd(a))
291 		/* a is even => a is prime if and only if a == 2 */
292 		return BN_is_word(a, 2);
293 	if (do_trial_division) {
294 		for (i = 1; i < NUMPRIMES; i++) {
295 			BN_ULONG mod = BN_mod_word(a, primes[i]);
296 			if (mod == (BN_ULONG)-1)
297 				goto err;
298 			if (mod == 0)
299 				return BN_is_word(a, primes[i]);
300 		}
301 		if (!BN_GENCB_call(cb, 1, -1))
302 			goto err;
303 	}
304 
305 	if (ctx_passed != NULL)
306 		ctx = ctx_passed;
307 	else if ((ctx = BN_CTX_new()) == NULL)
308 		goto err;
309 	BN_CTX_start(ctx);
310 
311 	/* A := abs(a) */
312 	if (a->neg) {
313 		BIGNUM *t;
314 		if ((t = BN_CTX_get(ctx)) == NULL)
315 			goto err;
316 		BN_copy(t, a);
317 		t->neg = 0;
318 		A = t;
319 	} else
320 		A = a;
321 	if ((A1 = BN_CTX_get(ctx)) == NULL)
322 		goto err;
323 	if ((A1_odd = BN_CTX_get(ctx)) == NULL)
324 		goto err;
325 	if ((check = BN_CTX_get(ctx)) == NULL)
326 		goto err;
327 
328 	/* compute A1 := A - 1 */
329 	if (!BN_copy(A1, A))
330 		goto err;
331 	if (!BN_sub_word(A1, 1))
332 		goto err;
333 	if (BN_is_zero(A1)) {
334 		ret = 0;
335 		goto err;
336 	}
337 
338 	/* write  A1  as  A1_odd * 2^k */
339 	k = 1;
340 	while (!BN_is_bit_set(A1, k))
341 		k++;
342 	if (!BN_rshift(A1_odd, A1, k))
343 		goto err;
344 
345 	/* Montgomery setup for computations mod A */
346 	mont = BN_MONT_CTX_new();
347 	if (mont == NULL)
348 		goto err;
349 	if (!BN_MONT_CTX_set(mont, A, ctx))
350 		goto err;
351 
352 	for (i = 0; i < checks; i++) {
353 		if (!BN_pseudo_rand_range(check, A1))
354 			goto err;
355 		if (!BN_add_word(check, 1))
356 			goto err;
357 		/* now 1 <= check < A */
358 
359 		j = witness(check, A, A1, A1_odd, k, ctx, mont);
360 		if (j == -1)
361 			goto err;
362 		if (j) {
363 			ret = 0;
364 			goto err;
365 		}
366 		if (!BN_GENCB_call(cb, 1, i))
367 			goto err;
368 	}
369 	ret = 1;
370 
371 err:
372 	if (ctx != NULL) {
373 		BN_CTX_end(ctx);
374 		if (ctx_passed == NULL)
375 			BN_CTX_free(ctx);
376 	}
377 	BN_MONT_CTX_free(mont);
378 
379 	return (ret);
380 }
381 
382 static int
383 witness(BIGNUM *w, const BIGNUM *a, const BIGNUM *a1, const BIGNUM *a1_odd,
384     int k, BN_CTX *ctx, BN_MONT_CTX *mont)
385 {
386 	if (!BN_mod_exp_mont_ct(w, w, a1_odd, a, ctx, mont))
387 		/* w := w^a1_odd mod a */
388 		return -1;
389 	if (BN_is_one(w))
390 		return 0; /* probably prime */
391 	if (BN_cmp(w, a1) == 0)
392 		return 0; /* w == -1 (mod a),  'a' is probably prime */
393 	while (--k) {
394 		if (!BN_mod_mul(w, w, w, a, ctx)) /* w := w^2 mod a */
395 			return -1;
396 		if (BN_is_one(w))
397 			return 1; /* 'a' is composite, otherwise a previous 'w' would
398 			           * have been == -1 (mod 'a') */
399 		if (BN_cmp(w, a1) == 0)
400 			return 0; /* w == -1 (mod a), 'a' is probably prime */
401 	}
402 	/* If we get here, 'w' is the (a-1)/2-th power of the original 'w',
403 	 * and it is neither -1 nor +1 -- so 'a' cannot be prime */
404 	bn_check_top(w);
405 	return 1;
406 }
407 
408 static int
409 probable_prime(BIGNUM *rnd, int bits)
410 {
411 	int i;
412 	prime_t mods[NUMPRIMES];
413 	BN_ULONG delta, maxdelta;
414 
415 again:
416 	if (!BN_rand(rnd, bits, 1, 1))
417 		return (0);
418 	/* we now have a random number 'rand' to test. */
419 	for (i = 1; i < NUMPRIMES; i++) {
420 		BN_ULONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);
421 		if (mod == (BN_ULONG)-1)
422 			return (0);
423 		mods[i] = (prime_t)mod;
424 	}
425 	maxdelta = BN_MASK2 - primes[NUMPRIMES - 1];
426 	delta = 0;
427 loop:
428 	for (i = 1; i < NUMPRIMES; i++) {
429 		/* check that rnd is not a prime and also
430 		 * that gcd(rnd-1,primes) == 1 (except for 2) */
431 		if (((mods[i] + delta) % primes[i]) <= 1) {
432 			delta += 2;
433 			if (delta > maxdelta)
434 				goto again;
435 			goto loop;
436 		}
437 	}
438 	if (!BN_add_word(rnd, delta))
439 		return (0);
440 	bn_check_top(rnd);
441 	return (1);
442 }
443 
444 static int
445 probable_prime_dh(BIGNUM *rnd, int bits, const BIGNUM *add, const BIGNUM *rem,
446     BN_CTX *ctx)
447 {
448 	int i, ret = 0;
449 	BIGNUM *t1;
450 
451 	BN_CTX_start(ctx);
452 	if ((t1 = BN_CTX_get(ctx)) == NULL)
453 		goto err;
454 
455 	if (!BN_rand(rnd, bits, 0, 1))
456 		goto err;
457 
458 	/* we need ((rnd-rem) % add) == 0 */
459 
460 	if (!BN_mod_ct(t1, rnd, add, ctx))
461 		goto err;
462 	if (!BN_sub(rnd, rnd, t1))
463 		goto err;
464 	if (rem == NULL) {
465 		if (!BN_add_word(rnd, 1))
466 			goto err;
467 	} else {
468 		if (!BN_add(rnd, rnd, rem))
469 			goto err;
470 	}
471 
472 	/* we now have a random number 'rand' to test. */
473 
474 loop:
475 	for (i = 1; i < NUMPRIMES; i++) {
476 		/* check that rnd is a prime */
477 		BN_LONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);
478 		if (mod == (BN_ULONG)-1)
479 			goto err;
480 		if (mod <= 1) {
481 			if (!BN_add(rnd, rnd, add))
482 				goto err;
483 			goto loop;
484 		}
485 	}
486 	ret = 1;
487 
488 err:
489 	BN_CTX_end(ctx);
490 	bn_check_top(rnd);
491 	return (ret);
492 }
493 
494 static int
495 probable_prime_dh_safe(BIGNUM *p, int bits, const BIGNUM *padd,
496     const BIGNUM *rem, BN_CTX *ctx)
497 {
498 	int i, ret = 0;
499 	BIGNUM *t1, *qadd, *q;
500 
501 	bits--;
502 	BN_CTX_start(ctx);
503 	if ((t1 = BN_CTX_get(ctx)) == NULL)
504 		goto err;
505 	if ((q = BN_CTX_get(ctx)) == NULL)
506 		goto err;
507 	if ((qadd = BN_CTX_get(ctx)) == NULL)
508 		goto err;
509 
510 	if (!BN_rshift1(qadd, padd))
511 		goto err;
512 
513 	if (!BN_rand(q, bits, 0, 1))
514 		goto err;
515 
516 	/* we need ((rnd-rem) % add) == 0 */
517 	if (!BN_mod_ct(t1, q,qadd, ctx))
518 		goto err;
519 	if (!BN_sub(q, q, t1))
520 		goto err;
521 	if (rem == NULL) {
522 		if (!BN_add_word(q, 1))
523 			goto err;
524 	} else {
525 		if (!BN_rshift1(t1, rem))
526 			goto err;
527 		if (!BN_add(q, q, t1))
528 			goto err;
529 	}
530 
531 	/* we now have a random number 'rand' to test. */
532 	if (!BN_lshift1(p, q))
533 		goto err;
534 	if (!BN_add_word(p, 1))
535 		goto err;
536 
537 loop:
538 	for (i = 1; i < NUMPRIMES; i++) {
539 		/* check that p and q are prime */
540 		/* check that for p and q
541 		 * gcd(p-1,primes) == 1 (except for 2) */
542 		BN_ULONG pmod = BN_mod_word(p, (BN_ULONG)primes[i]);
543 		BN_ULONG qmod = BN_mod_word(q, (BN_ULONG)primes[i]);
544 		if (pmod == (BN_ULONG)-1 || qmod == (BN_ULONG)-1)
545 			goto err;
546 		if (pmod == 0 || qmod == 0) {
547 			if (!BN_add(p, p, padd))
548 				goto err;
549 			if (!BN_add(q, q, qadd))
550 				goto err;
551 			goto loop;
552 		}
553 	}
554 	ret = 1;
555 
556 err:
557 	BN_CTX_end(ctx);
558 	bn_check_top(p);
559 	return (ret);
560 }
561