xref: /openbsd/lib/libcrypto/rsa/rsa_eay.c (revision 133306f0) 
1 /* crypto/rsa/rsa_eay.c */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  *
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  *
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  *
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  *
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  *
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58 
59 #include <stdio.h>
60 #include "cryptlib.h"
61 #include <openssl/bn.h>
62 #include <openssl/rsa.h>
63 #include <openssl/rand.h>
64 #include <openssl/engine.h>
65 
66 #ifndef RSA_NULL
67 
68 static int RSA_eay_public_encrypt(int flen, unsigned char *from,
69 		unsigned char *to, RSA *rsa,int padding);
70 static int RSA_eay_private_encrypt(int flen, unsigned char *from,
71 		unsigned char *to, RSA *rsa,int padding);
72 static int RSA_eay_public_decrypt(int flen, unsigned char *from,
73 		unsigned char *to, RSA *rsa,int padding);
74 static int RSA_eay_private_decrypt(int flen, unsigned char *from,
75 		unsigned char *to, RSA *rsa,int padding);
76 static int RSA_eay_mod_exp(BIGNUM *r0, BIGNUM *i, RSA *rsa);
77 static int RSA_eay_init(RSA *rsa);
78 static int RSA_eay_finish(RSA *rsa);
79 static RSA_METHOD rsa_pkcs1_eay_meth={
80 	"Eric Young's PKCS#1 RSA",
81 	RSA_eay_public_encrypt,
82 	RSA_eay_public_decrypt,
83 	RSA_eay_private_encrypt,
84 	RSA_eay_private_decrypt,
85 	RSA_eay_mod_exp,
86 	BN_mod_exp_mont,
87 	RSA_eay_init,
88 	RSA_eay_finish,
89 	0,
90 	NULL,
91 	};
92 
93 RSA_METHOD *RSA_PKCS1_SSLeay(void)
94 	{
95 	return(&rsa_pkcs1_eay_meth);
96 	}
97 
98 static int RSA_eay_public_encrypt(int flen, unsigned char *from,
99 	     unsigned char *to, RSA *rsa, int padding)
100 	{
101 	const RSA_METHOD *meth;
102 	BIGNUM f,ret;
103 	int i,j,k,num=0,r= -1;
104 	unsigned char *buf=NULL;
105 	BN_CTX *ctx=NULL;
106 
107 	meth = ENGINE_get_RSA(rsa->engine);
108 	BN_init(&f);
109 	BN_init(&ret);
110 	if ((ctx=BN_CTX_new()) == NULL) goto err;
111 	num=BN_num_bytes(rsa->n);
112 	if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
113 		{
114 		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,ERR_R_MALLOC_FAILURE);
115 		goto err;
116 		}
117 
118 	switch (padding)
119 		{
120 	case RSA_PKCS1_PADDING:
121 		i=RSA_padding_add_PKCS1_type_2(buf,num,from,flen);
122 		break;
123 #ifndef NO_SHA
124 	case RSA_PKCS1_OAEP_PADDING:
125 	        i=RSA_padding_add_PKCS1_OAEP(buf,num,from,flen,NULL,0);
126 		break;
127 #endif
128 	case RSA_SSLV23_PADDING:
129 		i=RSA_padding_add_SSLv23(buf,num,from,flen);
130 		break;
131 	case RSA_NO_PADDING:
132 		i=RSA_padding_add_none(buf,num,from,flen);
133 		break;
134 	default:
135 		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
136 		goto err;
137 		}
138 	if (i <= 0) goto err;
139 
140 	if (BN_bin2bn(buf,num,&f) == NULL) goto err;
141 
142 	if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
143 		{
144 		if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
145 			if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
146 			    goto err;
147 		}
148 
149 	if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
150 		rsa->_method_mod_n)) goto err;
151 
152 	/* put in leading 0 bytes if the number is less than the
153 	 * length of the modulus */
154 	j=BN_num_bytes(&ret);
155 	i=BN_bn2bin(&ret,&(to[num-j]));
156 	for (k=0; k<(num-i); k++)
157 		to[k]=0;
158 
159 	r=num;
160 err:
161 	if (ctx != NULL) BN_CTX_free(ctx);
162 	BN_clear_free(&f);
163 	BN_clear_free(&ret);
164 	if (buf != NULL)
165 		{
166 		memset(buf,0,num);
167 		OPENSSL_free(buf);
168 		}
169 	return(r);
170 	}
171 
172 static int RSA_eay_private_encrypt(int flen, unsigned char *from,
173 	     unsigned char *to, RSA *rsa, int padding)
174 	{
175 	const RSA_METHOD *meth;
176 	BIGNUM f,ret;
177 	int i,j,k,num=0,r= -1;
178 	unsigned char *buf=NULL;
179 	BN_CTX *ctx=NULL;
180 
181 	meth = ENGINE_get_RSA(rsa->engine);
182 	BN_init(&f);
183 	BN_init(&ret);
184 
185 	if ((ctx=BN_CTX_new()) == NULL) goto err;
186 	num=BN_num_bytes(rsa->n);
187 	if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
188 		{
189 		RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,ERR_R_MALLOC_FAILURE);
190 		goto err;
191 		}
192 
193 	switch (padding)
194 		{
195 	case RSA_PKCS1_PADDING:
196 		i=RSA_padding_add_PKCS1_type_1(buf,num,from,flen);
197 		break;
198 	case RSA_NO_PADDING:
199 		i=RSA_padding_add_none(buf,num,from,flen);
200 		break;
201 	case RSA_SSLV23_PADDING:
202 	default:
203 		RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
204 		goto err;
205 		}
206 	if (i <= 0) goto err;
207 
208 	if (BN_bin2bn(buf,num,&f) == NULL) goto err;
209 
210 	if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
211 		RSA_blinding_on(rsa,ctx);
212 	if (rsa->flags & RSA_FLAG_BLINDING)
213 		if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
214 
215 	if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
216 		((rsa->p != NULL) &&
217 		(rsa->q != NULL) &&
218 		(rsa->dmp1 != NULL) &&
219 		(rsa->dmq1 != NULL) &&
220 		(rsa->iqmp != NULL)) )
221 		{ if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
222 	else
223 		{
224 		if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL)) goto err;
225 		}
226 
227 	if (rsa->flags & RSA_FLAG_BLINDING)
228 		if (!BN_BLINDING_invert(&ret,rsa->blinding,ctx)) goto err;
229 
230 	/* put in leading 0 bytes if the number is less than the
231 	 * length of the modulus */
232 	j=BN_num_bytes(&ret);
233 	i=BN_bn2bin(&ret,&(to[num-j]));
234 	for (k=0; k<(num-i); k++)
235 		to[k]=0;
236 
237 	r=num;
238 err:
239 	if (ctx != NULL) BN_CTX_free(ctx);
240 	BN_clear_free(&ret);
241 	BN_clear_free(&f);
242 	if (buf != NULL)
243 		{
244 		memset(buf,0,num);
245 		OPENSSL_free(buf);
246 		}
247 	return(r);
248 	}
249 
250 static int RSA_eay_private_decrypt(int flen, unsigned char *from,
251 	     unsigned char *to, RSA *rsa, int padding)
252 	{
253 	const RSA_METHOD *meth;
254 	BIGNUM f,ret;
255 	int j,num=0,r= -1;
256 	unsigned char *p;
257 	unsigned char *buf=NULL;
258 	BN_CTX *ctx=NULL;
259 
260 	meth = ENGINE_get_RSA(rsa->engine);
261 	BN_init(&f);
262 	BN_init(&ret);
263 	ctx=BN_CTX_new();
264 	if (ctx == NULL) goto err;
265 
266 	num=BN_num_bytes(rsa->n);
267 
268 	if ((buf=(unsigned char *)OPENSSL_malloc(num)) == NULL)
269 		{
270 		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,ERR_R_MALLOC_FAILURE);
271 		goto err;
272 		}
273 
274 	/* This check was for equality but PGP does evil things
275 	 * and chops off the top '0' bytes */
276 	if (flen > num)
277 		{
278 		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
279 		goto err;
280 		}
281 
282 	/* make data into a big number */
283 	if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;
284 
285 	if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
286 		RSA_blinding_on(rsa,ctx);
287 	if (rsa->flags & RSA_FLAG_BLINDING)
288 		if (!BN_BLINDING_convert(&f,rsa->blinding,ctx)) goto err;
289 
290 	/* do the decrypt */
291 	if ( (rsa->flags & RSA_FLAG_EXT_PKEY) ||
292 		((rsa->p != NULL) &&
293 		(rsa->q != NULL) &&
294 		(rsa->dmp1 != NULL) &&
295 		(rsa->dmq1 != NULL) &&
296 		(rsa->iqmp != NULL)) )
297 		{ if (!meth->rsa_mod_exp(&ret,&f,rsa)) goto err; }
298 	else
299 		{
300 		if (!meth->bn_mod_exp(&ret,&f,rsa->d,rsa->n,ctx,NULL))
301 			goto err;
302 		}
303 
304 	if (rsa->flags & RSA_FLAG_BLINDING)
305 		if (!BN_BLINDING_invert(&ret,rsa->blinding,ctx)) goto err;
306 
307 	p=buf;
308 	j=BN_bn2bin(&ret,p); /* j is only used with no-padding mode */
309 
310 	switch (padding)
311 		{
312 	case RSA_PKCS1_PADDING:
313 		r=RSA_padding_check_PKCS1_type_2(to,num,buf,j,num);
314 		break;
315 #ifndef NO_SHA
316         case RSA_PKCS1_OAEP_PADDING:
317 	        r=RSA_padding_check_PKCS1_OAEP(to,num,buf,j,num,NULL,0);
318                 break;
319 #endif
320  	case RSA_SSLV23_PADDING:
321 		r=RSA_padding_check_SSLv23(to,num,buf,j,num);
322 		break;
323 	case RSA_NO_PADDING:
324 		r=RSA_padding_check_none(to,num,buf,j,num);
325 		break;
326 	default:
327 		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
328 		goto err;
329 		}
330 	if (r < 0)
331 		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
332 
333 err:
334 	if (ctx != NULL) BN_CTX_free(ctx);
335 	BN_clear_free(&f);
336 	BN_clear_free(&ret);
337 	if (buf != NULL)
338 		{
339 		memset(buf,0,num);
340 		OPENSSL_free(buf);
341 		}
342 	return(r);
343 	}
344 
345 static int RSA_eay_public_decrypt(int flen, unsigned char *from,
346 	     unsigned char *to, RSA *rsa, int padding)
347 	{
348 	const RSA_METHOD *meth;
349 	BIGNUM f,ret;
350 	int i,num=0,r= -1;
351 	unsigned char *p;
352 	unsigned char *buf=NULL;
353 	BN_CTX *ctx=NULL;
354 
355 	meth = ENGINE_get_RSA(rsa->engine);
356 	BN_init(&f);
357 	BN_init(&ret);
358 	ctx=BN_CTX_new();
359 	if (ctx == NULL) goto err;
360 
361 	num=BN_num_bytes(rsa->n);
362 	buf=(unsigned char *)OPENSSL_malloc(num);
363 	if (buf == NULL)
364 		{
365 		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,ERR_R_MALLOC_FAILURE);
366 		goto err;
367 		}
368 
369 	/* This check was for equality but PGP does evil things
370 	 * and chops off the top '0' bytes */
371 	if (flen > num)
372 		{
373 		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_GREATER_THAN_MOD_LEN);
374 		goto err;
375 		}
376 
377 	if (BN_bin2bn(from,flen,&f) == NULL) goto err;
378 	/* do the decrypt */
379 	if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
380 		{
381 		if ((rsa->_method_mod_n=BN_MONT_CTX_new()) != NULL)
382 			if (!BN_MONT_CTX_set(rsa->_method_mod_n,rsa->n,ctx))
383 			    goto err;
384 		}
385 
386 	if (!meth->bn_mod_exp(&ret,&f,rsa->e,rsa->n,ctx,
387 		rsa->_method_mod_n)) goto err;
388 
389 	p=buf;
390 	i=BN_bn2bin(&ret,p);
391 
392 	switch (padding)
393 		{
394 	case RSA_PKCS1_PADDING:
395 		r=RSA_padding_check_PKCS1_type_1(to,num,buf,i,num);
396 		break;
397 	case RSA_NO_PADDING:
398 		r=RSA_padding_check_none(to,num,buf,i,num);
399 		break;
400 	default:
401 		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_UNKNOWN_PADDING_TYPE);
402 		goto err;
403 		}
404 	if (r < 0)
405 		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_PADDING_CHECK_FAILED);
406 
407 err:
408 	if (ctx != NULL) BN_CTX_free(ctx);
409 	BN_clear_free(&f);
410 	BN_clear_free(&ret);
411 	if (buf != NULL)
412 		{
413 		memset(buf,0,num);
414 		OPENSSL_free(buf);
415 		}
416 	return(r);
417 	}
418 
419 static int RSA_eay_mod_exp(BIGNUM *r0, BIGNUM *I, RSA *rsa)
420 	{
421 	const RSA_METHOD *meth;
422 	BIGNUM r1,m1;
423 	int ret=0;
424 	BN_CTX *ctx;
425 
426 	meth = ENGINE_get_RSA(rsa->engine);
427 	if ((ctx=BN_CTX_new()) == NULL) goto err;
428 	BN_init(&m1);
429 	BN_init(&r1);
430 
431 	if (rsa->flags & RSA_FLAG_CACHE_PRIVATE)
432 		{
433 		if (rsa->_method_mod_p == NULL)
434 			{
435 			if ((rsa->_method_mod_p=BN_MONT_CTX_new()) != NULL)
436 				if (!BN_MONT_CTX_set(rsa->_method_mod_p,rsa->p,
437 						     ctx))
438 					goto err;
439 			}
440 		if (rsa->_method_mod_q == NULL)
441 			{
442 			if ((rsa->_method_mod_q=BN_MONT_CTX_new()) != NULL)
443 				if (!BN_MONT_CTX_set(rsa->_method_mod_q,rsa->q,
444 						     ctx))
445 					goto err;
446 			}
447 		}
448 
449 	if (!BN_mod(&r1,I,rsa->q,ctx)) goto err;
450 	if (!meth->bn_mod_exp(&m1,&r1,rsa->dmq1,rsa->q,ctx,
451 		rsa->_method_mod_q)) goto err;
452 
453 	if (!BN_mod(&r1,I,rsa->p,ctx)) goto err;
454 	if (!meth->bn_mod_exp(r0,&r1,rsa->dmp1,rsa->p,ctx,
455 		rsa->_method_mod_p)) goto err;
456 
457 	if (!BN_sub(r0,r0,&m1)) goto err;
458 	/* This will help stop the size of r0 increasing, which does
459 	 * affect the multiply if it optimised for a power of 2 size */
460 	if (r0->neg)
461 		if (!BN_add(r0,r0,rsa->p)) goto err;
462 
463 	if (!BN_mul(&r1,r0,rsa->iqmp,ctx)) goto err;
464 	if (!BN_mod(r0,&r1,rsa->p,ctx)) goto err;
465 	/* If p < q it is occasionally possible for the correction of
466          * adding 'p' if r0 is negative above to leave the result still
467 	 * negative. This can break the private key operations: the following
468 	 * second correction should *always* correct this rare occurrence.
469 	 * This will *never* happen with OpenSSL generated keys because
470          * they ensure p > q [steve]
471          */
472 	if (r0->neg)
473 		if (!BN_add(r0,r0,rsa->p)) goto err;
474 	if (!BN_mul(&r1,r0,rsa->q,ctx)) goto err;
475 	if (!BN_add(r0,&r1,&m1)) goto err;
476 
477 	ret=1;
478 err:
479 	BN_clear_free(&m1);
480 	BN_clear_free(&r1);
481 	BN_CTX_free(ctx);
482 	return(ret);
483 	}
484 
485 static int RSA_eay_init(RSA *rsa)
486 	{
487 	rsa->flags|=RSA_FLAG_CACHE_PUBLIC|RSA_FLAG_CACHE_PRIVATE;
488 	return(1);
489 	}
490 
491 static int RSA_eay_finish(RSA *rsa)
492 	{
493 	if (rsa->_method_mod_n != NULL)
494 		BN_MONT_CTX_free(rsa->_method_mod_n);
495 	if (rsa->_method_mod_p != NULL)
496 		BN_MONT_CTX_free(rsa->_method_mod_p);
497 	if (rsa->_method_mod_q != NULL)
498 		BN_MONT_CTX_free(rsa->_method_mod_q);
499 	return(1);
500 	}
501 
502 #endif
503