1 /* $OpenBSD: rsa_ameth.c,v 1.26 2022/06/27 12:36:06 tb Exp $ */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58 
59 #include <stdio.h>
60 
61 #include <openssl/opensslconf.h>
62 
63 #include <openssl/asn1t.h>
64 #include <openssl/bn.h>
65 #include <openssl/cms.h>
66 #include <openssl/err.h>
67 #include <openssl/x509.h>
68 
69 #include "asn1_locl.h"
70 #include "cryptlib.h"
71 #include "evp_locl.h"
72 #include "rsa_locl.h"
73 
74 #ifndef OPENSSL_NO_CMS
75 static int rsa_cms_sign(CMS_SignerInfo *si);
76 static int rsa_cms_verify(CMS_SignerInfo *si);
77 static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
78 static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
79 #endif
80 
81 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg);
82 
83 /* Set any parameters associated with pkey */
84 static int
85 rsa_param_encode(const EVP_PKEY *pkey, ASN1_STRING **pstr, int *pstrtype)
86 {
87 	const RSA *rsa = pkey->pkey.rsa;
88 
89 	*pstr = NULL;
90 
91 	/* If RSA it's just NULL type */
92 	if (pkey->ameth->pkey_id != EVP_PKEY_RSA_PSS) {
93 		*pstrtype = V_ASN1_NULL;
94 		return 1;
95 	}
96 
97 	/* If no PSS parameters we omit parameters entirely */
98 	if (rsa->pss == NULL) {
99 		*pstrtype = V_ASN1_UNDEF;
100 		return 1;
101 	}
102 
103 	/* Encode PSS parameters */
104 	if (ASN1_item_pack(rsa->pss, &RSA_PSS_PARAMS_it, pstr) == NULL)
105 		return 0;
106 
107 	*pstrtype = V_ASN1_SEQUENCE;
108 	return 1;
109 }
110 
111 /* Decode any parameters and set them in RSA structure */
112 static int
113 rsa_param_decode(RSA *rsa, const X509_ALGOR *alg)
114 {
115 	const ASN1_OBJECT *algoid;
116 	const void *algp;
117 	int algptype;
118 
119 	X509_ALGOR_get0(&algoid, &algptype, &algp, alg);
120 	if (OBJ_obj2nid(algoid) != EVP_PKEY_RSA_PSS)
121 		return 1;
122 	if (algptype == V_ASN1_UNDEF)
123 		return 1;
124 	if (algptype != V_ASN1_SEQUENCE) {
125 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
126 		return 0;
127 	}
128 	rsa->pss = rsa_pss_decode(alg);
129 	if (rsa->pss == NULL)
130 		return 0;
131 	return 1;
132 }
133 
134 static int
135 rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
136 {
137 	unsigned char *penc = NULL;
138 	int penclen;
139 	ASN1_STRING *str;
140 	int strtype;
141 
142 	if (!rsa_param_encode(pkey, &str, &strtype))
143 		return 0;
144 	penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
145 	if (penclen <= 0)
146 		return 0;
147 	if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id),
148 	    strtype, str, penc, penclen))
149 		return 1;
150 
151 	free(penc);
152 
153 	return 0;
154 }
155 
156 static int
157 rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
158 {
159 	const unsigned char *p;
160 	int pklen;
161 	X509_ALGOR *alg;
162 	RSA *rsa = NULL;
163 
164 	if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, &alg, pubkey))
165 		return 0;
166 	if ((rsa = d2i_RSAPublicKey(NULL, &p, pklen)) == NULL) {
167 		RSAerror(ERR_R_RSA_LIB);
168 		return 0;
169 	}
170 	if (!rsa_param_decode(rsa, alg)) {
171 		RSA_free(rsa);
172 		return 0;
173 	}
174 	if (!EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa)) {
175 		RSA_free(rsa);
176 		return 0;
177 	}
178 	return 1;
179 }
180 
181 static int
182 rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
183 {
184 	if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0 ||
185 	    BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
186 		return 0;
187 
188 	return 1;
189 }
190 
191 static int
192 old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
193 {
194 	RSA *rsa;
195 
196 	if ((rsa = d2i_RSAPrivateKey(NULL, pder, derlen)) == NULL) {
197 		RSAerror(ERR_R_RSA_LIB);
198 		return 0;
199 	}
200 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
201 	return 1;
202 }
203 
204 static int
205 old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder)
206 {
207 	return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
208 }
209 
210 static int
211 rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
212 {
213 	unsigned char *rk = NULL;
214 	int rklen;
215 	ASN1_STRING *str;
216 	int strtype;
217 
218 	if (!rsa_param_encode(pkey, &str, &strtype))
219 		return 0;
220 
221 	rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
222 	if (rklen <= 0) {
223 		RSAerror(ERR_R_MALLOC_FAILURE);
224 		ASN1_STRING_free(str);
225 		return 0;
226 	}
227 
228 	if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0,
229 	    strtype, str, rk, rklen)) {
230 		RSAerror(ERR_R_MALLOC_FAILURE);
231 		ASN1_STRING_free(str);
232 		return 0;
233 	}
234 
235 	return 1;
236 }
237 
238 static int
239 rsa_priv_decode(EVP_PKEY *pkey, const PKCS8_PRIV_KEY_INFO *p8)
240 {
241 	const unsigned char *p;
242 	RSA *rsa;
243 	int pklen;
244 	const X509_ALGOR *alg;
245 
246 	if (!PKCS8_pkey_get0(NULL, &p, &pklen, &alg, p8))
247 		return 0;
248 	rsa = d2i_RSAPrivateKey(NULL, &p, pklen);
249 	if (rsa == NULL) {
250 		RSAerror(ERR_R_RSA_LIB);
251 		return 0;
252 	}
253 	if (!rsa_param_decode(rsa, alg)) {
254 		RSA_free(rsa);
255 		return 0;
256 	}
257 	EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
258 
259 	return 1;
260 }
261 
262 static int
263 int_rsa_size(const EVP_PKEY *pkey)
264 {
265 	return RSA_size(pkey->pkey.rsa);
266 }
267 
268 static int
269 rsa_bits(const EVP_PKEY *pkey)
270 {
271 	return BN_num_bits(pkey->pkey.rsa->n);
272 }
273 
274 static int
275 rsa_security_bits(const EVP_PKEY *pkey)
276 {
277 	return RSA_security_bits(pkey->pkey.rsa);
278 }
279 
280 static void
281 int_rsa_free(EVP_PKEY *pkey)
282 {
283 	RSA_free(pkey->pkey.rsa);
284 }
285 
286 static X509_ALGOR *
287 rsa_mgf1_decode(X509_ALGOR *alg)
288 {
289 	if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
290 		return NULL;
291 
292 	return ASN1_TYPE_unpack_sequence(&X509_ALGOR_it, alg->parameter);
293 }
294 
295 static RSA_PSS_PARAMS *
296 rsa_pss_decode(const X509_ALGOR *alg)
297 {
298 	RSA_PSS_PARAMS *pss;
299 
300 	pss = ASN1_TYPE_unpack_sequence(&RSA_PSS_PARAMS_it, alg->parameter);
301 	if (pss == NULL)
302 		return NULL;
303 
304 	if (pss->maskGenAlgorithm != NULL) {
305 		pss->maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
306 		if (pss->maskHash == NULL) {
307 			RSA_PSS_PARAMS_free(pss);
308 			return NULL;
309 		}
310 	}
311 
312 	return pss;
313 }
314 
315 static int
316 rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, int indent)
317 {
318 	int rv = 0;
319 	X509_ALGOR *maskHash = NULL;
320 
321 	if (!BIO_indent(bp, indent, 128))
322 		goto err;
323 	if (pss_key) {
324 		if (pss == NULL) {
325 			if (BIO_puts(bp, "No PSS parameter restrictions\n") <= 0)
326 				return 0;
327 			return 1;
328 		} else {
329 			if (BIO_puts(bp, "PSS parameter restrictions:") <= 0)
330 				return 0;
331 		}
332 	} else if (pss == NULL) {
333 		if (BIO_puts(bp,"(INVALID PSS PARAMETERS)\n") <= 0)
334 			return 0;
335 		return 1;
336 	}
337 	if (BIO_puts(bp, "\n") <= 0)
338 		goto err;
339 	if (pss_key)
340 		indent += 2;
341 	if (!BIO_indent(bp, indent, 128))
342 		goto err;
343 	if (BIO_puts(bp, "Hash Algorithm: ") <= 0)
344 		goto err;
345 
346 	if (pss->hashAlgorithm) {
347 		if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0)
348 			goto err;
349 	} else if (BIO_puts(bp, "sha1 (default)") <= 0) {
350 		goto err;
351 	}
352 
353 	if (BIO_puts(bp, "\n") <= 0)
354 		goto err;
355 
356 	if (!BIO_indent(bp, indent, 128))
357 		goto err;
358 
359 	if (BIO_puts(bp, "Mask Algorithm: ") <= 0)
360 		goto err;
361 	if (pss->maskGenAlgorithm) {
362 		if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0)
363 			goto err;
364 		if (BIO_puts(bp, " with ") <= 0)
365 			goto err;
366 		maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
367 		if (maskHash != NULL) {
368 			if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
369 				goto err;
370 		} else if (BIO_puts(bp, "INVALID") <= 0) {
371 			goto err;
372 		}
373 	} else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0) {
374 		goto err;
375 	}
376 	BIO_puts(bp, "\n");
377 
378 	if (!BIO_indent(bp, indent, 128))
379 		goto err;
380 	if (BIO_printf(bp, "%s Salt Length: 0x", pss_key ? "Minimum" : "") <= 0)
381 		goto err;
382 	if (pss->saltLength) {
383 		if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0)
384 			goto err;
385 	} else if (BIO_puts(bp, "14 (default)") <= 0) {
386 		goto err;
387 	}
388 	BIO_puts(bp, "\n");
389 
390 	if (!BIO_indent(bp, indent, 128))
391 		goto err;
392 	if (BIO_puts(bp, "Trailer Field: 0x") <= 0)
393 		goto err;
394 	if (pss->trailerField) {
395 		if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0)
396 			goto err;
397 	} else if (BIO_puts(bp, "BC (default)") <= 0) {
398 		goto err;
399 	}
400 	BIO_puts(bp, "\n");
401 
402 	rv = 1;
403 
404  err:
405 	X509_ALGOR_free(maskHash);
406 	return rv;
407 
408 }
409 
410 static void
411 update_buflen(const BIGNUM *b, size_t *pbuflen)
412 {
413 	size_t i;
414 
415 	if (!b)
416 		return;
417 	if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
418 		*pbuflen = i;
419 }
420 
421 static int
422 pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
423 {
424 	const RSA *x = pkey->pkey.rsa;
425 	unsigned char *m = NULL;
426 	char *str;
427 	const char *s;
428 	int ret = 0, mod_len = 0;
429 	size_t buf_len = 0;
430 
431 	update_buflen(x->n, &buf_len);
432 	update_buflen(x->e, &buf_len);
433 
434 	if (priv) {
435 		update_buflen(x->d, &buf_len);
436 		update_buflen(x->p, &buf_len);
437 		update_buflen(x->q, &buf_len);
438 		update_buflen(x->dmp1, &buf_len);
439 		update_buflen(x->dmq1, &buf_len);
440 		update_buflen(x->iqmp, &buf_len);
441 	}
442 
443 	m = malloc(buf_len + 10);
444 	if (m == NULL) {
445 		RSAerror(ERR_R_MALLOC_FAILURE);
446 		goto err;
447 	}
448 
449 	if (x->n != NULL)
450 		mod_len = BN_num_bits(x->n);
451 
452 	if (!BIO_indent(bp, off, 128))
453 		goto err;
454 
455 	if (BIO_printf(bp, "%s ", pkey_is_pss(pkey) ?  "RSA-PSS" : "RSA") <= 0)
456 		goto err;
457 
458 	if (priv && x->d != NULL) {
459 		if (BIO_printf(bp, "Private-Key: (%d bit)\n", mod_len) <= 0)
460 			goto err;
461 		str = "modulus:";
462 		s = "publicExponent:";
463 	} else {
464 		if (BIO_printf(bp, "Public-Key: (%d bit)\n", mod_len) <= 0)
465 			goto err;
466 		str = "Modulus:";
467 		s = "Exponent:";
468 	}
469 	if (!ASN1_bn_print(bp, str, x->n, m, off))
470 		goto err;
471 	if (!ASN1_bn_print(bp, s, x->e, m, off))
472 		goto err;
473 	if (priv) {
474 		if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
475 			goto err;
476 		if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
477 			goto err;
478 		if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
479 			goto err;
480 		if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
481 			goto err;
482 		if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
483 			goto err;
484 		if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
485 			goto err;
486 	}
487 	if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
488 		goto err;
489 	ret = 1;
490  err:
491 	free(m);
492 	return ret;
493 }
494 
495 static int
496 rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
497 {
498 	return pkey_rsa_print(bp, pkey, indent, 0);
499 }
500 
501 static int
502 rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent, ASN1_PCTX *ctx)
503 {
504 	return pkey_rsa_print(bp, pkey, indent, 1);
505 }
506 
507 static int
508 rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg, const ASN1_STRING *sig,
509     int indent, ASN1_PCTX *pctx)
510 {
511 	if (OBJ_obj2nid(sigalg->algorithm) == EVP_PKEY_RSA_PSS) {
512 		int rv;
513 		RSA_PSS_PARAMS *pss = rsa_pss_decode(sigalg);
514 
515 		rv = rsa_pss_param_print(bp, 0, pss, indent);
516 		RSA_PSS_PARAMS_free(pss);
517 		if (!rv)
518 			return 0;
519 	} else if (!sig && BIO_puts(bp, "\n") <= 0) {
520 		return 0;
521 	}
522 	if (sig)
523 		return X509_signature_dump(bp, sig, indent);
524 	return 1;
525 }
526 
527 static int
528 rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
529 {
530 	X509_ALGOR *alg = NULL;
531 	const EVP_MD *md;
532 	const EVP_MD *mgf1md;
533 	int min_saltlen;
534 
535 	switch (op) {
536 	case ASN1_PKEY_CTRL_PKCS7_SIGN:
537 		if (arg1 == 0)
538 			PKCS7_SIGNER_INFO_get0_algs(arg2, NULL, NULL, &alg);
539 		break;
540 
541 	case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
542 		if (pkey_is_pss(pkey))
543 			return -2;
544 		if (arg1 == 0)
545 			PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
546 		break;
547 #ifndef OPENSSL_NO_CMS
548 	case ASN1_PKEY_CTRL_CMS_SIGN:
549 		if (arg1 == 0)
550 			return rsa_cms_sign(arg2);
551 		else if (arg1 == 1)
552 			return rsa_cms_verify(arg2);
553 		break;
554 
555 	case ASN1_PKEY_CTRL_CMS_ENVELOPE:
556 		if (pkey_is_pss(pkey))
557 			return -2;
558 		if (arg1 == 0)
559 			return rsa_cms_encrypt(arg2);
560 		else if (arg1 == 1)
561 			return rsa_cms_decrypt(arg2);
562 		break;
563 
564 	case ASN1_PKEY_CTRL_CMS_RI_TYPE:
565 		if (pkey_is_pss(pkey))
566 			return -2;
567 		*(int *)arg2 = CMS_RECIPINFO_TRANS;
568 		return 1;
569 #endif
570 
571 	case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
572 		if (pkey->pkey.rsa->pss != NULL) {
573 			if (!rsa_pss_get_param(pkey->pkey.rsa->pss, &md, &mgf1md,
574 			    &min_saltlen)) {
575 				RSAerror(ERR_R_INTERNAL_ERROR);
576 				return 0;
577 			}
578 			*(int *)arg2 = EVP_MD_type(md);
579 			/* Return of 2 indicates this MD is mandatory */
580 			return 2;
581 		}
582 		*(int *)arg2 = NID_sha256;
583 		return 1;
584 
585 	default:
586 		return -2;
587 	}
588 
589 	if (alg)
590 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
591 		    V_ASN1_NULL, 0);
592 
593 	return 1;
594 }
595 
596 /* Allocate and set algorithm ID from EVP_MD, defaults to SHA1. */
597 static int
598 rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
599 {
600 	if (md == NULL || EVP_MD_type(md) == NID_sha1)
601 		return 1;
602 	*palg = X509_ALGOR_new();
603 	if (*palg == NULL)
604 		return 0;
605 	X509_ALGOR_set_md(*palg, md);
606 	return 1;
607 }
608 
609 /* Allocate and set MGF1 algorithm ID from EVP_MD. */
610 static int
611 rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
612 {
613 	X509_ALGOR *algtmp = NULL;
614 	ASN1_STRING *stmp = NULL;
615 
616 	*palg = NULL;
617 	if (mgf1md == NULL || EVP_MD_type(mgf1md) == NID_sha1)
618 		return 1;
619 	/* need to embed algorithm ID inside another */
620 	if (!rsa_md_to_algor(&algtmp, mgf1md))
621 		goto err;
622 	if (ASN1_item_pack(algtmp, &X509_ALGOR_it, &stmp) == NULL)
623 		 goto err;
624 	*palg = X509_ALGOR_new();
625 	if (*palg == NULL)
626 		goto err;
627 	X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
628 	stmp = NULL;
629  err:
630 	ASN1_STRING_free(stmp);
631 	X509_ALGOR_free(algtmp);
632 	if (*palg)
633 		return 1;
634 	return 0;
635 }
636 
637 /* Convert algorithm ID to EVP_MD, defaults to SHA1. */
638 static const EVP_MD *
639 rsa_algor_to_md(X509_ALGOR *alg)
640 {
641 	const EVP_MD *md;
642 
643 	if (!alg)
644 		return EVP_sha1();
645 	md = EVP_get_digestbyobj(alg->algorithm);
646 	if (md == NULL)
647 		RSAerror(RSA_R_UNKNOWN_DIGEST);
648 	return md;
649 }
650 
651 /*
652  * Convert EVP_PKEY_CTX in PSS mode into corresponding algorithm parameter,
653  * suitable for setting an AlgorithmIdentifier.
654  */
655 static RSA_PSS_PARAMS *
656 rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
657 {
658 	const EVP_MD *sigmd, *mgf1md;
659 	EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
660 	int saltlen;
661 
662 	if (EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) <= 0)
663 		return NULL;
664 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
665 		return NULL;
666 	if (!EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen))
667 		return NULL;
668 	if (saltlen == -1) {
669 		saltlen = EVP_MD_size(sigmd);
670 	} else if (saltlen == -2 || saltlen == -3) {
671 		saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
672 		if ((EVP_PKEY_bits(pk) & 0x7) == 1)
673 			saltlen--;
674 		if (saltlen < 0)
675 			return NULL;
676 	}
677 
678 	return rsa_pss_params_create(sigmd, mgf1md, saltlen);
679 }
680 
681 RSA_PSS_PARAMS *
682 rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md, int saltlen)
683 {
684 	RSA_PSS_PARAMS *pss = RSA_PSS_PARAMS_new();
685 
686 	if (pss == NULL)
687 		goto err;
688 	if (saltlen != 20) {
689 		pss->saltLength = ASN1_INTEGER_new();
690 		if (pss->saltLength == NULL)
691 			goto err;
692 		if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
693 			goto err;
694 	}
695 	if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
696 		goto err;
697 	if (mgf1md == NULL)
698 		mgf1md = sigmd;
699 	if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
700 		goto err;
701 	if (!rsa_md_to_algor(&pss->maskHash, mgf1md))
702 		goto err;
703 	return pss;
704  err:
705 	RSA_PSS_PARAMS_free(pss);
706 	return NULL;
707 }
708 
709 static ASN1_STRING *
710 rsa_ctx_to_pss_string(EVP_PKEY_CTX *pkctx)
711 {
712 	RSA_PSS_PARAMS *pss = rsa_ctx_to_pss(pkctx);
713 	ASN1_STRING *os;
714 
715 	if (pss == NULL)
716 		return NULL;
717 
718 	os = ASN1_item_pack(pss, &RSA_PSS_PARAMS_it, NULL);
719 	RSA_PSS_PARAMS_free(pss);
720 	return os;
721 }
722 
723 /*
724  * From PSS AlgorithmIdentifier set public key parameters. If pkey isn't NULL
725  * then the EVP_MD_CTX is setup and initialised. If it is NULL parameters are
726  * passed to pkctx instead.
727  */
728 
729 static int
730 rsa_pss_to_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pkctx,
731     X509_ALGOR *sigalg, EVP_PKEY *pkey)
732 {
733 	int rv = -1;
734 	int saltlen;
735 	const EVP_MD *mgf1md = NULL, *md = NULL;
736 	RSA_PSS_PARAMS *pss;
737 
738 	/* Sanity check: make sure it is PSS */
739 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
740 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
741 		return -1;
742 	}
743 	/* Decode PSS parameters */
744 	pss = rsa_pss_decode(sigalg);
745 
746 	if (!rsa_pss_get_param(pss, &md, &mgf1md, &saltlen)) {
747 		RSAerror(RSA_R_INVALID_PSS_PARAMETERS);
748 		goto err;
749 	}
750 
751 	/* We have all parameters now set up context */
752 	if (pkey) {
753 		if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey))
754 			goto err;
755 	} else {
756 		const EVP_MD *checkmd;
757 		if (EVP_PKEY_CTX_get_signature_md(pkctx, &checkmd) <= 0)
758 			goto err;
759 		if (EVP_MD_type(md) != EVP_MD_type(checkmd)) {
760 			RSAerror(RSA_R_DIGEST_DOES_NOT_MATCH);
761 			goto err;
762 		}
763 	}
764 
765 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) <= 0)
766 		goto err;
767 
768 	if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) <= 0)
769 		goto err;
770 
771 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
772 		goto err;
773 	/* Carry on */
774 	rv = 1;
775 
776  err:
777 	RSA_PSS_PARAMS_free(pss);
778 	return rv;
779 }
780 
781 int
782 rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,
783     const EVP_MD **pmgf1md, int *psaltlen)
784 {
785 	if (pss == NULL)
786 		return 0;
787 	*pmd = rsa_algor_to_md(pss->hashAlgorithm);
788 	if (*pmd == NULL)
789 		return 0;
790 	*pmgf1md = rsa_algor_to_md(pss->maskHash);
791 	if (*pmgf1md == NULL)
792 		return 0;
793 	if (pss->saltLength) {
794 		*psaltlen = ASN1_INTEGER_get(pss->saltLength);
795 		if (*psaltlen < 0) {
796 			RSAerror(RSA_R_INVALID_SALT_LENGTH);
797 			return 0;
798 		}
799 	} else {
800 		*psaltlen = 20;
801 	}
802 
803 	/*
804 	 * low-level routines support only trailer field 0xbc (value 1) and
805 	 * PKCS#1 says we should reject any other value anyway.
806 	 */
807 	if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
808 		RSAerror(RSA_R_INVALID_TRAILER);
809 		return 0;
810 	}
811 
812 	return 1;
813 }
814 
815 #ifndef OPENSSL_NO_CMS
816 static int
817 rsa_cms_verify(CMS_SignerInfo *si)
818 {
819 	int nid, nid2;
820 	X509_ALGOR *alg;
821 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
822 
823 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
824 	nid = OBJ_obj2nid(alg->algorithm);
825 	if (nid == EVP_PKEY_RSA_PSS)
826 		return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
827 	/* Only PSS allowed for PSS keys */
828 	if (pkey_ctx_is_pss(pkctx)) {
829 		RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
830 		return 0;
831 	}
832 	if (nid == NID_rsaEncryption)
833 		return 1;
834 	/* Workaround for some implementation that use a signature OID */
835 	if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
836 		if (nid2 == NID_rsaEncryption)
837 			return 1;
838 	}
839 	return 0;
840 }
841 #endif
842 
843 /*
844  * Customised RSA item verification routine. This is called when a signature
845  * is encountered requiring special handling. We currently only handle PSS.
846  */
847 static int
848 rsa_item_verify(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
849     X509_ALGOR *sigalg, ASN1_BIT_STRING *sig, EVP_PKEY *pkey)
850 {
851 	/* Sanity check: make sure it is PSS */
852 	if (OBJ_obj2nid(sigalg->algorithm) != EVP_PKEY_RSA_PSS) {
853 		RSAerror(RSA_R_UNSUPPORTED_SIGNATURE_TYPE);
854 		return -1;
855 	}
856 	if (rsa_pss_to_ctx(ctx, NULL, sigalg, pkey) > 0) {
857 		/* Carry on */
858 		return 2;
859 	}
860 	return -1;
861 }
862 
863 #ifndef OPENSSL_NO_CMS
864 static int
865 rsa_cms_sign(CMS_SignerInfo *si)
866 {
867 	int pad_mode = RSA_PKCS1_PADDING;
868 	X509_ALGOR *alg;
869 	EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
870 	ASN1_STRING *os = NULL;
871 
872 	CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
873 	if (pkctx) {
874 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
875 			return 0;
876 	}
877 	if (pad_mode == RSA_PKCS1_PADDING) {
878 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
879 		return 1;
880 	}
881 	/* We don't support it */
882 	if (pad_mode != RSA_PKCS1_PSS_PADDING)
883 		return 0;
884 	os = rsa_ctx_to_pss_string(pkctx);
885 	if (!os)
886 		return 0;
887 	X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os);
888 	return 1;
889 }
890 #endif
891 
892 static int
893 rsa_item_sign(EVP_MD_CTX *ctx, const ASN1_ITEM *it, void *asn,
894     X509_ALGOR *alg1, X509_ALGOR *alg2, ASN1_BIT_STRING *sig)
895 {
896 	EVP_PKEY_CTX *pkctx = ctx->pctx;
897 	int pad_mode;
898 
899 	if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
900 		return 0;
901 	if (pad_mode == RSA_PKCS1_PADDING)
902 		return 2;
903 	if (pad_mode == RSA_PKCS1_PSS_PADDING) {
904 		ASN1_STRING *os1 = NULL;
905 		os1 = rsa_ctx_to_pss_string(pkctx);
906 		if (!os1)
907 			return 0;
908 		/* Duplicate parameters if we have to */
909 		if (alg2) {
910 			ASN1_STRING *os2 = ASN1_STRING_dup(os1);
911 			if (!os2) {
912 				ASN1_STRING_free(os1);
913 				return 0;
914 			}
915 			X509_ALGOR_set0(alg2, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
916 			    V_ASN1_SEQUENCE, os2);
917 		}
918 		X509_ALGOR_set0(alg1, OBJ_nid2obj(EVP_PKEY_RSA_PSS),
919 		    V_ASN1_SEQUENCE, os1);
920 		return 3;
921 	}
922 	return 2;
923 }
924 
925 static int
926 rsa_pkey_check(const EVP_PKEY *pkey)
927 {
928 	return RSA_check_key(pkey->pkey.rsa);
929 }
930 
931 #ifndef OPENSSL_NO_CMS
932 static RSA_OAEP_PARAMS *
933 rsa_oaep_decode(const X509_ALGOR *alg)
934 {
935 	RSA_OAEP_PARAMS *oaep;
936 
937 	oaep = ASN1_TYPE_unpack_sequence(&RSA_OAEP_PARAMS_it, alg->parameter);
938 	if (oaep == NULL)
939 		return NULL;
940 
941 	if (oaep->maskGenFunc != NULL) {
942 		oaep->maskHash = rsa_mgf1_decode(oaep->maskGenFunc);
943 		if (oaep->maskHash == NULL) {
944 			RSA_OAEP_PARAMS_free(oaep);
945 			return NULL;
946 		}
947 	}
948 	return oaep;
949 }
950 
951 static int
952 rsa_cms_decrypt(CMS_RecipientInfo *ri)
953 {
954 	EVP_PKEY_CTX *pkctx;
955 	X509_ALGOR *cmsalg;
956 	int nid;
957 	int rv = -1;
958 	unsigned char *label = NULL;
959 	int labellen = 0;
960 	const EVP_MD *mgf1md = NULL, *md = NULL;
961 	RSA_OAEP_PARAMS *oaep;
962 
963 	pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
964 	if (pkctx == NULL)
965 		return 0;
966 	if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
967 		return -1;
968 	nid = OBJ_obj2nid(cmsalg->algorithm);
969 	if (nid == NID_rsaEncryption)
970 		return 1;
971 	if (nid != NID_rsaesOaep) {
972 		RSAerror(RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
973 		return -1;
974 	}
975 	/* Decode OAEP parameters */
976 	oaep = rsa_oaep_decode(cmsalg);
977 
978 	if (oaep == NULL) {
979 		RSAerror(RSA_R_INVALID_OAEP_PARAMETERS);
980 		goto err;
981 	}
982 
983 	mgf1md = rsa_algor_to_md(oaep->maskHash);
984 	if (mgf1md == NULL)
985 		goto err;
986 	md = rsa_algor_to_md(oaep->hashFunc);
987 	if (md == NULL)
988 		goto err;
989 
990 	if (oaep->pSourceFunc != NULL) {
991 		X509_ALGOR *plab = oaep->pSourceFunc;
992 
993 		if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
994 			RSAerror(RSA_R_UNSUPPORTED_LABEL_SOURCE);
995 			goto err;
996 		}
997 		if (plab->parameter->type != V_ASN1_OCTET_STRING) {
998 			RSAerror(RSA_R_INVALID_LABEL);
999 			goto err;
1000 		}
1001 
1002 		label = plab->parameter->value.octet_string->data;
1003 
1004 		/* Stop label being freed when OAEP parameters are freed */
1005 		/* XXX - this leaks label on error... */
1006 		plab->parameter->value.octet_string->data = NULL;
1007 		labellen = plab->parameter->value.octet_string->length;
1008 	}
1009 
1010 	if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
1011 		goto err;
1012 	if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
1013 		goto err;
1014 	if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
1015 		goto err;
1016 	if (EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
1017 		goto err;
1018 
1019 	rv = 1;
1020 
1021  err:
1022 	RSA_OAEP_PARAMS_free(oaep);
1023 	return rv;
1024 }
1025 
1026 static int
1027 rsa_cms_encrypt(CMS_RecipientInfo *ri)
1028 {
1029 	const EVP_MD *md, *mgf1md;
1030 	RSA_OAEP_PARAMS *oaep = NULL;
1031 	ASN1_STRING *os = NULL;
1032 	X509_ALGOR *alg;
1033 	EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
1034 	int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
1035 	unsigned char *label;
1036 
1037 	if (CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg) <= 0)
1038 		return 0;
1039 	if (pkctx) {
1040 		if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
1041 			return 0;
1042 	}
1043 	if (pad_mode == RSA_PKCS1_PADDING) {
1044 		X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
1045 		return 1;
1046 	}
1047 	/* Not supported */
1048 	if (pad_mode != RSA_PKCS1_OAEP_PADDING)
1049 		return 0;
1050 	if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
1051 		goto err;
1052 	if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
1053 		goto err;
1054 	labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
1055 	if (labellen < 0)
1056 		goto err;
1057 	oaep = RSA_OAEP_PARAMS_new();
1058 	if (oaep == NULL)
1059 		goto err;
1060 	if (!rsa_md_to_algor(&oaep->hashFunc, md))
1061 		goto err;
1062 	if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
1063 		goto err;
1064 	if (labellen > 0) {
1065 		ASN1_OCTET_STRING *los;
1066 		oaep->pSourceFunc = X509_ALGOR_new();
1067 		if (oaep->pSourceFunc == NULL)
1068 			goto err;
1069 		los = ASN1_OCTET_STRING_new();
1070 		if (los == NULL)
1071 			goto err;
1072 		if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
1073 			ASN1_OCTET_STRING_free(los);
1074 			goto err;
1075 		}
1076 		X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
1077 		    V_ASN1_OCTET_STRING, los);
1078 	}
1079 	/* create string with pss parameter encoding. */
1080 	if (!ASN1_item_pack(oaep, &RSA_OAEP_PARAMS_it, &os))
1081 		 goto err;
1082 	X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
1083 	os = NULL;
1084 	rv = 1;
1085  err:
1086 	RSA_OAEP_PARAMS_free(oaep);
1087 	ASN1_STRING_free(os);
1088 	return rv;
1089 }
1090 #endif
1091 
1092 const EVP_PKEY_ASN1_METHOD rsa_asn1_meths[] = {
1093 	{
1094 		.pkey_id = EVP_PKEY_RSA,
1095 		.pkey_base_id = EVP_PKEY_RSA,
1096 		.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1097 
1098 		.pem_str = "RSA",
1099 		.info = "OpenSSL RSA method",
1100 
1101 		.pub_decode = rsa_pub_decode,
1102 		.pub_encode = rsa_pub_encode,
1103 		.pub_cmp = rsa_pub_cmp,
1104 		.pub_print = rsa_pub_print,
1105 
1106 		.priv_decode = rsa_priv_decode,
1107 		.priv_encode = rsa_priv_encode,
1108 		.priv_print = rsa_priv_print,
1109 
1110 		.pkey_size = int_rsa_size,
1111 		.pkey_bits = rsa_bits,
1112 		.pkey_security_bits = rsa_security_bits,
1113 
1114 		.sig_print = rsa_sig_print,
1115 
1116 		.pkey_free = int_rsa_free,
1117 		.pkey_ctrl = rsa_pkey_ctrl,
1118 		.old_priv_decode = old_rsa_priv_decode,
1119 		.old_priv_encode = old_rsa_priv_encode,
1120 		.item_verify = rsa_item_verify,
1121 		.item_sign = rsa_item_sign,
1122 
1123 		.pkey_check = rsa_pkey_check,
1124 	},
1125 
1126 	{
1127 		.pkey_id = EVP_PKEY_RSA2,
1128 		.pkey_base_id = EVP_PKEY_RSA,
1129 		.pkey_flags = ASN1_PKEY_ALIAS,
1130 
1131 		.pkey_check = rsa_pkey_check,
1132 	},
1133 };
1134 
1135 const EVP_PKEY_ASN1_METHOD rsa_pss_asn1_meth = {
1136 	.pkey_id = EVP_PKEY_RSA_PSS,
1137 	.pkey_base_id = EVP_PKEY_RSA_PSS,
1138 	.pkey_flags = ASN1_PKEY_SIGPARAM_NULL,
1139 
1140 	.pem_str = "RSA-PSS",
1141 	.info = "OpenSSL RSA-PSS method",
1142 
1143 	.pub_decode = rsa_pub_decode,
1144 	.pub_encode = rsa_pub_encode,
1145 	.pub_cmp = rsa_pub_cmp,
1146 	.pub_print = rsa_pub_print,
1147 
1148 	.priv_decode = rsa_priv_decode,
1149 	.priv_encode = rsa_priv_encode,
1150 	.priv_print = rsa_priv_print,
1151 
1152 	.pkey_size = int_rsa_size,
1153 	.pkey_bits = rsa_bits,
1154 
1155 	.sig_print = rsa_sig_print,
1156 
1157 	.pkey_free = int_rsa_free,
1158 	.pkey_ctrl = rsa_pkey_ctrl,
1159 	.item_verify = rsa_item_verify,
1160 	.item_sign = rsa_item_sign
1161 };
1162