1 /* $OpenBSD: rsa_pmeth.c,v 1.32 2019/10/31 14:05:30 jsing Exp $ */
2 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
3  * project 2006.
4  */
5 /* ====================================================================
6  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  *
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in
17  *    the documentation and/or other materials provided with the
18  *    distribution.
19  *
20  * 3. All advertising materials mentioning features or use of this
21  *    software must display the following acknowledgment:
22  *    "This product includes software developed by the OpenSSL Project
23  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
24  *
25  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
26  *    endorse or promote products derived from this software without
27  *    prior written permission. For written permission, please contact
28  *    licensing@OpenSSL.org.
29  *
30  * 5. Products derived from this software may not be called "OpenSSL"
31  *    nor may "OpenSSL" appear in their names without prior written
32  *    permission of the OpenSSL Project.
33  *
34  * 6. Redistributions of any form whatsoever must retain the following
35  *    acknowledgment:
36  *    "This product includes software developed by the OpenSSL Project
37  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
38  *
39  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
40  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
42  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
43  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
44  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
45  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
46  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
47  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
48  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
49  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
50  * OF THE POSSIBILITY OF SUCH DAMAGE.
51  * ====================================================================
52  *
53  * This product includes cryptographic software written by Eric Young
54  * (eay@cryptsoft.com).  This product includes software written by Tim
55  * Hudson (tjh@cryptsoft.com).
56  *
57  */
58 
59 #include <limits.h>
60 #include <stdio.h>
61 #include <string.h>
62 
63 #include <openssl/opensslconf.h>
64 
65 #include <openssl/asn1t.h>
66 #include <openssl/bn.h>
67 #include <openssl/err.h>
68 #include <openssl/evp.h>
69 #include <openssl/rsa.h>
70 #include <openssl/x509.h>
71 #include <openssl/x509v3.h>
72 
73 #include "evp_locl.h"
74 #include "rsa_locl.h"
75 
76 /* RSA pkey context structure */
77 
78 typedef struct {
79 	/* Key gen parameters */
80 	int nbits;
81 	BIGNUM *pub_exp;
82 	/* Keygen callback info */
83 	int gentmp[2];
84 	/* RSA padding mode */
85 	int pad_mode;
86 	/* message digest */
87 	const EVP_MD *md;
88 	/* message digest for MGF1 */
89 	const EVP_MD *mgf1md;
90 	/* PSS salt length */
91 	int saltlen;
92 	/* Minimum salt length or -1 if no PSS parameter restriction */
93 	int min_saltlen;
94 	/* Temp buffer */
95 	unsigned char *tbuf;
96 	/* OAEP label */
97 	unsigned char *oaep_label;
98 	size_t oaep_labellen;
99 } RSA_PKEY_CTX;
100 
101 /* True if PSS parameters are restricted */
102 #define rsa_pss_restricted(rctx) (rctx->min_saltlen != -1)
103 
104 static int
105 pkey_rsa_init(EVP_PKEY_CTX *ctx)
106 {
107 	RSA_PKEY_CTX *rctx;
108 
109 	if ((rctx = calloc(1, sizeof(RSA_PKEY_CTX))) == NULL)
110 		return 0;
111 
112 	rctx->nbits = 2048;
113 
114 	if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
115 		rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
116 	else
117 		rctx->pad_mode = RSA_PKCS1_PADDING;
118 
119 	/* Maximum for sign, auto for verify */
120 	rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
121 	rctx->min_saltlen = -1;
122 
123 	ctx->data = rctx;
124 	ctx->keygen_info = rctx->gentmp;
125 	ctx->keygen_info_count = 2;
126 
127 	return 1;
128 }
129 
130 static int
131 pkey_rsa_copy(EVP_PKEY_CTX *dst, EVP_PKEY_CTX *src)
132 {
133 	RSA_PKEY_CTX *dctx, *sctx;
134 
135 	if (!pkey_rsa_init(dst))
136 		return 0;
137 
138 	sctx = src->data;
139 	dctx = dst->data;
140 	dctx->nbits = sctx->nbits;
141 	if (sctx->pub_exp != NULL) {
142 		BN_free(dctx->pub_exp);
143 		if ((dctx->pub_exp = BN_dup(sctx->pub_exp)) == NULL)
144 			return 0;
145 	}
146 	dctx->pad_mode = sctx->pad_mode;
147 	dctx->md = sctx->md;
148 	dctx->mgf1md = sctx->mgf1md;
149 	if (sctx->oaep_label != NULL) {
150 		free(dctx->oaep_label);
151 		if ((dctx->oaep_label = calloc(1, sctx->oaep_labellen)) == NULL)
152 			return 0;
153 		memcpy(dctx->oaep_label, sctx->oaep_label, sctx->oaep_labellen);
154 		dctx->oaep_labellen = sctx->oaep_labellen;
155 	}
156 
157 	return 1;
158 }
159 
160 static int
161 setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk)
162 {
163 	if (ctx->tbuf != NULL)
164 		return 1;
165 	if ((ctx->tbuf = calloc(1, EVP_PKEY_size(pk->pkey))) == NULL) {
166 		RSAerror(ERR_R_MALLOC_FAILURE);
167 		return 0;
168 	}
169 	return 1;
170 }
171 
172 static void
173 pkey_rsa_cleanup(EVP_PKEY_CTX *ctx)
174 {
175 	RSA_PKEY_CTX *rctx = ctx->data;
176 
177 	if (rctx) {
178 		BN_free(rctx->pub_exp);
179 		free(rctx->tbuf);
180 		free(rctx->oaep_label);
181 		free(rctx);
182 	}
183 }
184 
185 static int
186 pkey_rsa_sign(EVP_PKEY_CTX *ctx, unsigned char *sig, size_t *siglen,
187     const unsigned char *tbs, size_t tbslen)
188 {
189 	int ret;
190 	RSA_PKEY_CTX *rctx = ctx->data;
191 	RSA *rsa = ctx->pkey->pkey.rsa;
192 
193 	if (rctx->md) {
194 		if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
195 			RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
196 			return -1;
197 		}
198 
199 		if (rctx->pad_mode == RSA_X931_PADDING) {
200 			if ((size_t)EVP_PKEY_size(ctx->pkey) < tbslen + 1) {
201 				RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
202 				return -1;
203 			}
204 			if (!setup_tbuf(rctx, ctx)) {
205 				RSAerror(ERR_R_MALLOC_FAILURE);
206 				return -1;
207 			}
208 			memcpy(rctx->tbuf, tbs, tbslen);
209 			rctx->tbuf[tbslen] =
210 			    RSA_X931_hash_id(EVP_MD_type(rctx->md));
211 			ret = RSA_private_encrypt(tbslen + 1, rctx->tbuf, sig,
212 			    rsa, RSA_X931_PADDING);
213 		} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
214 			unsigned int sltmp;
215 
216 			ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig,
217 			    &sltmp, rsa);
218 			if (ret <= 0)
219 				return ret;
220 			ret = sltmp;
221 		} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
222 			if (!setup_tbuf(rctx, ctx))
223 				return -1;
224 			if (!RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf,
225 			    tbs, rctx->md, rctx->mgf1md, rctx->saltlen))
226 				return -1;
227 			ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf,
228 			    sig, rsa, RSA_NO_PADDING);
229 		} else {
230 			return -1;
231 		}
232 	} else {
233 		ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa,
234 		    rctx->pad_mode);
235 	}
236 	if (ret < 0)
237 		return ret;
238 	*siglen = ret;
239 	return 1;
240 }
241 
242 static int
243 pkey_rsa_verifyrecover(EVP_PKEY_CTX *ctx, unsigned char *rout, size_t *routlen,
244     const unsigned char *sig, size_t siglen)
245 {
246 	int ret;
247 	RSA_PKEY_CTX *rctx = ctx->data;
248 
249 	if (rctx->md) {
250 		if (rctx->pad_mode == RSA_X931_PADDING) {
251 			if (!setup_tbuf(rctx, ctx))
252 				return -1;
253 			ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
254 			    ctx->pkey->pkey.rsa, RSA_X931_PADDING);
255 			if (ret < 1)
256 				return 0;
257 			ret--;
258 			if (rctx->tbuf[ret] !=
259 			    RSA_X931_hash_id(EVP_MD_type(rctx->md))) {
260 				RSAerror(RSA_R_ALGORITHM_MISMATCH);
261 				return 0;
262 			}
263 			if (ret != EVP_MD_size(rctx->md)) {
264 				RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
265 				return 0;
266 			}
267 			if (rout)
268 				memcpy(rout, rctx->tbuf, ret);
269 		} else if (rctx->pad_mode == RSA_PKCS1_PADDING) {
270 			size_t sltmp;
271 
272 			ret = int_rsa_verify(EVP_MD_type(rctx->md), NULL, 0,
273 			    rout, &sltmp, sig, siglen, ctx->pkey->pkey.rsa);
274 			if (ret <= 0)
275 				return 0;
276 			ret = sltmp;
277 		} else {
278 			return -1;
279 		}
280 	} else {
281 		ret = RSA_public_decrypt(siglen, sig, rout, ctx->pkey->pkey.rsa,
282 		    rctx->pad_mode);
283 	}
284 	if (ret < 0)
285 		return ret;
286 	*routlen = ret;
287 	return 1;
288 }
289 
290 static int
291 pkey_rsa_verify(EVP_PKEY_CTX *ctx, const unsigned char *sig, size_t siglen,
292     const unsigned char *tbs, size_t tbslen)
293 {
294 	RSA_PKEY_CTX *rctx = ctx->data;
295 	RSA *rsa = ctx->pkey->pkey.rsa;
296 	size_t rslen;
297 
298 	if (rctx->md) {
299 		if (rctx->pad_mode == RSA_PKCS1_PADDING)
300 			return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen,
301 			    sig, siglen, rsa);
302 		if (tbslen != (size_t)EVP_MD_size(rctx->md)) {
303 			RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
304 			return -1;
305 		}
306 		if (rctx->pad_mode == RSA_X931_PADDING) {
307 			if (pkey_rsa_verifyrecover(ctx, NULL, &rslen, sig,
308 			    siglen) <= 0)
309 				return 0;
310 		} else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) {
311 			int ret;
312 
313 			if (!setup_tbuf(rctx, ctx))
314 				return -1;
315 			ret = RSA_public_decrypt(siglen, sig, rctx->tbuf,
316 			    rsa, RSA_NO_PADDING);
317 			if (ret <= 0)
318 				return 0;
319 			ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md,
320 			    rctx->mgf1md, rctx->tbuf, rctx->saltlen);
321 			if (ret <= 0)
322 				return 0;
323 			return 1;
324 		} else {
325 			return -1;
326 		}
327 	} else {
328 		if (!setup_tbuf(rctx, ctx))
329 			return -1;
330 		rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa,
331 		    rctx->pad_mode);
332 		if (rslen == 0)
333 			return 0;
334 	}
335 
336 	if (rslen != tbslen || timingsafe_bcmp(tbs, rctx->tbuf, rslen))
337 		return 0;
338 
339 	return 1;
340 }
341 
342 static int
343 pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
344     const unsigned char *in, size_t inlen)
345 {
346 	RSA_PKEY_CTX *rctx = ctx->data;
347 	int ret;
348 
349 	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
350 		int klen = RSA_size(ctx->pkey->pkey.rsa);
351 		if (!setup_tbuf(rctx, ctx))
352 			return -1;
353 		if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen,
354 		    in, inlen, rctx->oaep_label, rctx->oaep_labellen,
355 		    rctx->md, rctx->mgf1md))
356 			return -1;
357 		ret = RSA_public_encrypt(klen, rctx->tbuf, out,
358 		    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
359 	} else {
360 		ret = RSA_public_encrypt(inlen, in, out, ctx->pkey->pkey.rsa,
361 		    rctx->pad_mode);
362 	}
363 	if (ret < 0)
364 		return ret;
365 	*outlen = ret;
366 	return 1;
367 }
368 
369 static int
370 pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, unsigned char *out, size_t *outlen,
371     const unsigned char *in, size_t inlen)
372 {
373 	int ret;
374 	RSA_PKEY_CTX *rctx = ctx->data;
375 
376 	if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) {
377 		if (!setup_tbuf(rctx, ctx))
378 			return -1;
379 		ret = RSA_private_decrypt(inlen, in, rctx->tbuf,
380 		    ctx->pkey->pkey.rsa, RSA_NO_PADDING);
381 		if (ret <= 0)
382 			return ret;
383 		ret = RSA_padding_check_PKCS1_OAEP_mgf1(out, ret, rctx->tbuf,
384 		    ret, ret, rctx->oaep_label, rctx->oaep_labellen, rctx->md,
385 		    rctx->mgf1md);
386 	} else {
387 		ret = RSA_private_decrypt(inlen, in, out, ctx->pkey->pkey.rsa,
388 		    rctx->pad_mode);
389 	}
390 	if (ret < 0)
391 		return ret;
392 	*outlen = ret;
393 	return 1;
394 }
395 
396 static int
397 check_padding_md(const EVP_MD *md, int padding)
398 {
399 	if (md == NULL)
400 		return 1;
401 
402 	if (padding == RSA_NO_PADDING) {
403 		RSAerror(RSA_R_INVALID_PADDING_MODE);
404 		return 0;
405 	}
406 
407 	if (padding == RSA_X931_PADDING) {
408 		if (RSA_X931_hash_id(EVP_MD_type(md)) == -1) {
409 			RSAerror(RSA_R_INVALID_X931_DIGEST);
410 			return 0;
411 		}
412 	} else {
413 		/* List of all supported RSA digests. */
414 		switch(EVP_MD_type(md)) {
415 		case NID_sha1:
416 		case NID_sha224:
417 		case NID_sha256:
418 		case NID_sha384:
419 		case NID_sha512:
420 		case NID_md5:
421 		case NID_md5_sha1:
422 		case NID_md4:
423 		case NID_ripemd160:
424 			return 1;
425 
426 		default:
427 			RSAerror(RSA_R_INVALID_DIGEST);
428 			return 0;
429 		}
430 	}
431 
432 	return 1;
433 }
434 
435 static int
436 pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
437 {
438 	RSA_PKEY_CTX *rctx = ctx->data;
439 
440 	switch (type) {
441 	case EVP_PKEY_CTRL_RSA_PADDING:
442 		if (p1 >= RSA_PKCS1_PADDING && p1 <= RSA_PKCS1_PSS_PADDING) {
443 			if (!check_padding_md(rctx->md, p1))
444 				return 0;
445 			if (p1 == RSA_PKCS1_PSS_PADDING) {
446 				if (!(ctx->operation &
447 				    (EVP_PKEY_OP_SIGN | EVP_PKEY_OP_VERIFY)))
448 					goto bad_pad;
449 				if (!rctx->md)
450 					rctx->md = EVP_sha1();
451 			} else if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
452 				goto bad_pad;
453 			}
454 			if (p1 == RSA_PKCS1_OAEP_PADDING) {
455 				if (!(ctx->operation & EVP_PKEY_OP_TYPE_CRYPT))
456 					goto bad_pad;
457 				if (!rctx->md)
458 					rctx->md = EVP_sha1();
459 			}
460 			rctx->pad_mode = p1;
461 			return 1;
462 		}
463  bad_pad:
464 		RSAerror(RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
465 		return -2;
466 
467 	case EVP_PKEY_CTRL_GET_RSA_PADDING:
468 		*(int *)p2 = rctx->pad_mode;
469 		return 1;
470 
471 	case EVP_PKEY_CTRL_RSA_PSS_SALTLEN:
472 	case EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN:
473 		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING) {
474 			RSAerror(RSA_R_INVALID_PSS_SALTLEN);
475 			return -2;
476 		}
477 		if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
478 			*(int *)p2 = rctx->saltlen;
479 		} else {
480 			if (p1 < RSA_PSS_SALTLEN_MAX)
481 				return -2;
482 			if (rsa_pss_restricted(rctx)) {
483 				if (p1 == RSA_PSS_SALTLEN_AUTO &&
484 				    ctx->operation == EVP_PKEY_OP_VERIFY) {
485 					RSAerror(RSA_R_INVALID_PSS_SALTLEN);
486 					return -2;
487 				}
488 				if ((p1 == RSA_PSS_SALTLEN_DIGEST &&
489 				    rctx->min_saltlen > EVP_MD_size(rctx->md)) ||
490 				    (p1 >= 0 && p1 < rctx->min_saltlen)) {
491 					RSAerror(RSA_R_PSS_SALTLEN_TOO_SMALL);
492 					return 0;
493 				}
494 			}
495 			rctx->saltlen = p1;
496 		}
497 		return 1;
498 
499 	case EVP_PKEY_CTRL_RSA_KEYGEN_BITS:
500 		if (p1 < RSA_MIN_MODULUS_BITS) {
501 			RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
502 			return -2;
503 		}
504 		rctx->nbits = p1;
505 		return 1;
506 
507 	case EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP:
508 		if (p2 == NULL || !BN_is_odd((BIGNUM *)p2) ||
509 		    BN_is_one((BIGNUM *)p2)) {
510 			RSAerror(RSA_R_BAD_E_VALUE);
511 			return -2;
512 		}
513 		BN_free(rctx->pub_exp);
514 		rctx->pub_exp = p2;
515 		return 1;
516 
517 	case EVP_PKEY_CTRL_RSA_OAEP_MD:
518 	case EVP_PKEY_CTRL_GET_RSA_OAEP_MD:
519 		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
520 			RSAerror(RSA_R_INVALID_PADDING_MODE);
521 			return -2;
522 		}
523 		if (type == EVP_PKEY_CTRL_GET_RSA_OAEP_MD)
524 			*(const EVP_MD **)p2 = rctx->md;
525 		else
526 			rctx->md = p2;
527 		return 1;
528 
529 	case EVP_PKEY_CTRL_MD:
530 		if (!check_padding_md(p2, rctx->pad_mode))
531 			return 0;
532 		if (rsa_pss_restricted(rctx)) {
533 			if (EVP_MD_type(rctx->md) == EVP_MD_type(p2))
534 				return 1;
535 			RSAerror(RSA_R_DIGEST_NOT_ALLOWED);
536 			return 0;
537 		}
538 		rctx->md = p2;
539 		return 1;
540 
541 	case EVP_PKEY_CTRL_GET_MD:
542 		*(const EVP_MD **)p2 = rctx->md;
543 		return 1;
544 
545 	case EVP_PKEY_CTRL_RSA_MGF1_MD:
546 	case EVP_PKEY_CTRL_GET_RSA_MGF1_MD:
547 		if (rctx->pad_mode != RSA_PKCS1_PSS_PADDING &&
548 		    rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
549 			RSAerror(RSA_R_INVALID_MGF1_MD);
550 			return -2;
551 		}
552 		if (type == EVP_PKEY_CTRL_GET_RSA_MGF1_MD) {
553 			if (rctx->mgf1md)
554 				*(const EVP_MD **)p2 = rctx->mgf1md;
555 			else
556 				*(const EVP_MD **)p2 = rctx->md;
557 		} else {
558 			if (rsa_pss_restricted(rctx)) {
559 				if (EVP_MD_type(rctx->mgf1md) == EVP_MD_type(p2))
560 					return 1;
561 				RSAerror(RSA_R_MGF1_DIGEST_NOT_ALLOWED);
562 				return 0;
563 			}
564 			rctx->mgf1md = p2;
565 		}
566 		return 1;
567 
568 	case EVP_PKEY_CTRL_RSA_OAEP_LABEL:
569 		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
570 			RSAerror(RSA_R_INVALID_PADDING_MODE);
571 			return -2;
572 		}
573 		free(rctx->oaep_label);
574 		if (p2 != NULL && p1 > 0) {
575 			rctx->oaep_label = p2;
576 			rctx->oaep_labellen = p1;
577 		} else {
578 			rctx->oaep_label = NULL;
579 			rctx->oaep_labellen = 0;
580 		}
581 		return 1;
582 
583 	case EVP_PKEY_CTRL_GET_RSA_OAEP_LABEL:
584 		if (rctx->pad_mode != RSA_PKCS1_OAEP_PADDING) {
585 			RSAerror(RSA_R_INVALID_PADDING_MODE);
586 			return -2;
587 		}
588 		*(unsigned char **)p2 = rctx->oaep_label;
589 		return rctx->oaep_labellen;
590 
591 	case EVP_PKEY_CTRL_DIGESTINIT:
592 	case EVP_PKEY_CTRL_PKCS7_SIGN:
593 #ifndef OPENSSL_NO_CMS
594 	case EVP_PKEY_CTRL_CMS_SIGN:
595 #endif
596 		return 1;
597 
598 	case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
599 	case EVP_PKEY_CTRL_PKCS7_DECRYPT:
600 #ifndef OPENSSL_NO_CMS
601 	case EVP_PKEY_CTRL_CMS_DECRYPT:
602 	case EVP_PKEY_CTRL_CMS_ENCRYPT:
603 #endif
604 		if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
605 			return 1;
606 
607 	/* fall through */
608 	case EVP_PKEY_CTRL_PEER_KEY:
609 		RSAerror(RSA_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
610 		return -2;
611 
612 	default:
613 		return -2;
614 
615 	}
616 }
617 
618 static int
619 pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx, const char *type, const char *value)
620 {
621 	if (!value) {
622 		RSAerror(RSA_R_VALUE_MISSING);
623 		return 0;
624 	}
625 	if (!strcmp(type, "rsa_padding_mode")) {
626 		int pm;
627 		if (!strcmp(value, "pkcs1"))
628 			pm = RSA_PKCS1_PADDING;
629 		else if (!strcmp(value, "none"))
630 			pm = RSA_NO_PADDING;
631 		else if (!strcmp(value, "oeap"))
632 			pm = RSA_PKCS1_OAEP_PADDING;
633 		else if (!strcmp(value, "oaep"))
634 			pm = RSA_PKCS1_OAEP_PADDING;
635 		else if (!strcmp(value, "x931"))
636 			pm = RSA_X931_PADDING;
637 		else if (!strcmp(value, "pss"))
638 			pm = RSA_PKCS1_PSS_PADDING;
639 		else {
640 			RSAerror(RSA_R_UNKNOWN_PADDING_TYPE);
641 			return -2;
642 		}
643 		return EVP_PKEY_CTX_set_rsa_padding(ctx, pm);
644 	}
645 
646 	if (strcmp(type, "rsa_pss_saltlen") == 0) {
647 		int saltlen;
648 
649 		if (!strcmp(value, "digest"))
650 			saltlen = RSA_PSS_SALTLEN_DIGEST;
651 		else if (!strcmp(value, "max"))
652 			saltlen = RSA_PSS_SALTLEN_MAX;
653 		else if (!strcmp(value, "auto"))
654 			saltlen = RSA_PSS_SALTLEN_AUTO;
655 		else
656 			saltlen = atoi(value);
657 		return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
658 	}
659 
660 	if (strcmp(type, "rsa_keygen_bits") == 0) {
661 		int nbits = atoi(value);
662 
663 		return EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, nbits);
664 	}
665 
666 	if (strcmp(type, "rsa_keygen_pubexp") == 0) {
667 		BIGNUM *pubexp = NULL;
668 		int ret;
669 
670 		if (!BN_asc2bn(&pubexp, value))
671 			return 0;
672 		ret = EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp);
673 		if (ret <= 0)
674 			BN_free(pubexp);
675 		return ret;
676 	}
677 
678 	if (strcmp(type, "rsa_mgf1_md") == 0)
679 		return EVP_PKEY_CTX_md(ctx,
680 		    EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
681 		    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
682 
683 	if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
684 		if (strcmp(type, "rsa_pss_keygen_mgf1_md") == 0)
685 			return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
686 			    EVP_PKEY_CTRL_RSA_MGF1_MD, value);
687 
688 		if (strcmp(type, "rsa_pss_keygen_md") == 0)
689 			return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
690 			    EVP_PKEY_CTRL_MD, value);
691 
692 		if (strcmp(type, "rsa_pss_keygen_saltlen") == 0) {
693 			int saltlen = atoi(value);
694 
695 			return EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, saltlen);
696 		}
697 	}
698 
699 	if (strcmp(type, "rsa_oaep_md") == 0)
700 		return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_TYPE_CRYPT,
701 		    EVP_PKEY_CTRL_RSA_OAEP_MD, value);
702 
703 	if (strcmp(type, "rsa_oaep_label") == 0) {
704 		unsigned char *lab;
705 		long lablen;
706 		int ret;
707 
708 		if ((lab = string_to_hex(value, &lablen)) == NULL)
709 			return 0;
710 		ret = EVP_PKEY_CTX_set0_rsa_oaep_label(ctx, lab, lablen);
711 		if (ret <= 0)
712 			free(lab);
713 
714 		return ret;
715 	}
716 
717 	return -2;
718 }
719 
720 /* Set PSS parameters when generating a key, if necessary. */
721 static int
722 rsa_set_pss_param(RSA *rsa, EVP_PKEY_CTX *ctx)
723 {
724 	RSA_PKEY_CTX *rctx = ctx->data;
725 
726 	if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
727 		return 1;
728 
729 	/* If all parameters are default values then do not set PSS. */
730 	if (rctx->md == NULL && rctx->mgf1md == NULL &&
731 	    rctx->saltlen == RSA_PSS_SALTLEN_AUTO)
732 		return 1;
733 
734 	rsa->pss = rsa_pss_params_create(rctx->md, rctx->mgf1md,
735 	    rctx->saltlen == RSA_PSS_SALTLEN_AUTO ? 0 : rctx->saltlen);
736 	if (rsa->pss == NULL)
737 		return 0;
738 
739 	return 1;
740 }
741 
742 static int
743 pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
744 {
745 	RSA *rsa = NULL;
746 	RSA_PKEY_CTX *rctx = ctx->data;
747 	BN_GENCB *pcb, cb;
748 	int ret;
749 
750 	if (rctx->pub_exp == NULL) {
751 		if ((rctx->pub_exp = BN_new()) == NULL)
752 			return 0;
753 		if (!BN_set_word(rctx->pub_exp, RSA_F4))
754 			return 0;
755 	}
756 	if ((rsa = RSA_new()) == NULL)
757 		return 0;
758 	if (ctx->pkey_gencb != NULL) {
759 		pcb = &cb;
760 		evp_pkey_set_cb_translate(pcb, ctx);
761 	} else {
762 		pcb = NULL;
763 	}
764 	ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
765 	if (ret > 0 && !rsa_set_pss_param(rsa, ctx)) {
766 		RSA_free(rsa);
767 		return 0;
768 	}
769 	if (ret > 0)
770 		EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa);
771 	else
772 		RSA_free(rsa);
773 	return ret;
774 }
775 
776 const EVP_PKEY_METHOD rsa_pkey_meth = {
777 	.pkey_id = EVP_PKEY_RSA,
778 	.flags = EVP_PKEY_FLAG_AUTOARGLEN,
779 
780 	.init = pkey_rsa_init,
781 	.copy = pkey_rsa_copy,
782 	.cleanup = pkey_rsa_cleanup,
783 
784 	.keygen = pkey_rsa_keygen,
785 
786 	.sign = pkey_rsa_sign,
787 
788 	.verify = pkey_rsa_verify,
789 
790 	.verify_recover = pkey_rsa_verifyrecover,
791 
792 	.encrypt = pkey_rsa_encrypt,
793 
794 	.decrypt = pkey_rsa_decrypt,
795 
796 	.ctrl = pkey_rsa_ctrl,
797 	.ctrl_str = pkey_rsa_ctrl_str
798 };
799 
800 /*
801  * Called for PSS sign or verify initialisation: checks PSS parameter
802  * sanity and sets any restrictions on key usage.
803  */
804 
805 static int
806 pkey_pss_init(EVP_PKEY_CTX *ctx)
807 {
808 	RSA *rsa;
809 	RSA_PKEY_CTX *rctx = ctx->data;
810 	const EVP_MD *md;
811 	const EVP_MD *mgf1md;
812 	int min_saltlen, max_saltlen;
813 
814 	/* Should never happen */
815 	if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
816 		return 0;
817 	rsa = ctx->pkey->pkey.rsa;
818 
819 	/* If no restrictions just return */
820 	if (rsa->pss == NULL)
821 		return 1;
822 
823 	/* Get and check parameters */
824 	if (!rsa_pss_get_param(rsa->pss, &md, &mgf1md, &min_saltlen))
825 		return 0;
826 
827 	/* See if minimum salt length exceeds maximum possible */
828 	max_saltlen = RSA_size(rsa) - EVP_MD_size(md);
829 	if ((RSA_bits(rsa) & 0x7) == 1)
830 		max_saltlen--;
831 	if (min_saltlen > max_saltlen) {
832 		RSAerror(RSA_R_INVALID_SALT_LENGTH);
833 		return 0;
834 	}
835 	rctx->min_saltlen = min_saltlen;
836 
837 	/*
838 	 * Set PSS restrictions as defaults: we can then block any attempt to
839 	 * use invalid values in pkey_rsa_ctrl
840 	 */
841 
842 	rctx->md = md;
843 	rctx->mgf1md = mgf1md;
844 	rctx->saltlen = min_saltlen;
845 
846 	return 1;
847 }
848 
849 const EVP_PKEY_METHOD rsa_pss_pkey_meth = {
850 	.pkey_id = EVP_PKEY_RSA_PSS,
851 	.flags = EVP_PKEY_FLAG_AUTOARGLEN,
852 
853 	.init = pkey_rsa_init,
854 	.copy = pkey_rsa_copy,
855 	.cleanup = pkey_rsa_cleanup,
856 
857 	.keygen = pkey_rsa_keygen,
858 
859 	.sign_init = pkey_pss_init,
860 	.sign = pkey_rsa_sign,
861 
862 	.verify_init = pkey_pss_init,
863 	.verify = pkey_rsa_verify,
864 
865 	.ctrl = pkey_rsa_ctrl,
866 	.ctrl_str = pkey_rsa_ctrl_str
867 };
868 
869