xref: /illumos-gate/usr/src/uts/common/crypto/io/rsa.c (revision 3db86aab)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * RSA provider for the Kernel Cryptographic Framework (KCF)
31  */
32 
33 #include <sys/types.h>
34 #include <sys/systm.h>
35 #include <sys/modctl.h>
36 #include <sys/cmn_err.h>
37 #include <sys/ddi.h>
38 #include <sys/crypto/spi.h>
39 #include <sys/sysmacros.h>
40 #include <sys/strsun.h>
41 #include <sys/md5.h>
42 #include <sys/sha1.h>
43 #include <sys/sha2.h>
44 #include <sys/random.h>
45 #include "rsa_impl.h"
46 
47 extern struct mod_ops mod_cryptoops;
48 
49 /*
50  * Module linkage information for the kernel.
51  */
52 static struct modlcrypto modlcrypto = {
53 	&mod_cryptoops,
54 	"RSA Kernel SW Provider %I%"
55 };
56 
57 static struct modlinkage modlinkage = {
58 	MODREV_1,
59 	(void *)&modlcrypto,
60 	NULL
61 };
62 
63 /*
64  * CSPI information (entry points, provider info, etc.)
65  */
66 typedef enum rsa_mech_type {
67 	RSA_PKCS_MECH_INFO_TYPE,	/* SUN_CKM_RSA_PKCS */
68 	RSA_X_509_MECH_INFO_TYPE,	/* SUN_CKM_RSA_X_509 */
69 	MD5_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_MD5_RSA_PKCS */
70 	SHA1_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA1_RSA_PKCS */
71 	SHA256_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA256_RSA_PKCS */
72 	SHA384_RSA_PKCS_MECH_INFO_TYPE,	/* SUN_SHA384_RSA_PKCS */
73 	SHA512_RSA_PKCS_MECH_INFO_TYPE	/* SUN_SHA512_RSA_PKCS */
74 } rsa_mech_type_t;
75 
76 /*
77  * Context for RSA_PKCS and RSA_X_509 mechanisms.
78  */
79 typedef struct rsa_ctx {
80 	rsa_mech_type_t	mech_type;
81 	crypto_key_t *key;
82 	size_t keychunk_size;
83 } rsa_ctx_t;
84 
85 /*
86  * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
87  */
88 typedef struct digest_rsa_ctx {
89 	rsa_mech_type_t	mech_type;
90 	crypto_key_t *key;
91 	size_t keychunk_size;
92 	union {
93 		MD5_CTX md5ctx;
94 		SHA1_CTX sha1ctx;
95 		SHA2_CTX sha2ctx;
96 	} dctx_u;
97 } digest_rsa_ctx_t;
98 
99 #define	md5_ctx		dctx_u.md5ctx
100 #define	sha1_ctx	dctx_u.sha1ctx
101 #define	sha2_ctx	dctx_u.sha2ctx
102 
103 /*
104  * Mechanism info structure passed to KCF during registration.
105  */
106 static crypto_mech_info_t rsa_mech_info_tab[] = {
107 	/* RSA_PKCS */
108 	{SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
109 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
110 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
111 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
112 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
113 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
114 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
115 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
116 
117 	/* RSA_X_509 */
118 	{SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
119 	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
120 	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
121 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
122 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
123 	    CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
124 	    CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
125 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
126 
127 	/* MD5_RSA_PKCS */
128 	{SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
129 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
130 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
131 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
132 
133 	/* SHA1_RSA_PKCS */
134 	{SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
135 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
136 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
137 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
138 
139 	/* SHA256_RSA_PKCS */
140 	{SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
141 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
142 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
143 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
144 
145 	/* SHA384_RSA_PKCS */
146 	{SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
147 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
148 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
149 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
150 
151 	/* SHA512_RSA_PKCS */
152 	{SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
153 	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
154 	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
155 	    RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
156 
157 };
158 
159 #define	RSA_VALID_MECH(mech)					\
160 	(((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||		\
161 	(mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||		\
162 	(mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||	\
163 	(mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE ||	\
164 	(mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE ||	\
165 	(mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE ||	\
166 	(mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
167 
168 /* operations are in-place if the output buffer is NULL */
169 #define	RSA_ARG_INPLACE(input, output)				\
170 	if ((output) == NULL)					\
171 		(output) = (input);
172 
173 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
174 
175 static crypto_control_ops_t rsa_control_ops = {
176 	rsa_provider_status
177 };
178 
179 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
180     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
181 static int rsa_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
182     crypto_req_handle_t);
183 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
184     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
185     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
186 static int rsa_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
187     crypto_req_handle_t);
188 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
189     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
190     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
191 
192 /*
193  * The RSA mechanisms do not have multiple-part cipher operations.
194  * So, the update and final routines are set to NULL.
195  */
196 static crypto_cipher_ops_t rsa_cipher_ops = {
197 	rsa_common_init,
198 	rsa_encrypt,
199 	NULL,
200 	NULL,
201 	rsa_encrypt_atomic,
202 	rsa_common_init,
203 	rsa_decrypt,
204 	NULL,
205 	NULL,
206 	rsa_decrypt_atomic
207 };
208 
209 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
210     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
211 static int rsa_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
212     crypto_req_handle_t);
213 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
214     crypto_req_handle_t);
215 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
216     crypto_req_handle_t);
217 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
218     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
219     crypto_spi_ctx_template_t, crypto_req_handle_t);
220 
221 /*
222  * We use the same routine for sign_init and sign_recover_init fields
223  * as they do the same thing. Same holds for sign and sign_recover fields,
224  * and sign_atomic and sign_recover_atomic fields.
225  */
226 static crypto_sign_ops_t rsa_sign_ops = {
227 	rsa_sign_verify_common_init,
228 	rsa_sign,
229 	rsa_sign_update,
230 	rsa_sign_final,
231 	rsa_sign_atomic,
232 	rsa_sign_verify_common_init,
233 	rsa_sign,
234 	rsa_sign_atomic
235 };
236 
237 static int rsa_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
238     crypto_req_handle_t);
239 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
240     crypto_req_handle_t);
241 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
242     crypto_req_handle_t);
243 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
244     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
245     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
246 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
247     crypto_data_t *, crypto_req_handle_t);
248 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
249     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
250     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
251     crypto_req_handle_t);
252 
253 /*
254  * We use the same routine (rsa_sign_verify_common_init) for verify_init
255  * and verify_recover_init fields as they do the same thing.
256  */
257 static crypto_verify_ops_t rsa_verify_ops = {
258 	rsa_sign_verify_common_init,
259 	rsa_verify,
260 	rsa_verify_update,
261 	rsa_verify_final,
262 	rsa_verify_atomic,
263 	rsa_sign_verify_common_init,
264 	rsa_verify_recover,
265 	rsa_verify_recover_atomic
266 };
267 
268 static int rsa_free_context(crypto_ctx_t *);
269 
270 static crypto_ctx_ops_t rsa_ctx_ops = {
271 	NULL,
272 	rsa_free_context
273 };
274 
275 static crypto_ops_t rsa_crypto_ops = {
276 	&rsa_control_ops,
277 	NULL,
278 	&rsa_cipher_ops,
279 	NULL,
280 	&rsa_sign_ops,
281 	&rsa_verify_ops,
282 	NULL,
283 	NULL,
284 	NULL,
285 	NULL,
286 	NULL,
287 	NULL,
288 	NULL,
289 	&rsa_ctx_ops
290 };
291 
292 static crypto_provider_info_t rsa_prov_info = {
293 	CRYPTO_SPI_VERSION_1,
294 	"RSA Software Provider",
295 	CRYPTO_SW_PROVIDER,
296 	{&modlinkage},
297 	NULL,
298 	&rsa_crypto_ops,
299 	sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
300 	rsa_mech_info_tab
301 };
302 
303 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
304     crypto_data_t *, crypto_data_t *, int);
305 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
306     crypto_data_t *, crypto_data_t *, int);
307 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
308     crypto_data_t *, crypto_data_t *, int);
309 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
310     crypto_data_t *, crypto_data_t *, int);
311 static int compare_data(crypto_data_t *, uchar_t *);
312 
313 /* EXPORT DELETE START */
314 
315 static int core_rsa_encrypt(crypto_key_t *, uchar_t *,
316     int, uchar_t *, int, int);
317 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int,
318     uchar_t *, int);
319 
320 /* EXPORT DELETE END */
321 
322 static crypto_kcf_provider_handle_t rsa_prov_handle = NULL;
323 
324 int
325 _init(void)
326 {
327 	int ret;
328 
329 	/*
330 	 * Register with KCF. If the registration fails, return error.
331 	 */
332 	if ((ret = crypto_register_provider(&rsa_prov_info,
333 	    &rsa_prov_handle)) != CRYPTO_SUCCESS) {
334 		cmn_err(CE_WARN, "rsa _init: crypto_register_provider()"
335 		    "failed (0x%x)", ret);
336 		return (EACCES);
337 	}
338 
339 	if ((ret = mod_install(&modlinkage)) != 0) {
340 		int rv;
341 
342 		ASSERT(rsa_prov_handle != NULL);
343 		/* We should not return if the unregister returns busy. */
344 		while ((rv = crypto_unregister_provider(rsa_prov_handle))
345 		    == CRYPTO_BUSY) {
346 			cmn_err(CE_WARN, "rsa _init: "
347 			    "crypto_unregister_provider() "
348 			    "failed (0x%x). Retrying.", rv);
349 			/* wait 10 seconds and try again. */
350 			delay(10 * drv_usectohz(1000000));
351 		}
352 	}
353 
354 	return (ret);
355 }
356 
357 int
358 _fini(void)
359 {
360 	int ret;
361 
362 	/*
363 	 * Unregister from KCF if previous registration succeeded.
364 	 */
365 	if (rsa_prov_handle != NULL) {
366 		if ((ret = crypto_unregister_provider(rsa_prov_handle)) !=
367 		    CRYPTO_SUCCESS) {
368 			cmn_err(CE_WARN, "rsa _fini: "
369 			    "crypto_unregister_provider() "
370 			    "failed (0x%x)", ret);
371 			return (EBUSY);
372 		}
373 		rsa_prov_handle = NULL;
374 	}
375 
376 	return (mod_remove(&modlinkage));
377 }
378 
379 int
380 _info(struct modinfo *modinfop)
381 {
382 	return (mod_info(&modlinkage, modinfop));
383 }
384 
385 /* ARGSUSED */
386 static void
387 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
388 {
389 	*status = CRYPTO_PROVIDER_READY;
390 }
391 
392 /*
393  * Utility routine to look up a attribute of type, 'type',
394  * in the key.
395  */
396 static int
397 get_key_attr(crypto_key_t *key, crypto_attr_type_t type,
398     uchar_t **value, ssize_t *value_len)
399 {
400 	int i;
401 
402 	ASSERT(key->ck_format == CRYPTO_KEY_ATTR_LIST);
403 	for (i = 0; i < key->ck_count; i++) {
404 		if (key->ck_attrs[i].oa_type == type) {
405 			*value = (uchar_t *)key->ck_attrs[i].oa_value;
406 			*value_len = key->ck_attrs[i].oa_value_len;
407 			return (CRYPTO_SUCCESS);
408 		}
409 	}
410 
411 	return (CRYPTO_FAILED);
412 }
413 
414 static int
415 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
416 {
417 	int rv = CRYPTO_FAILED;
418 
419 /* EXPORT DELETE START */
420 
421 	uchar_t *modulus;
422 	ssize_t modulus_len; /* In bytes */
423 
424 	if (!RSA_VALID_MECH(mechanism))
425 		return (CRYPTO_MECHANISM_INVALID);
426 
427 	/*
428 	 * We only support RSA keys that are passed as a list of
429 	 * object attributes.
430 	 */
431 	if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
432 		return (CRYPTO_KEY_TYPE_INCONSISTENT);
433 	}
434 
435 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
436 	    &modulus_len)) != CRYPTO_SUCCESS) {
437 		return (rv);
438 	}
439 	if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
440 	    modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
441 		return (CRYPTO_KEY_SIZE_RANGE);
442 
443 /* EXPORT DELETE END */
444 
445 	return (rv);
446 }
447 
448 void
449 kmemset(uint8_t *buf, char pattern, size_t len)
450 {
451 	int i = 0;
452 
453 	while (i < len)
454 		buf[i++] = pattern;
455 }
456 
457 /*
458  * This function guarantees to return non-zero random numbers.
459  * This is needed as the /dev/urandom kernel interface,
460  * random_get_pseudo_bytes(), may return zeros.
461  */
462 int
463 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
464 {
465 	int rv;
466 	size_t ebc = 0; /* count of extra bytes in extrarand */
467 	size_t i = 0;
468 	uint8_t extrarand[32];
469 	size_t extrarand_len;
470 
471 	if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
472 		return (rv);
473 
474 	/*
475 	 * Walk through the returned random numbers pointed by ran_out,
476 	 * and look for any random number which is zero.
477 	 * If we find zero, call random_get_pseudo_bytes() to generate
478 	 * another 32 random numbers pool. Replace any zeros in ran_out[]
479 	 * from the random number in pool.
480 	 */
481 	while (i < ran_len) {
482 		if (ran_out[i] != 0) {
483 			i++;
484 			continue;
485 		}
486 
487 		/*
488 		 * Note that it is 'while' so we are guaranteed a
489 		 * non-zero value on exit.
490 		 */
491 		if (ebc == 0) {
492 			/* refresh extrarand */
493 			extrarand_len = sizeof (extrarand);
494 			if ((rv = random_get_pseudo_bytes(extrarand,
495 			    extrarand_len)) != 0) {
496 				return (rv);
497 			}
498 
499 			ebc = extrarand_len;
500 		}
501 		/* Replace zero with byte from extrarand. */
502 		-- ebc;
503 
504 		/*
505 		 * The new random byte zero/non-zero will be checked in
506 		 * the next pass through the loop.
507 		 */
508 		ran_out[i] = extrarand[ebc];
509 	}
510 
511 	return (CRYPTO_SUCCESS);
512 }
513 
514 typedef enum cmd_type {
515 	COPY_FROM_DATA,
516 	COPY_TO_DATA,
517 	COMPARE_TO_DATA,
518 	MD5_DIGEST_DATA,
519 	SHA1_DIGEST_DATA,
520 	SHA2_DIGEST_DATA
521 } cmd_type_t;
522 
523 /*
524  * Utility routine to apply the command, 'cmd', to the
525  * data in the uio structure.
526  */
527 static int
528 process_uio_data(crypto_data_t *data, uchar_t *buf, int len,
529     cmd_type_t cmd, void *digest_ctx)
530 {
531 	uio_t *uiop = data->cd_uio;
532 	off_t offset = data->cd_offset;
533 	size_t length = len;
534 	uint_t vec_idx;
535 	size_t cur_len;
536 	uchar_t *datap;
537 
538 	ASSERT(data->cd_format == CRYPTO_DATA_UIO);
539 	if (uiop->uio_segflg != UIO_SYSSPACE) {
540 		return (CRYPTO_ARGUMENTS_BAD);
541 	}
542 
543 	/*
544 	 * Jump to the first iovec containing data to be
545 	 * processed.
546 	 */
547 	for (vec_idx = 0; vec_idx < uiop->uio_iovcnt &&
548 	    offset >= uiop->uio_iov[vec_idx].iov_len;
549 	    offset -= uiop->uio_iov[vec_idx++].iov_len);
550 
551 	if (vec_idx == uiop->uio_iovcnt) {
552 		/*
553 		 * The caller specified an offset that is larger than
554 		 * the total size of the buffers it provided.
555 		 */
556 		return (CRYPTO_DATA_LEN_RANGE);
557 	}
558 
559 	while (vec_idx < uiop->uio_iovcnt && length > 0) {
560 		cur_len = MIN(uiop->uio_iov[vec_idx].iov_len -
561 		    offset, length);
562 
563 		datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base +
564 		    offset);
565 		switch (cmd) {
566 		case COPY_FROM_DATA:
567 			bcopy(datap, buf, cur_len);
568 			buf += cur_len;
569 			break;
570 		case COPY_TO_DATA:
571 			bcopy(buf, datap, cur_len);
572 			buf += cur_len;
573 			break;
574 		case COMPARE_TO_DATA:
575 			if (bcmp(datap, buf, cur_len))
576 				return (CRYPTO_SIGNATURE_INVALID);
577 			buf += cur_len;
578 			break;
579 		case MD5_DIGEST_DATA:
580 			MD5Update(digest_ctx, datap, cur_len);
581 			break;
582 		case SHA1_DIGEST_DATA:
583 			SHA1Update(digest_ctx, datap, cur_len);
584 			break;
585 		case SHA2_DIGEST_DATA:
586 			SHA2Update(digest_ctx, datap, cur_len);
587 			break;
588 		}
589 
590 		length -= cur_len;
591 		vec_idx++;
592 		offset = 0;
593 	}
594 
595 	if (vec_idx == uiop->uio_iovcnt && length > 0) {
596 		/*
597 		 * The end of the specified iovec's was reached but
598 		 * the length requested could not be processed.
599 		 */
600 		switch (cmd) {
601 		case COPY_TO_DATA:
602 			data->cd_length = len;
603 			return (CRYPTO_BUFFER_TOO_SMALL);
604 		default:
605 			return (CRYPTO_DATA_LEN_RANGE);
606 		}
607 	}
608 
609 	return (CRYPTO_SUCCESS);
610 }
611 
612 /*
613  * Utility routine to apply the command, 'cmd', to the
614  * data in the mblk structure.
615  */
616 static int
617 process_mblk_data(crypto_data_t *data, uchar_t *buf, int len,
618     cmd_type_t cmd, void *digest_ctx)
619 {
620 	off_t offset = data->cd_offset;
621 	size_t length = len;
622 	mblk_t *mp;
623 	size_t cur_len;
624 	uchar_t *datap;
625 
626 	ASSERT(data->cd_format == CRYPTO_DATA_MBLK);
627 	/*
628 	 * Jump to the first mblk_t containing data to be processed.
629 	 */
630 	for (mp = data->cd_mp; mp != NULL && offset >= MBLKL(mp);
631 	    offset -= MBLKL(mp), mp = mp->b_cont);
632 	if (mp == NULL) {
633 		/*
634 		 * The caller specified an offset that is larger
635 		 * than the total size of the buffers it provided.
636 		 */
637 		return (CRYPTO_DATA_LEN_RANGE);
638 	}
639 
640 	/*
641 	 * Now do the processing on the mblk chain.
642 	 */
643 	while (mp != NULL && length > 0) {
644 		cur_len = MIN(MBLKL(mp) - offset, length);
645 
646 		datap = (uchar_t *)(mp->b_rptr + offset);
647 		switch (cmd) {
648 		case COPY_FROM_DATA:
649 			bcopy(datap, buf, cur_len);
650 			buf += cur_len;
651 			break;
652 		case COPY_TO_DATA:
653 			bcopy(buf, datap, cur_len);
654 			buf += cur_len;
655 			break;
656 		case COMPARE_TO_DATA:
657 			if (bcmp(datap, buf, cur_len))
658 				return (CRYPTO_SIGNATURE_INVALID);
659 			buf += cur_len;
660 			break;
661 		case MD5_DIGEST_DATA:
662 			MD5Update(digest_ctx, datap, cur_len);
663 			break;
664 		case SHA1_DIGEST_DATA:
665 			SHA1Update(digest_ctx, datap, cur_len);
666 			break;
667 		case SHA2_DIGEST_DATA:
668 			SHA2Update(digest_ctx, datap, cur_len);
669 			break;
670 		}
671 
672 		length -= cur_len;
673 		offset = 0;
674 		mp = mp->b_cont;
675 	}
676 
677 	if (mp == NULL && length > 0) {
678 		/*
679 		 * The end of the mblk was reached but the length
680 		 * requested could not be processed.
681 		 */
682 		switch (cmd) {
683 		case COPY_TO_DATA:
684 			data->cd_length = len;
685 			return (CRYPTO_BUFFER_TOO_SMALL);
686 		default:
687 			return (CRYPTO_DATA_LEN_RANGE);
688 		}
689 	}
690 
691 	return (CRYPTO_SUCCESS);
692 }
693 
694 static int
695 compare_data(crypto_data_t *data, uchar_t *buf)
696 {
697 	int len;
698 	uchar_t *dptr;
699 
700 	len = data->cd_length;
701 	switch (data->cd_format) {
702 	case CRYPTO_DATA_RAW:
703 		dptr = (uchar_t *)(data->cd_raw.iov_base +
704 		    data->cd_offset);
705 
706 		return (bcmp(dptr, buf, len));
707 
708 	case CRYPTO_DATA_UIO:
709 		return (process_uio_data(data, buf, len,
710 		    COMPARE_TO_DATA, NULL));
711 
712 	case CRYPTO_DATA_MBLK:
713 		return (process_mblk_data(data, buf, len,
714 		    COMPARE_TO_DATA, NULL));
715 	}
716 
717 	return (CRYPTO_FAILED);
718 }
719 
720 /*
721  * Utility routine to copy a buffer to a crypto_data structure.
722  */
723 static int
724 put_output_data(uchar_t *buf, crypto_data_t *output, int len)
725 {
726 	switch (output->cd_format) {
727 	case CRYPTO_DATA_RAW:
728 		if (output->cd_raw.iov_len < len) {
729 			output->cd_length = len;
730 			return (CRYPTO_BUFFER_TOO_SMALL);
731 		}
732 		bcopy(buf, (uchar_t *)(output->cd_raw.iov_base +
733 		    output->cd_offset), len);
734 		break;
735 
736 	case CRYPTO_DATA_UIO:
737 		return (process_uio_data(output, buf, len, COPY_TO_DATA, NULL));
738 
739 	case CRYPTO_DATA_MBLK:
740 		return (process_mblk_data(output, buf, len,
741 		    COPY_TO_DATA, NULL));
742 
743 	default:
744 		return (CRYPTO_ARGUMENTS_BAD);
745 	}
746 
747 	return (CRYPTO_SUCCESS);
748 }
749 
750 /*
751  * Utility routine to get data from a crypto_data structure.
752  *
753  * '*dptr' contains a pointer to a buffer on return. 'buf'
754  * is allocated by the caller and is ignored for CRYPTO_DATA_RAW case.
755  */
756 static int
757 get_input_data(crypto_data_t *input, uchar_t **dptr, uchar_t *buf)
758 {
759 	int rv;
760 
761 	switch (input->cd_format) {
762 	case CRYPTO_DATA_RAW:
763 		if (input->cd_raw.iov_len < input->cd_length)
764 			return (CRYPTO_ARGUMENTS_BAD);
765 		*dptr = (uchar_t *)(input->cd_raw.iov_base +
766 		    input->cd_offset);
767 		break;
768 
769 	case CRYPTO_DATA_UIO:
770 		if ((rv = process_uio_data(input, buf, input->cd_length,
771 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
772 			return (rv);
773 		*dptr = buf;
774 		break;
775 
776 	case CRYPTO_DATA_MBLK:
777 		if ((rv = process_mblk_data(input, buf, input->cd_length,
778 		    COPY_FROM_DATA, NULL)) != CRYPTO_SUCCESS)
779 			return (rv);
780 		*dptr = buf;
781 		break;
782 
783 	default:
784 		return (CRYPTO_ARGUMENTS_BAD);
785 	}
786 
787 	return (CRYPTO_SUCCESS);
788 }
789 
790 static int
791 copy_key_to_ctx(crypto_key_t *in_key, rsa_ctx_t *ctx, int kmflag)
792 {
793 	int i, count;
794 	size_t len;
795 	caddr_t attr_val;
796 	crypto_object_attribute_t *k_attrs = NULL;
797 
798 	ASSERT(in_key->ck_format == CRYPTO_KEY_ATTR_LIST);
799 
800 	count = in_key->ck_count;
801 	/* figure out how much memory to allocate for everything */
802 	len = sizeof (crypto_key_t) +
803 	    count * sizeof (crypto_object_attribute_t);
804 	for (i = 0; i < count; i++) {
805 		len += roundup(in_key->ck_attrs[i].oa_value_len,
806 		    sizeof (caddr_t));
807 	}
808 
809 	/* one big allocation for everything */
810 	ctx->key = kmem_alloc(len, kmflag);
811 	if (ctx->key == NULL)
812 		return (CRYPTO_HOST_MEMORY);
813 	k_attrs = (crypto_object_attribute_t *)((caddr_t)(ctx->key) +
814 	    sizeof (crypto_key_t));
815 
816 	attr_val = (caddr_t)k_attrs +
817 	    count * sizeof (crypto_object_attribute_t);
818 	for (i = 0; i < count; i++) {
819 		k_attrs[i].oa_type = in_key->ck_attrs[i].oa_type;
820 		bcopy(in_key->ck_attrs[i].oa_value, attr_val,
821 		    in_key->ck_attrs[i].oa_value_len);
822 		k_attrs[i].oa_value = attr_val;
823 		k_attrs[i].oa_value_len = in_key->ck_attrs[i].oa_value_len;
824 		attr_val += roundup(k_attrs[i].oa_value_len, sizeof (caddr_t));
825 	}
826 
827 	ctx->keychunk_size = len;	/* save the size to be freed */
828 	ctx->key->ck_format = CRYPTO_KEY_ATTR_LIST;
829 	ctx->key->ck_count = count;
830 	ctx->key->ck_attrs = k_attrs;
831 
832 	return (CRYPTO_SUCCESS);
833 }
834 
835 /* ARGSUSED */
836 static int
837 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
838     crypto_key_t *key, crypto_spi_ctx_template_t template,
839     crypto_req_handle_t req)
840 {
841 	int rv;
842 	int kmflag;
843 	rsa_ctx_t *ctxp;
844 
845 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
846 		return (rv);
847 
848 	/*
849 	 * Allocate a RSA context.
850 	 */
851 	kmflag = crypto_kmflag(req);
852 	if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
853 		return (CRYPTO_HOST_MEMORY);
854 
855 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
856 		kmem_free(ctxp, sizeof (rsa_ctx_t));
857 		return (rv);
858 	}
859 	ctxp->mech_type = mechanism->cm_type;
860 
861 	ctx->cc_provider_private = ctxp;
862 
863 	return (CRYPTO_SUCCESS);
864 }
865 
866 /* ARGSUSED */
867 static int
868 rsa_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
869     crypto_data_t *ciphertext, crypto_req_handle_t req)
870 {
871 	int rv;
872 	rsa_ctx_t *ctxp;
873 
874 	ASSERT(ctx->cc_provider_private != NULL);
875 	ctxp = ctx->cc_provider_private;
876 
877 	RSA_ARG_INPLACE(plaintext, ciphertext);
878 
879 	/*
880 	 * Note on the KM_SLEEP flag passed to the routine below -
881 	 * rsa_encrypt() is a single-part encryption routine which is
882 	 * currently usable only by /dev/crypto. Since /dev/crypto calls are
883 	 * always synchronous, we can safely pass KM_SLEEP here.
884 	 */
885 	rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
886 	    ciphertext, KM_SLEEP);
887 
888 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
889 		(void) rsa_free_context(ctx);
890 
891 	return (rv);
892 }
893 
894 /* ARGSUSED */
895 static int
896 rsa_encrypt_atomic(crypto_provider_handle_t provider,
897     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
898     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
899     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
900 {
901 	int rv;
902 
903 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
904 		return (rv);
905 	RSA_ARG_INPLACE(plaintext, ciphertext);
906 
907 	return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
908 	    ciphertext, crypto_kmflag(req)));
909 }
910 
911 static int
912 rsa_free_context(crypto_ctx_t *ctx)
913 {
914 	rsa_ctx_t *ctxp = ctx->cc_provider_private;
915 
916 	if (ctxp != NULL) {
917 		bzero(ctxp->key, ctxp->keychunk_size);
918 		kmem_free(ctxp->key, ctxp->keychunk_size);
919 
920 		if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
921 		    ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
922 			kmem_free(ctxp, sizeof (rsa_ctx_t));
923 		else
924 			kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
925 
926 		ctx->cc_provider_private = NULL;
927 	}
928 
929 	return (CRYPTO_SUCCESS);
930 }
931 
932 static int
933 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
934     crypto_data_t *plaintext, crypto_data_t *ciphertext, int kmflag)
935 {
936 	int rv = CRYPTO_FAILED;
937 
938 /* EXPORT DELETE START */
939 
940 	int plen;
941 	uchar_t *ptptr;
942 	uchar_t *modulus;
943 	ssize_t modulus_len;
944 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
945 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
946 	uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
947 
948 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
949 	    &modulus_len)) != CRYPTO_SUCCESS) {
950 		return (rv);
951 	}
952 
953 	plen = plaintext->cd_length;
954 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
955 		if (plen > (modulus_len - MIN_PKCS1_PADLEN))
956 			return (CRYPTO_DATA_LEN_RANGE);
957 	} else {
958 		if (plen > modulus_len)
959 			return (CRYPTO_DATA_LEN_RANGE);
960 	}
961 
962 	/*
963 	 * Output buf len must not be less than RSA modulus size.
964 	 */
965 	if (ciphertext->cd_length < modulus_len) {
966 		ciphertext->cd_length = modulus_len;
967 		return (CRYPTO_BUFFER_TOO_SMALL);
968 	}
969 
970 	ASSERT(plaintext->cd_length <= sizeof (tmp_data));
971 	if ((rv = get_input_data(plaintext, &ptptr, tmp_data))
972 	    != CRYPTO_SUCCESS)
973 		return (rv);
974 
975 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
976 		rv = soft_encrypt_rsa_pkcs_encode(ptptr, plen,
977 		    plain_data, modulus_len);
978 
979 		if (rv != CRYPTO_SUCCESS)
980 			return (rv);
981 	} else {
982 		bzero(plain_data, modulus_len - plen);
983 		bcopy(ptptr, &plain_data[modulus_len - plen], plen);
984 	}
985 
986 	rv = core_rsa_encrypt(key, plain_data, modulus_len,
987 	    cipher_data, kmflag, 1);
988 	if (rv == CRYPTO_SUCCESS) {
989 		/* copy out to ciphertext */
990 		if ((rv = put_output_data(cipher_data,
991 		    ciphertext, modulus_len)) != CRYPTO_SUCCESS)
992 			return (rv);
993 
994 		ciphertext->cd_length = modulus_len;
995 	}
996 
997 /* EXPORT DELETE END */
998 
999 	return (rv);
1000 }
1001 
1002 /* EXPORT DELETE START */
1003 
1004 static int
1005 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
1006     int in_len, uchar_t *out, int kmflag, int is_public)
1007 {
1008 	int rv;
1009 	uchar_t *expo, *modulus;
1010 	ssize_t	expo_len;
1011 	ssize_t modulus_len;
1012 	BIGNUM msg;
1013 	RSAkey *rsakey;
1014 
1015 	if (is_public) {
1016 		if ((rv = get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT, &expo,
1017 		    &expo_len)) != CRYPTO_SUCCESS)
1018 			return (rv);
1019 	} else {
1020 		/*
1021 		 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
1022 		 * RSA secret key. See the comments in core_rsa_decrypt
1023 		 * routine which calls this routine with a private key.
1024 		 */
1025 		if ((rv = get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT, &expo,
1026 		    &expo_len)) != CRYPTO_SUCCESS)
1027 			return (rv);
1028 	}
1029 
1030 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1031 	    &modulus_len)) != CRYPTO_SUCCESS) {
1032 		return (rv);
1033 	}
1034 
1035 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1036 	if (rsakey == NULL)
1037 		return (CRYPTO_HOST_MEMORY);
1038 
1039 	/* psize and qsize for RSA_key_init is in bits. */
1040 	if (RSA_key_init(rsakey, modulus_len * 4, modulus_len * 4) != BIG_OK) {
1041 		rv = CRYPTO_HOST_MEMORY;
1042 		goto clean1;
1043 	}
1044 
1045 	/* Size for big_init is in (32-bit) words. */
1046 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1047 	    (int)sizeof (uint32_t)) != BIG_OK) {
1048 		rv = CRYPTO_HOST_MEMORY;
1049 		goto clean2;
1050 	}
1051 
1052 	/* Convert octet string exponent to big integer format. */
1053 	bytestring2bignum(&(rsakey->e), expo, expo_len);
1054 
1055 	/* Convert octet string modulus to big integer format. */
1056 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1057 
1058 	/* Convert octet string input data to big integer format. */
1059 	bytestring2bignum(&msg, in, in_len);
1060 
1061 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1062 		rv = CRYPTO_DATA_LEN_RANGE;
1063 		goto clean3;
1064 	}
1065 
1066 	/* Perform RSA computation on big integer input data. */
1067 	if (big_modexp(&msg, &msg, &(rsakey->e), &(rsakey->n), NULL)
1068 	    != BIG_OK) {
1069 		rv = CRYPTO_HOST_MEMORY;
1070 		goto clean3;
1071 	}
1072 
1073 	/* Convert the big integer output data to octet string. */
1074 	bignum2bytestring(out, &msg, modulus_len);
1075 
1076 	/*
1077 	 * Should not free modulus and expo as both are just pointers
1078 	 * to an attribute value buffer from the caller.
1079 	 */
1080 clean3:
1081 	big_finish(&msg);
1082 clean2:
1083 	RSA_key_finish(rsakey);
1084 clean1:
1085 	kmem_free(rsakey, sizeof (RSAkey));
1086 
1087 	return (rv);
1088 }
1089 
1090 /* EXPORT DELETE END */
1091 
1092 /* ARGSUSED */
1093 static int
1094 rsa_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
1095     crypto_data_t *plaintext, crypto_req_handle_t req)
1096 {
1097 	int rv;
1098 	rsa_ctx_t *ctxp;
1099 
1100 	ASSERT(ctx->cc_provider_private != NULL);
1101 	ctxp = ctx->cc_provider_private;
1102 
1103 	RSA_ARG_INPLACE(ciphertext, plaintext);
1104 
1105 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1106 	rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
1107 	    ciphertext, plaintext, KM_SLEEP);
1108 
1109 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1110 		(void) rsa_free_context(ctx);
1111 
1112 	return (rv);
1113 }
1114 
1115 /* ARGSUSED */
1116 static int
1117 rsa_decrypt_atomic(crypto_provider_handle_t provider,
1118     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1119     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
1120     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1121 {
1122 	int rv;
1123 
1124 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1125 		return (rv);
1126 	RSA_ARG_INPLACE(ciphertext, plaintext);
1127 
1128 	return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
1129 	    plaintext, crypto_kmflag(req)));
1130 }
1131 
1132 static int
1133 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1134     crypto_data_t *ciphertext, crypto_data_t *plaintext, int kmflag)
1135 {
1136 	int rv = CRYPTO_FAILED;
1137 
1138 /* EXPORT DELETE START */
1139 
1140 	int plain_len;
1141 	uchar_t *ctptr;
1142 	uchar_t *modulus;
1143 	ssize_t modulus_len;
1144 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1145 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1146 
1147 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1148 	    &modulus_len)) != CRYPTO_SUCCESS) {
1149 		return (rv);
1150 	}
1151 
1152 	/*
1153 	 * Ciphertext length must be equal to RSA modulus size.
1154 	 */
1155 	if (ciphertext->cd_length != modulus_len)
1156 		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
1157 
1158 	ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
1159 	if ((rv = get_input_data(ciphertext, &ctptr, tmp_data))
1160 	    != CRYPTO_SUCCESS)
1161 		return (rv);
1162 
1163 	rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data, kmflag);
1164 	if (rv == CRYPTO_SUCCESS) {
1165 		plain_len = modulus_len;
1166 
1167 		if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1168 			/* Strip off the PKCS block formatting data. */
1169 			rv = soft_decrypt_rsa_pkcs_decode(plain_data,
1170 			    &plain_len);
1171 			if (rv != CRYPTO_SUCCESS)
1172 				return (rv);
1173 		}
1174 
1175 		if (plain_len > plaintext->cd_length) {
1176 			plaintext->cd_length = plain_len;
1177 			return (CRYPTO_BUFFER_TOO_SMALL);
1178 		}
1179 
1180 		if ((rv = put_output_data(plain_data + modulus_len - plain_len,
1181 		    plaintext, plain_len)) != CRYPTO_SUCCESS)
1182 			return (rv);
1183 
1184 		plaintext->cd_length = plain_len;
1185 	}
1186 
1187 /* EXPORT DELETE END */
1188 
1189 	return (rv);
1190 }
1191 
1192 /* EXPORT DELETE START */
1193 
1194 static int
1195 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len,
1196     uchar_t *out, int kmflag)
1197 {
1198 	int rv;
1199 	uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
1200 	ssize_t modulus_len;
1201 	ssize_t	prime1_len, prime2_len;
1202 	ssize_t	expo1_len, expo2_len, coef_len;
1203 	BIGNUM msg;
1204 	RSAkey *rsakey;
1205 
1206 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1207 	    &modulus_len)) != CRYPTO_SUCCESS) {
1208 		return (rv);
1209 	}
1210 
1211 	/*
1212 	 * The following attributes are not required to be
1213 	 * present in a RSA secret key. If any of them is not present
1214 	 * we call the encrypt routine with a flag indicating use of
1215 	 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
1216 	 * a required attribute for a RSA secret key.
1217 	 */
1218 	if ((get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
1219 		!= CRYPTO_SUCCESS) ||
1220 	    (get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
1221 		!= CRYPTO_SUCCESS) ||
1222 	    (get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
1223 		!= CRYPTO_SUCCESS) ||
1224 	    (get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
1225 		!= CRYPTO_SUCCESS) ||
1226 	    (get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
1227 		!= CRYPTO_SUCCESS)) {
1228 		return (core_rsa_encrypt(key, in, in_len, out, kmflag, 0));
1229 	}
1230 
1231 	rsakey = kmem_alloc(sizeof (RSAkey), kmflag);
1232 	if (rsakey == NULL)
1233 		return (CRYPTO_HOST_MEMORY);
1234 
1235 	/* psize and qsize for RSA_key_init is in bits. */
1236 	if (RSA_key_init(rsakey, prime2_len * 8, prime1_len * 8) != BIG_OK) {
1237 		rv = CRYPTO_HOST_MEMORY;
1238 		goto clean1;
1239 	}
1240 
1241 	/* Size for big_init is in (32-bit) words. */
1242 	if (big_init(&msg, (in_len + (int)sizeof (uint32_t) - 1) /
1243 	    (int)sizeof (uint32_t)) != BIG_OK) {
1244 		rv = CRYPTO_HOST_MEMORY;
1245 		goto clean2;
1246 	}
1247 
1248 	/* Convert octet string input data to big integer format. */
1249 	bytestring2bignum(&msg, in, in_len);
1250 
1251 	/* Convert octet string modulus to big integer format. */
1252 	bytestring2bignum(&(rsakey->n), modulus, modulus_len);
1253 
1254 	if (big_cmp_abs(&msg, &(rsakey->n)) > 0) {
1255 		rv = CRYPTO_DATA_LEN_RANGE;
1256 		goto clean3;
1257 	}
1258 
1259 	/* Convert the rest of private key attributes to big integer format. */
1260 	bytestring2bignum(&(rsakey->dmodpminus1), expo2, expo2_len);
1261 	bytestring2bignum(&(rsakey->dmodqminus1), expo1, expo1_len);
1262 	bytestring2bignum(&(rsakey->p), prime2, prime2_len);
1263 	bytestring2bignum(&(rsakey->q), prime1, prime1_len);
1264 	bytestring2bignum(&(rsakey->pinvmodq), coef, coef_len);
1265 
1266 	if ((big_cmp_abs(&(rsakey->dmodpminus1), &(rsakey->p)) > 0) ||
1267 	    (big_cmp_abs(&(rsakey->dmodqminus1), &(rsakey->q)) > 0) ||
1268 	    (big_cmp_abs(&(rsakey->pinvmodq), &(rsakey->q)) > 0)) {
1269 		rv = CRYPTO_KEY_SIZE_RANGE;
1270 		goto clean3;
1271 	}
1272 
1273 	/* Perform RSA computation on big integer input data. */
1274 	if (big_modexp_crt(&msg, &msg, &(rsakey->dmodpminus1),
1275 	    &(rsakey->dmodqminus1), &(rsakey->p), &(rsakey->q),
1276 	    &(rsakey->pinvmodq), NULL, NULL) != BIG_OK) {
1277 		rv = CRYPTO_HOST_MEMORY;
1278 		goto clean3;
1279 	}
1280 
1281 	/* Convert the big integer output data to octet string. */
1282 	bignum2bytestring(out, &msg, modulus_len);
1283 
1284 	/*
1285 	 * Should not free modulus and friends as they are just pointers
1286 	 * to an attribute value buffer from the caller.
1287 	 */
1288 clean3:
1289 	big_finish(&msg);
1290 clean2:
1291 	RSA_key_finish(rsakey);
1292 clean1:
1293 	kmem_free(rsakey, sizeof (RSAkey));
1294 
1295 	return (rv);
1296 }
1297 
1298 /* EXPORT DELETE END */
1299 
1300 /* ARGSUSED */
1301 static int
1302 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
1303     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
1304     crypto_req_handle_t req)
1305 {
1306 	int rv;
1307 	int kmflag;
1308 	rsa_ctx_t *ctxp;
1309 	digest_rsa_ctx_t *dctxp;
1310 
1311 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1312 		return (rv);
1313 
1314 	/*
1315 	 * Allocate a RSA context.
1316 	 */
1317 	kmflag = crypto_kmflag(req);
1318 	switch (mechanism->cm_type) {
1319 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1320 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1321 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1322 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1323 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1324 		dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
1325 		ctxp = (rsa_ctx_t *)dctxp;
1326 		break;
1327 	default:
1328 		ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
1329 		break;
1330 	}
1331 
1332 	if (ctxp == NULL)
1333 		return (CRYPTO_HOST_MEMORY);
1334 
1335 	ctxp->mech_type = mechanism->cm_type;
1336 	if ((rv = copy_key_to_ctx(key, ctxp, kmflag)) != CRYPTO_SUCCESS) {
1337 		switch (mechanism->cm_type) {
1338 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1339 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1340 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1341 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1342 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1343 			kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
1344 			break;
1345 		default:
1346 			kmem_free(ctxp, sizeof (rsa_ctx_t));
1347 			break;
1348 		}
1349 		return (rv);
1350 	}
1351 
1352 	switch (mechanism->cm_type) {
1353 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1354 		MD5Init(&(dctxp->md5_ctx));
1355 		break;
1356 
1357 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1358 		SHA1Init(&(dctxp->sha1_ctx));
1359 		break;
1360 
1361 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1362 		SHA2Init(SHA256, &(dctxp->sha2_ctx));
1363 		break;
1364 
1365 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1366 		SHA2Init(SHA384, &(dctxp->sha2_ctx));
1367 		break;
1368 
1369 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1370 		SHA2Init(SHA512, &(dctxp->sha2_ctx));
1371 		break;
1372 	}
1373 
1374 	ctx->cc_provider_private = ctxp;
1375 
1376 	return (CRYPTO_SUCCESS);
1377 }
1378 
1379 #define	SHA1_DIGEST_SIZE 20
1380 #define	MD5_DIGEST_SIZE 16
1381 
1382 #define	INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)	\
1383 	(data).cd_format = CRYPTO_DATA_RAW;		\
1384 	(data).cd_offset = 0;				\
1385 	(data).cd_raw.iov_base = (char *)base;		\
1386 	(data).cd_raw.iov_len = len;			\
1387 	(data).cd_length = cd_len;
1388 
1389 #define	DO_UPDATE	0x01
1390 #define	DO_FINAL	0x02
1391 #define	DO_MD5		0x04
1392 #define	DO_SHA1		0x08
1393 #define	DO_SIGN		0x10
1394 #define	DO_VERIFY	0x20
1395 #define	DO_SHA2		0x40
1396 
1397 static int
1398 digest_data(crypto_data_t *data, void *dctx, uchar_t *digest,
1399     uchar_t flag)
1400 {
1401 	int rv, dlen;
1402 	uchar_t *dptr;
1403 
1404 	ASSERT(flag & DO_MD5 || flag & DO_SHA1 || flag & DO_SHA2);
1405 	if (data == NULL) {
1406 		ASSERT((flag & DO_UPDATE) == 0);
1407 		goto dofinal;
1408 	}
1409 
1410 	dlen = data->cd_length;
1411 
1412 	if (flag & DO_UPDATE) {
1413 
1414 		switch (data->cd_format) {
1415 		case CRYPTO_DATA_RAW:
1416 			dptr = (uchar_t *)(data->cd_raw.iov_base +
1417 			    data->cd_offset);
1418 
1419 			if (flag & DO_MD5)
1420 				MD5Update(dctx, dptr, dlen);
1421 
1422 			else if (flag & DO_SHA1)
1423 				SHA1Update(dctx, dptr, dlen);
1424 
1425 			else
1426 				SHA2Update(dctx, dptr, dlen);
1427 
1428 		break;
1429 
1430 		case CRYPTO_DATA_UIO:
1431 			if (flag & DO_MD5)
1432 				rv = process_uio_data(data, NULL, dlen,
1433 				    MD5_DIGEST_DATA, dctx);
1434 
1435 			else if (flag & DO_SHA1)
1436 				rv = process_uio_data(data, NULL, dlen,
1437 				    SHA1_DIGEST_DATA, dctx);
1438 
1439 			else
1440 				rv = process_uio_data(data, NULL, dlen,
1441 				    SHA2_DIGEST_DATA, dctx);
1442 
1443 			if (rv != CRYPTO_SUCCESS)
1444 				return (rv);
1445 
1446 			break;
1447 
1448 		case CRYPTO_DATA_MBLK:
1449 			if (flag & DO_MD5)
1450 				rv = process_mblk_data(data, NULL, dlen,
1451 				    MD5_DIGEST_DATA, dctx);
1452 
1453 			else if (flag & DO_SHA1)
1454 				rv = process_mblk_data(data, NULL, dlen,
1455 				    SHA1_DIGEST_DATA, dctx);
1456 
1457 			else
1458 				rv = process_mblk_data(data, NULL, dlen,
1459 				    SHA2_DIGEST_DATA, dctx);
1460 
1461 			if (rv != CRYPTO_SUCCESS)
1462 				return (rv);
1463 
1464 			break;
1465 		}
1466 	}
1467 
1468 dofinal:
1469 	if (flag & DO_FINAL) {
1470 		if (flag & DO_MD5)
1471 			MD5Final(digest, dctx);
1472 
1473 		else if (flag & DO_SHA1)
1474 			SHA1Final(digest, dctx);
1475 
1476 		else
1477 			SHA2Final(digest, dctx);
1478 	}
1479 
1480 	return (CRYPTO_SUCCESS);
1481 }
1482 
1483 static int
1484 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
1485     crypto_data_t *signature, int kmflag, uchar_t flag)
1486 {
1487 	int rv = CRYPTO_FAILED;
1488 
1489 /* EXPORT DELETE START */
1490 
1491 	uchar_t digest[SHA512_DIGEST_LENGTH];
1492 	/* The der_data size is enough for MD5 also */
1493 	uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1494 	ulong_t der_data_len;
1495 	crypto_data_t der_cd;
1496 	rsa_mech_type_t mech_type;
1497 
1498 	ASSERT(flag & DO_SIGN || flag & DO_VERIFY);
1499 	ASSERT(data != NULL || (flag & DO_FINAL));
1500 
1501 	mech_type = ctxp->mech_type;
1502 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1503 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
1504 		return (CRYPTO_MECHANISM_INVALID);
1505 
1506 	/*
1507 	 * We need to do the BUFFER_TOO_SMALL check before digesting
1508 	 * the data. No check is needed for verify as signature is not
1509 	 * an output argument for verify.
1510 	 */
1511 	if (flag & DO_SIGN) {
1512 		uchar_t *modulus;
1513 		ssize_t modulus_len;
1514 
1515 		if ((rv = get_key_attr(ctxp->key, SUN_CKA_MODULUS, &modulus,
1516 		    &modulus_len)) != CRYPTO_SUCCESS) {
1517 			return (rv);
1518 		}
1519 
1520 		if (signature->cd_length < modulus_len) {
1521 			signature->cd_length = modulus_len;
1522 			return (CRYPTO_BUFFER_TOO_SMALL);
1523 		}
1524 	}
1525 
1526 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1527 		rv = digest_data(data, &(ctxp->md5_ctx),
1528 		    digest, flag | DO_MD5);
1529 
1530 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1531 		rv = digest_data(data, &(ctxp->sha1_ctx),
1532 		    digest, flag | DO_SHA1);
1533 
1534 	else
1535 		rv = digest_data(data, &(ctxp->sha2_ctx),
1536 		    digest, flag | DO_SHA2);
1537 
1538 	if (rv != CRYPTO_SUCCESS)
1539 		return (rv);
1540 
1541 
1542 	/*
1543 	 * Prepare the DER encoding of the DigestInfo value as follows:
1544 	 * MD5:		MD5_DER_PREFIX || H
1545 	 * SHA-1:	SHA1_DER_PREFIX || H
1546 	 *
1547 	 * See rsa_impl.c for more details.
1548 	 */
1549 	switch (mech_type) {
1550 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1551 		bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1552 		bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1553 		der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1554 		break;
1555 
1556 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1557 		bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1558 		bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1559 		    SHA1_DIGEST_SIZE);
1560 		der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1561 		break;
1562 
1563 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1564 		bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1565 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1566 		    SHA256_DIGEST_LENGTH);
1567 		der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1568 		break;
1569 
1570 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1571 		bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1572 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1573 		    SHA384_DIGEST_LENGTH);
1574 		der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1575 		break;
1576 
1577 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1578 		bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1579 		bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1580 		    SHA512_DIGEST_LENGTH);
1581 		der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1582 		break;
1583 	}
1584 
1585 	INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1586 	/*
1587 	 * Now, we are ready to sign or verify the DER_ENCODED data.
1588 	 */
1589 	if (flag & DO_SIGN)
1590 		rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1591 		    signature, kmflag);
1592 	else
1593 		rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1594 		    signature, kmflag);
1595 
1596 /* EXPORT DELETE END */
1597 
1598 	return (rv);
1599 }
1600 
1601 static int
1602 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1603     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1604 {
1605 	int rv = CRYPTO_FAILED;
1606 
1607 /* EXPORT DELETE START */
1608 
1609 	int dlen;
1610 	uchar_t *dataptr, *modulus;
1611 	ssize_t modulus_len;
1612 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1613 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1614 	uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1615 
1616 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1617 	    &modulus_len)) != CRYPTO_SUCCESS) {
1618 		return (rv);
1619 	}
1620 
1621 	dlen = data->cd_length;
1622 	switch (mech_type) {
1623 	case RSA_PKCS_MECH_INFO_TYPE:
1624 		if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1625 			return (CRYPTO_DATA_LEN_RANGE);
1626 		break;
1627 	case RSA_X_509_MECH_INFO_TYPE:
1628 		if (dlen > modulus_len)
1629 			return (CRYPTO_DATA_LEN_RANGE);
1630 		break;
1631 	}
1632 
1633 	if (signature->cd_length < modulus_len) {
1634 		signature->cd_length = modulus_len;
1635 		return (CRYPTO_BUFFER_TOO_SMALL);
1636 	}
1637 
1638 	ASSERT(data->cd_length <= sizeof (tmp_data));
1639 	if ((rv = get_input_data(data, &dataptr, tmp_data))
1640 	    != CRYPTO_SUCCESS)
1641 		return (rv);
1642 
1643 	switch (mech_type) {
1644 	case RSA_PKCS_MECH_INFO_TYPE:
1645 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1646 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1647 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1648 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1649 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1650 		/*
1651 		 * Add PKCS padding to the input data to format a block
1652 		 * type "01" encryption block.
1653 		 */
1654 		rv = soft_sign_rsa_pkcs_encode(dataptr, dlen, plain_data,
1655 		    modulus_len);
1656 		if (rv != CRYPTO_SUCCESS)
1657 			return (rv);
1658 
1659 		break;
1660 
1661 	case RSA_X_509_MECH_INFO_TYPE:
1662 		bzero(plain_data, modulus_len - dlen);
1663 		bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1664 		break;
1665 	}
1666 
1667 	rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data,
1668 	    kmflag);
1669 	if (rv == CRYPTO_SUCCESS) {
1670 		/* copy out to signature */
1671 		if ((rv = put_output_data(signed_data,
1672 		    signature, modulus_len)) != CRYPTO_SUCCESS)
1673 			return (rv);
1674 
1675 		signature->cd_length = modulus_len;
1676 	}
1677 
1678 /* EXPORT DELETE END */
1679 
1680 	return (rv);
1681 }
1682 
1683 /* ARGSUSED */
1684 static int
1685 rsa_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1686     crypto_req_handle_t req)
1687 {
1688 	int rv;
1689 	rsa_ctx_t *ctxp;
1690 
1691 	ASSERT(ctx->cc_provider_private != NULL);
1692 	ctxp = ctx->cc_provider_private;
1693 
1694 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1695 	switch (ctxp->mech_type) {
1696 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1697 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1698 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1699 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1700 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1701 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1702 		    signature, KM_SLEEP, DO_SIGN | DO_UPDATE | DO_FINAL);
1703 		break;
1704 	default:
1705 		rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1706 		    signature, KM_SLEEP);
1707 		break;
1708 	}
1709 
1710 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1711 		(void) rsa_free_context(ctx);
1712 
1713 	return (rv);
1714 }
1715 
1716 /* ARGSUSED */
1717 static int
1718 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1719 {
1720 	int rv;
1721 	digest_rsa_ctx_t *ctxp;
1722 	rsa_mech_type_t mech_type;
1723 
1724 	ASSERT(ctx->cc_provider_private != NULL);
1725 	ctxp = ctx->cc_provider_private;
1726 	mech_type = ctxp->mech_type;
1727 
1728 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1729 	    mech_type == RSA_X_509_MECH_INFO_TYPE)
1730 		return (CRYPTO_MECHANISM_INVALID);
1731 
1732 	if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1733 		rv = digest_data(data, &(ctxp->md5_ctx),
1734 		    NULL, DO_MD5 | DO_UPDATE);
1735 
1736 	else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1737 		rv = digest_data(data, &(ctxp->sha1_ctx),
1738 		    NULL, DO_SHA1 | DO_UPDATE);
1739 
1740 	else
1741 		rv = digest_data(data, &(ctxp->sha2_ctx),
1742 		    NULL, DO_SHA2 | DO_UPDATE);
1743 
1744 	return (rv);
1745 }
1746 
1747 static int
1748 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1749     crypto_req_handle_t req)
1750 {
1751 	int rv;
1752 	digest_rsa_ctx_t *ctxp;
1753 
1754 	ASSERT(ctx->cc_provider_private != NULL);
1755 	ctxp = ctx->cc_provider_private;
1756 
1757 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1758 	    crypto_kmflag(req), DO_SIGN | DO_FINAL);
1759 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1760 		(void) rsa_free_context(ctx);
1761 
1762 	return (rv);
1763 }
1764 
1765 /* ARGSUSED */
1766 static int
1767 rsa_sign_atomic(crypto_provider_handle_t provider,
1768     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1769     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1770     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1771 {
1772 	int rv;
1773 	digest_rsa_ctx_t dctx;
1774 
1775 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1776 		return (rv);
1777 
1778 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1779 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1780 		rv = rsa_sign_common(mechanism->cm_type, key, data,
1781 		    signature, crypto_kmflag(req));
1782 
1783 	else {
1784 		dctx.mech_type = mechanism->cm_type;
1785 		dctx.key = key;
1786 		switch (mechanism->cm_type) {
1787 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1788 			MD5Init(&(dctx.md5_ctx));
1789 			break;
1790 
1791 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1792 			SHA1Init(&(dctx.sha1_ctx));
1793 			break;
1794 
1795 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1796 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1797 			break;
1798 
1799 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1800 			SHA2Init(SHA384, &(dctx.sha2_ctx));
1801 			break;
1802 
1803 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1804 			SHA2Init(SHA512, &(dctx.sha2_ctx));
1805 			break;
1806 		}
1807 
1808 		rv = rsa_digest_svrfy_common(&dctx, data, signature,
1809 		    crypto_kmflag(req), DO_SIGN | DO_UPDATE | DO_FINAL);
1810 	}
1811 
1812 	return (rv);
1813 }
1814 
1815 static int
1816 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1817     crypto_data_t *data, crypto_data_t *signature, int kmflag)
1818 {
1819 	int rv = CRYPTO_FAILED;
1820 
1821 /* EXPORT DELETE START */
1822 
1823 	uchar_t *sigptr, *modulus;
1824 	ssize_t modulus_len;
1825 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1826 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1827 
1828 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1829 	    &modulus_len)) != CRYPTO_SUCCESS) {
1830 		return (rv);
1831 	}
1832 
1833 	if (signature->cd_length != modulus_len)
1834 		return (CRYPTO_SIGNATURE_LEN_RANGE);
1835 
1836 	ASSERT(signature->cd_length <= sizeof (tmp_data));
1837 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
1838 	    != CRYPTO_SUCCESS)
1839 		return (rv);
1840 
1841 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
1842 	if (rv != CRYPTO_SUCCESS)
1843 		return (rv);
1844 
1845 	if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1846 		if (compare_data(data, (plain_data + modulus_len
1847 		    - data->cd_length)) != 0)
1848 			rv = CRYPTO_SIGNATURE_INVALID;
1849 
1850 	} else {
1851 		int data_len = modulus_len;
1852 
1853 		/*
1854 		 * Strip off the encoded padding bytes in front of the
1855 		 * recovered data, then compare the recovered data with
1856 		 * the original data.
1857 		 */
1858 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
1859 		if (rv != CRYPTO_SUCCESS)
1860 			return (rv);
1861 
1862 		if (data_len != data->cd_length)
1863 			return (CRYPTO_SIGNATURE_LEN_RANGE);
1864 
1865 		if (compare_data(data, (plain_data + modulus_len
1866 		    - data_len)) != 0)
1867 			rv = CRYPTO_SIGNATURE_INVALID;
1868 	}
1869 
1870 /* EXPORT DELETE END */
1871 
1872 	return (rv);
1873 }
1874 
1875 /* ARGSUSED */
1876 static int
1877 rsa_verify(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1878     crypto_req_handle_t req)
1879 {
1880 	int rv;
1881 	rsa_ctx_t *ctxp;
1882 
1883 	ASSERT(ctx->cc_provider_private != NULL);
1884 	ctxp = ctx->cc_provider_private;
1885 
1886 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
1887 	switch (ctxp->mech_type) {
1888 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1889 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1890 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1891 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1892 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1893 		rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1894 		    signature, KM_SLEEP, DO_VERIFY | DO_UPDATE | DO_FINAL);
1895 		break;
1896 	default:
1897 		rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1898 		    signature, KM_SLEEP);
1899 		break;
1900 	}
1901 
1902 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1903 		(void) rsa_free_context(ctx);
1904 
1905 	return (rv);
1906 }
1907 
1908 /* ARGSUSED */
1909 static int
1910 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1911     crypto_req_handle_t req)
1912 {
1913 	int rv;
1914 	digest_rsa_ctx_t *ctxp;
1915 
1916 	ASSERT(ctx->cc_provider_private != NULL);
1917 	ctxp = ctx->cc_provider_private;
1918 
1919 	switch (ctxp->mech_type) {
1920 
1921 	case MD5_RSA_PKCS_MECH_INFO_TYPE:
1922 		rv = digest_data(data, &(ctxp->md5_ctx),
1923 		    NULL, DO_MD5 | DO_UPDATE);
1924 		break;
1925 
1926 	case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1927 		rv = digest_data(data, &(ctxp->sha1_ctx),
1928 		    NULL, DO_SHA1 | DO_UPDATE);
1929 		break;
1930 
1931 	case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1932 	case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1933 	case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1934 		rv = digest_data(data, &(ctxp->sha2_ctx),
1935 		    NULL, DO_SHA2 | DO_UPDATE);
1936 		break;
1937 
1938 	default:
1939 		return (CRYPTO_MECHANISM_INVALID);
1940 	}
1941 
1942 	return (rv);
1943 }
1944 
1945 static int
1946 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1947     crypto_req_handle_t req)
1948 {
1949 	int rv;
1950 	digest_rsa_ctx_t *ctxp;
1951 
1952 	ASSERT(ctx->cc_provider_private != NULL);
1953 	ctxp = ctx->cc_provider_private;
1954 
1955 	rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1956 	    crypto_kmflag(req), DO_VERIFY | DO_FINAL);
1957 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
1958 		(void) rsa_free_context(ctx);
1959 
1960 	return (rv);
1961 }
1962 
1963 
1964 /* ARGSUSED */
1965 static int
1966 rsa_verify_atomic(crypto_provider_handle_t provider,
1967     crypto_session_id_t session_id,
1968     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1969     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1970     crypto_req_handle_t req)
1971 {
1972 	int rv;
1973 	digest_rsa_ctx_t dctx;
1974 
1975 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1976 		return (rv);
1977 
1978 	if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1979 	    mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1980 		rv = rsa_verify_common(mechanism->cm_type, key, data,
1981 		    signature, crypto_kmflag(req));
1982 
1983 	else {
1984 		dctx.mech_type = mechanism->cm_type;
1985 		dctx.key = key;
1986 
1987 		switch (mechanism->cm_type) {
1988 		case MD5_RSA_PKCS_MECH_INFO_TYPE:
1989 			MD5Init(&(dctx.md5_ctx));
1990 			break;
1991 
1992 		case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1993 			SHA1Init(&(dctx.sha1_ctx));
1994 			break;
1995 
1996 		case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1997 			SHA2Init(SHA256, &(dctx.sha2_ctx));
1998 			break;
1999 
2000 		case SHA384_RSA_PKCS_MECH_INFO_TYPE:
2001 			SHA2Init(SHA384, &(dctx.sha2_ctx));
2002 			break;
2003 
2004 		case SHA512_RSA_PKCS_MECH_INFO_TYPE:
2005 			SHA2Init(SHA512, &(dctx.sha2_ctx));
2006 			break;
2007 		}
2008 
2009 		rv = rsa_digest_svrfy_common(&dctx, data,
2010 		    signature, crypto_kmflag(req),
2011 		    DO_VERIFY | DO_UPDATE | DO_FINAL);
2012 	}
2013 
2014 	return (rv);
2015 }
2016 
2017 static int
2018 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
2019     crypto_data_t *signature, crypto_data_t *data, int kmflag)
2020 {
2021 	int rv = CRYPTO_FAILED;
2022 
2023 /* EXPORT DELETE START */
2024 
2025 	int data_len;
2026 	uchar_t *sigptr, *modulus;
2027 	ssize_t modulus_len;
2028 	uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
2029 	uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
2030 
2031 	if ((rv = get_key_attr(key, SUN_CKA_MODULUS, &modulus,
2032 	    &modulus_len)) != CRYPTO_SUCCESS) {
2033 		return (rv);
2034 	}
2035 
2036 	if (signature->cd_length != modulus_len)
2037 		return (CRYPTO_SIGNATURE_LEN_RANGE);
2038 
2039 	ASSERT(signature->cd_length <= sizeof (tmp_data));
2040 	if ((rv = get_input_data(signature, &sigptr, tmp_data))
2041 	    != CRYPTO_SUCCESS)
2042 		return (rv);
2043 
2044 	rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, kmflag, 1);
2045 	if (rv != CRYPTO_SUCCESS)
2046 		return (rv);
2047 
2048 	data_len = modulus_len;
2049 
2050 	if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
2051 		/*
2052 		 * Strip off the encoded padding bytes in front of the
2053 		 * recovered data, then compare the recovered data with
2054 		 * the original data.
2055 		 */
2056 		rv = soft_verify_rsa_pkcs_decode(plain_data, &data_len);
2057 		if (rv != CRYPTO_SUCCESS)
2058 			return (rv);
2059 	}
2060 
2061 	if (data->cd_length < data_len) {
2062 		data->cd_length = data_len;
2063 		return (CRYPTO_BUFFER_TOO_SMALL);
2064 	}
2065 
2066 	if ((rv = put_output_data(plain_data + modulus_len - data_len,
2067 	    data, data_len)) != CRYPTO_SUCCESS)
2068 		return (rv);
2069 	data->cd_length = data_len;
2070 
2071 /* EXPORT DELETE END */
2072 
2073 	return (rv);
2074 }
2075 
2076 /* ARGSUSED */
2077 static int
2078 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
2079     crypto_data_t *data, crypto_req_handle_t req)
2080 {
2081 	int rv;
2082 	rsa_ctx_t *ctxp;
2083 
2084 	ASSERT(ctx->cc_provider_private != NULL);
2085 	ctxp = ctx->cc_provider_private;
2086 
2087 	/* See the comments on KM_SLEEP flag in rsa_encrypt() */
2088 	rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
2089 	    signature, data, KM_SLEEP);
2090 
2091 	if (rv != CRYPTO_BUFFER_TOO_SMALL)
2092 		(void) rsa_free_context(ctx);
2093 
2094 	return (rv);
2095 }
2096 
2097 /* ARGSUSED */
2098 static int
2099 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
2100     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
2101     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
2102     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
2103 {
2104 	int rv;
2105 
2106 	if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
2107 		return (rv);
2108 
2109 	return (rsa_verify_recover_common(mechanism->cm_type, key,
2110 	    signature, data, crypto_kmflag(req)));
2111 }
2112