xref: /freebsd/sys/opencrypto/cryptodev.c (revision e17f5b1d)
1 /*	$OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $	*/
2 
3 /*-
4  * Copyright (c) 2001 Theo de Raadt
5  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6  * Copyright (c) 2014 The FreeBSD Foundation
7  * All rights reserved.
8  *
9  * Portions of this software were developed by John-Mark Gurney
10  * under sponsorship of the FreeBSD Foundation and
11  * Rubicon Communications, LLC (Netgate).
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *   notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *   notice, this list of conditions and the following disclaimer in the
21  *   documentation and/or other materials provided with the distribution.
22  * 3. The name of the author may not be used to endorse or promote products
23  *   derived from this software without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
29  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Effort sponsored in part by the Defense Advanced Research Projects
37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
39  */
40 
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
43 
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/sysctl.h>
51 #include <sys/file.h>
52 #include <sys/filedesc.h>
53 #include <sys/errno.h>
54 #include <sys/random.h>
55 #include <sys/conf.h>
56 #include <sys/kernel.h>
57 #include <sys/module.h>
58 #include <sys/fcntl.h>
59 #include <sys/bus.h>
60 #include <sys/user.h>
61 #include <sys/sdt.h>
62 
63 #include <opencrypto/cryptodev.h>
64 #include <opencrypto/xform.h>
65 
66 SDT_PROVIDER_DECLARE(opencrypto);
67 
68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
69 
70 #ifdef COMPAT_FREEBSD32
71 #include <sys/mount.h>
72 #include <compat/freebsd32/freebsd32.h>
73 
74 struct session_op32 {
75 	u_int32_t	cipher;
76 	u_int32_t	mac;
77 	u_int32_t	keylen;
78 	u_int32_t	key;
79 	int		mackeylen;
80 	u_int32_t	mackey;
81 	u_int32_t	ses;
82 };
83 
84 struct session2_op32 {
85 	u_int32_t	cipher;
86 	u_int32_t	mac;
87 	u_int32_t	keylen;
88 	u_int32_t	key;
89 	int		mackeylen;
90 	u_int32_t	mackey;
91 	u_int32_t	ses;
92 	int		crid;
93 	int		pad[4];
94 };
95 
96 struct crypt_op32 {
97 	u_int32_t	ses;
98 	u_int16_t	op;
99 	u_int16_t	flags;
100 	u_int		len;
101 	u_int32_t	src, dst;
102 	u_int32_t	mac;
103 	u_int32_t	iv;
104 };
105 
106 struct crparam32 {
107 	u_int32_t	crp_p;
108 	u_int		crp_nbits;
109 };
110 
111 struct crypt_kop32 {
112 	u_int		crk_op;
113 	u_int		crk_status;
114 	u_short		crk_iparams;
115 	u_short		crk_oparams;
116 	u_int		crk_crid;
117 	struct crparam32	crk_param[CRK_MAXPARAM];
118 };
119 
120 #define	CIOCGSESSION32	_IOWR('c', 101, struct session_op32)
121 #define	CIOCCRYPT32	_IOWR('c', 103, struct crypt_op32)
122 #define	CIOCKEY32	_IOWR('c', 104, struct crypt_kop32)
123 #define	CIOCGSESSION232	_IOWR('c', 106, struct session2_op32)
124 #define	CIOCKEY232	_IOWR('c', 107, struct crypt_kop32)
125 
126 static void
127 session_op_from_32(const struct session_op32 *from, struct session_op *to)
128 {
129 
130 	CP(*from, *to, cipher);
131 	CP(*from, *to, mac);
132 	CP(*from, *to, keylen);
133 	PTRIN_CP(*from, *to, key);
134 	CP(*from, *to, mackeylen);
135 	PTRIN_CP(*from, *to, mackey);
136 	CP(*from, *to, ses);
137 }
138 
139 static void
140 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
141 {
142 
143 	session_op_from_32((const struct session_op32 *)from,
144 	    (struct session_op *)to);
145 	CP(*from, *to, crid);
146 }
147 
148 static void
149 session_op_to_32(const struct session_op *from, struct session_op32 *to)
150 {
151 
152 	CP(*from, *to, cipher);
153 	CP(*from, *to, mac);
154 	CP(*from, *to, keylen);
155 	PTROUT_CP(*from, *to, key);
156 	CP(*from, *to, mackeylen);
157 	PTROUT_CP(*from, *to, mackey);
158 	CP(*from, *to, ses);
159 }
160 
161 static void
162 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
163 {
164 
165 	session_op_to_32((const struct session_op *)from,
166 	    (struct session_op32 *)to);
167 	CP(*from, *to, crid);
168 }
169 
170 static void
171 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
172 {
173 
174 	CP(*from, *to, ses);
175 	CP(*from, *to, op);
176 	CP(*from, *to, flags);
177 	CP(*from, *to, len);
178 	PTRIN_CP(*from, *to, src);
179 	PTRIN_CP(*from, *to, dst);
180 	PTRIN_CP(*from, *to, mac);
181 	PTRIN_CP(*from, *to, iv);
182 }
183 
184 static void
185 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
186 {
187 
188 	CP(*from, *to, ses);
189 	CP(*from, *to, op);
190 	CP(*from, *to, flags);
191 	CP(*from, *to, len);
192 	PTROUT_CP(*from, *to, src);
193 	PTROUT_CP(*from, *to, dst);
194 	PTROUT_CP(*from, *to, mac);
195 	PTROUT_CP(*from, *to, iv);
196 }
197 
198 static void
199 crparam_from_32(const struct crparam32 *from, struct crparam *to)
200 {
201 
202 	PTRIN_CP(*from, *to, crp_p);
203 	CP(*from, *to, crp_nbits);
204 }
205 
206 static void
207 crparam_to_32(const struct crparam *from, struct crparam32 *to)
208 {
209 
210 	PTROUT_CP(*from, *to, crp_p);
211 	CP(*from, *to, crp_nbits);
212 }
213 
214 static void
215 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to)
216 {
217 	int i;
218 
219 	CP(*from, *to, crk_op);
220 	CP(*from, *to, crk_status);
221 	CP(*from, *to, crk_iparams);
222 	CP(*from, *to, crk_oparams);
223 	CP(*from, *to, crk_crid);
224 	for (i = 0; i < CRK_MAXPARAM; i++)
225 		crparam_from_32(&from->crk_param[i], &to->crk_param[i]);
226 }
227 
228 static void
229 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to)
230 {
231 	int i;
232 
233 	CP(*from, *to, crk_op);
234 	CP(*from, *to, crk_status);
235 	CP(*from, *to, crk_iparams);
236 	CP(*from, *to, crk_oparams);
237 	CP(*from, *to, crk_crid);
238 	for (i = 0; i < CRK_MAXPARAM; i++)
239 		crparam_to_32(&from->crk_param[i], &to->crk_param[i]);
240 }
241 #endif
242 
243 struct csession {
244 	TAILQ_ENTRY(csession) next;
245 	crypto_session_t cses;
246 	volatile u_int	refs;
247 	u_int32_t	ses;
248 	struct mtx	lock;		/* for op submission */
249 
250 	struct enc_xform *txform;
251 	int		hashsize;
252 	int		ivsize;
253 	int		mode;
254 
255 	void		*key;
256 	void		*mackey;
257 };
258 
259 struct cryptop_data {
260 	struct csession *cse;
261 
262 	char		*buf;
263 	char		*obuf;
264 	char		*aad;
265 	bool		done;
266 };
267 
268 struct fcrypt {
269 	TAILQ_HEAD(csessionlist, csession) csessions;
270 	int		sesn;
271 	struct mtx	lock;
272 };
273 
274 static bool use_outputbuffers;
275 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
276     &use_outputbuffers, 0,
277     "Use separate output buffers for /dev/crypto requests.");
278 
279 static bool use_separate_aad;
280 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
281     &use_separate_aad, 0,
282     "Use separate AAD buffer for /dev/crypto requests.");
283 
284 static	int cryptof_ioctl(struct file *, u_long, void *,
285 		    struct ucred *, struct thread *);
286 static	int cryptof_stat(struct file *, struct stat *,
287 		    struct ucred *, struct thread *);
288 static	int cryptof_close(struct file *, struct thread *);
289 static	int cryptof_fill_kinfo(struct file *, struct kinfo_file *,
290 		    struct filedesc *);
291 
292 static struct fileops cryptofops = {
293     .fo_read = invfo_rdwr,
294     .fo_write = invfo_rdwr,
295     .fo_truncate = invfo_truncate,
296     .fo_ioctl = cryptof_ioctl,
297     .fo_poll = invfo_poll,
298     .fo_kqfilter = invfo_kqfilter,
299     .fo_stat = cryptof_stat,
300     .fo_close = cryptof_close,
301     .fo_chmod = invfo_chmod,
302     .fo_chown = invfo_chown,
303     .fo_sendfile = invfo_sendfile,
304     .fo_fill_kinfo = cryptof_fill_kinfo,
305 };
306 
307 static struct csession *csefind(struct fcrypt *, u_int);
308 static bool csedelete(struct fcrypt *, u_int);
309 static struct csession *csecreate(struct fcrypt *, crypto_session_t,
310     struct crypto_session_params *, struct enc_xform *, void *,
311     struct auth_hash *, void *);
312 static void csefree(struct csession *);
313 
314 static	int cryptodev_op(struct csession *, struct crypt_op *,
315 			struct ucred *, struct thread *td);
316 static	int cryptodev_aead(struct csession *, struct crypt_aead *,
317 			struct ucred *, struct thread *);
318 static	int cryptodev_key(struct crypt_kop *);
319 static	int cryptodev_find(struct crypt_find_op *);
320 
321 /*
322  * Check a crypto identifier to see if it requested
323  * a software device/driver.  This can be done either
324  * by device name/class or through search constraints.
325  */
326 static int
327 checkforsoftware(int *cridp)
328 {
329 	int crid;
330 
331 	crid = *cridp;
332 
333 	if (!crypto_devallowsoft) {
334 		if (crid & CRYPTOCAP_F_SOFTWARE) {
335 			if (crid & CRYPTOCAP_F_HARDWARE) {
336 				*cridp = CRYPTOCAP_F_HARDWARE;
337 				return 0;
338 			}
339 			return EINVAL;
340 		}
341 		if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
342 		    (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
343 			return EINVAL;
344 	}
345 	return 0;
346 }
347 
348 /* ARGSUSED */
349 static int
350 cryptof_ioctl(
351 	struct file *fp,
352 	u_long cmd,
353 	void *data,
354 	struct ucred *active_cred,
355 	struct thread *td)
356 {
357 #define	SES2(p)	((struct session2_op *)p)
358 	struct crypto_session_params csp;
359 	struct fcrypt *fcr = fp->f_data;
360 	struct csession *cse;
361 	struct session_op *sop;
362 	struct crypt_op *cop;
363 	struct crypt_aead *caead;
364 	struct enc_xform *txform = NULL;
365 	struct auth_hash *thash = NULL;
366 	void *key = NULL;
367 	void *mackey = NULL;
368 	struct crypt_kop *kop;
369 	crypto_session_t cses;
370 	u_int32_t ses;
371 	int error = 0, crid;
372 #ifdef COMPAT_FREEBSD32
373 	struct session2_op sopc;
374 	struct crypt_op copc;
375 	struct crypt_kop kopc;
376 #endif
377 
378 	switch (cmd) {
379 	case CIOCGSESSION:
380 	case CIOCGSESSION2:
381 #ifdef COMPAT_FREEBSD32
382 	case CIOCGSESSION32:
383 	case CIOCGSESSION232:
384 		if (cmd == CIOCGSESSION32) {
385 			session_op_from_32(data, (struct session_op *)&sopc);
386 			sop = (struct session_op *)&sopc;
387 		} else if (cmd == CIOCGSESSION232) {
388 			session2_op_from_32(data, &sopc);
389 			sop = (struct session_op *)&sopc;
390 		} else
391 #endif
392 			sop = (struct session_op *)data;
393 		switch (sop->cipher) {
394 		case 0:
395 			break;
396 		case CRYPTO_AES_CBC:
397 			txform = &enc_xform_rijndael128;
398 			break;
399 		case CRYPTO_AES_XTS:
400 			txform = &enc_xform_aes_xts;
401 			break;
402 		case CRYPTO_NULL_CBC:
403 			txform = &enc_xform_null;
404 			break;
405  		case CRYPTO_CAMELLIA_CBC:
406  			txform = &enc_xform_camellia;
407  			break;
408 		case CRYPTO_AES_ICM:
409 			txform = &enc_xform_aes_icm;
410  			break;
411 		case CRYPTO_AES_NIST_GCM_16:
412 			txform = &enc_xform_aes_nist_gcm;
413  			break;
414 		case CRYPTO_CHACHA20:
415 			txform = &enc_xform_chacha20;
416 			break;
417 		case CRYPTO_AES_CCM_16:
418 			txform = &enc_xform_ccm;
419 			break;
420 
421 		default:
422 			CRYPTDEB("invalid cipher");
423 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
424 			return (EINVAL);
425 		}
426 
427 		switch (sop->mac) {
428 		case 0:
429 			break;
430 		case CRYPTO_POLY1305:
431 			thash = &auth_hash_poly1305;
432 			break;
433 		case CRYPTO_SHA1_HMAC:
434 			thash = &auth_hash_hmac_sha1;
435 			break;
436 		case CRYPTO_SHA2_224_HMAC:
437 			thash = &auth_hash_hmac_sha2_224;
438 			break;
439 		case CRYPTO_SHA2_256_HMAC:
440 			thash = &auth_hash_hmac_sha2_256;
441 			break;
442 		case CRYPTO_SHA2_384_HMAC:
443 			thash = &auth_hash_hmac_sha2_384;
444 			break;
445 		case CRYPTO_SHA2_512_HMAC:
446 			thash = &auth_hash_hmac_sha2_512;
447 			break;
448 		case CRYPTO_RIPEMD160_HMAC:
449 			thash = &auth_hash_hmac_ripemd_160;
450 			break;
451 #ifdef COMPAT_FREEBSD12
452 		case CRYPTO_AES_128_NIST_GMAC:
453 		case CRYPTO_AES_192_NIST_GMAC:
454 		case CRYPTO_AES_256_NIST_GMAC:
455 			/* Should always be paired with GCM. */
456 			if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
457 				CRYPTDEB("GMAC without GCM");
458 				SDT_PROBE1(opencrypto, dev, ioctl, error,
459 				    __LINE__);
460 				return (EINVAL);
461 			}
462 			break;
463 #endif
464 		case CRYPTO_AES_NIST_GMAC:
465 			switch (sop->mackeylen * 8) {
466 			case 128:
467 				thash = &auth_hash_nist_gmac_aes_128;
468 				break;
469 			case 192:
470 				thash = &auth_hash_nist_gmac_aes_192;
471 				break;
472 			case 256:
473 				thash = &auth_hash_nist_gmac_aes_256;
474 				break;
475 			default:
476 				CRYPTDEB("invalid GMAC key length");
477 				SDT_PROBE1(opencrypto, dev, ioctl, error,
478 				    __LINE__);
479 				return (EINVAL);
480 			}
481 			break;
482 		case CRYPTO_AES_CCM_CBC_MAC:
483 			switch (sop->mackeylen) {
484 			case 16:
485 				thash = &auth_hash_ccm_cbc_mac_128;
486 				break;
487 			case 24:
488 				thash = &auth_hash_ccm_cbc_mac_192;
489 				break;
490 			case 32:
491 				thash = &auth_hash_ccm_cbc_mac_256;
492 				break;
493 			default:
494 				CRYPTDEB("Invalid CBC MAC key size %d",
495 				    sop->keylen);
496 				SDT_PROBE1(opencrypto, dev, ioctl,
497 				    error, __LINE__);
498 				return (EINVAL);
499 			}
500 			break;
501 		case CRYPTO_SHA1:
502 			thash = &auth_hash_sha1;
503 			break;
504 		case CRYPTO_SHA2_224:
505 			thash = &auth_hash_sha2_224;
506 			break;
507 		case CRYPTO_SHA2_256:
508 			thash = &auth_hash_sha2_256;
509 			break;
510 		case CRYPTO_SHA2_384:
511 			thash = &auth_hash_sha2_384;
512 			break;
513 		case CRYPTO_SHA2_512:
514 			thash = &auth_hash_sha2_512;
515 			break;
516 
517 		case CRYPTO_NULL_HMAC:
518 			thash = &auth_hash_null;
519 			break;
520 
521 		case CRYPTO_BLAKE2B:
522 			thash = &auth_hash_blake2b;
523 			break;
524 		case CRYPTO_BLAKE2S:
525 			thash = &auth_hash_blake2s;
526 			break;
527 
528 		default:
529 			CRYPTDEB("invalid mac");
530 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
531 			return (EINVAL);
532 		}
533 
534 		if (txform == NULL && thash == NULL) {
535 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
536 			return (EINVAL);
537 		}
538 
539 		memset(&csp, 0, sizeof(csp));
540 		if (use_outputbuffers)
541 			csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
542 
543 		if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
544 			switch (sop->mac) {
545 #ifdef COMPAT_FREEBSD12
546 			case CRYPTO_AES_128_NIST_GMAC:
547 			case CRYPTO_AES_192_NIST_GMAC:
548 			case CRYPTO_AES_256_NIST_GMAC:
549 				if (sop->keylen != sop->mackeylen) {
550 					SDT_PROBE1(opencrypto, dev, ioctl,
551 					    error, __LINE__);
552 					return (EINVAL);
553 				}
554 				break;
555 #endif
556 			case 0:
557 				break;
558 			default:
559 				SDT_PROBE1(opencrypto, dev, ioctl, error,
560 				    __LINE__);
561 				return (EINVAL);
562 			}
563 			csp.csp_mode = CSP_MODE_AEAD;
564 		} else if (sop->cipher == CRYPTO_AES_CCM_16) {
565 			switch (sop->mac) {
566 #ifdef COMPAT_FREEBSD12
567 			case CRYPTO_AES_CCM_CBC_MAC:
568 				if (sop->keylen != sop->mackeylen) {
569 					SDT_PROBE1(opencrypto, dev, ioctl,
570 					    error, __LINE__);
571 					return (EINVAL);
572 				}
573 				thash = NULL;
574 				break;
575 #endif
576 			case 0:
577 				break;
578 			default:
579 				SDT_PROBE1(opencrypto, dev, ioctl, error,
580 				    __LINE__);
581 				return (EINVAL);
582 			}
583 			csp.csp_mode = CSP_MODE_AEAD;
584 		} else if (txform && thash)
585 			csp.csp_mode = CSP_MODE_ETA;
586 		else if (txform)
587 			csp.csp_mode = CSP_MODE_CIPHER;
588 		else
589 			csp.csp_mode = CSP_MODE_DIGEST;
590 
591 		switch (csp.csp_mode) {
592 		case CSP_MODE_AEAD:
593 		case CSP_MODE_ETA:
594 			if (use_separate_aad)
595 				csp.csp_flags |= CSP_F_SEPARATE_AAD;
596 			break;
597 		}
598 
599 		if (txform) {
600 			csp.csp_cipher_alg = txform->type;
601 			csp.csp_cipher_klen = sop->keylen;
602 			if (sop->keylen > txform->maxkey ||
603 			    sop->keylen < txform->minkey) {
604 				CRYPTDEB("invalid cipher parameters");
605 				error = EINVAL;
606 				SDT_PROBE1(opencrypto, dev, ioctl, error,
607 				    __LINE__);
608 				goto bail;
609 			}
610 
611 			key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
612 			error = copyin(sop->key, key, csp.csp_cipher_klen);
613 			if (error) {
614 				CRYPTDEB("invalid key");
615 				SDT_PROBE1(opencrypto, dev, ioctl, error,
616 				    __LINE__);
617 				goto bail;
618 			}
619 			csp.csp_cipher_key = key;
620 			csp.csp_ivlen = txform->ivsize;
621 		}
622 
623 		if (thash) {
624 			csp.csp_auth_alg = thash->type;
625 			csp.csp_auth_klen = sop->mackeylen;
626 			if (sop->mackeylen > thash->keysize ||
627 			    sop->mackeylen < 0) {
628 				CRYPTDEB("invalid mac key length");
629 				error = EINVAL;
630 				SDT_PROBE1(opencrypto, dev, ioctl, error,
631 				    __LINE__);
632 				goto bail;
633 			}
634 
635 			if (csp.csp_auth_klen) {
636 				mackey = malloc(csp.csp_auth_klen, M_XDATA,
637 				    M_WAITOK);
638 				error = copyin(sop->mackey, mackey,
639 				    csp.csp_auth_klen);
640 				if (error) {
641 					CRYPTDEB("invalid mac key");
642 					SDT_PROBE1(opencrypto, dev, ioctl,
643 					    error, __LINE__);
644 					goto bail;
645 				}
646 				csp.csp_auth_key = mackey;
647 			}
648 
649 			if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
650 				csp.csp_ivlen = AES_GCM_IV_LEN;
651 			if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
652 				csp.csp_ivlen = AES_CCM_IV_LEN;
653 		}
654 
655 		/* NB: CIOCGSESSION2 has the crid */
656 		if (cmd == CIOCGSESSION2
657 #ifdef COMPAT_FREEBSD32
658 		    || cmd == CIOCGSESSION232
659 #endif
660 			) {
661 			crid = SES2(sop)->crid;
662 			error = checkforsoftware(&crid);
663 			if (error) {
664 				CRYPTDEB("checkforsoftware");
665 				SDT_PROBE1(opencrypto, dev, ioctl, error,
666 				    __LINE__);
667 				goto bail;
668 			}
669 		} else
670 			crid = CRYPTOCAP_F_HARDWARE;
671 		error = crypto_newsession(&cses, &csp, crid);
672 		if (error) {
673 			CRYPTDEB("crypto_newsession");
674 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
675 			goto bail;
676 		}
677 
678 		cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey);
679 
680 		if (cse == NULL) {
681 			crypto_freesession(cses);
682 			error = EINVAL;
683 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
684 			CRYPTDEB("csecreate");
685 			goto bail;
686 		}
687 		sop->ses = cse->ses;
688 		if (cmd == CIOCGSESSION2
689 #ifdef COMPAT_FREEBSD32
690 		    || cmd == CIOCGSESSION232
691 #endif
692 		    ) {
693 			/* return hardware/driver id */
694 			SES2(sop)->crid = crypto_ses2hid(cse->cses);
695 		}
696 bail:
697 		if (error) {
698 			free(key, M_XDATA);
699 			free(mackey, M_XDATA);
700 		}
701 #ifdef COMPAT_FREEBSD32
702 		else {
703 			if (cmd == CIOCGSESSION32)
704 				session_op_to_32(sop, data);
705 			else if (cmd == CIOCGSESSION232)
706 				session2_op_to_32((struct session2_op *)sop,
707 				    data);
708 		}
709 #endif
710 		break;
711 	case CIOCFSESSION:
712 		ses = *(u_int32_t *)data;
713 		if (!csedelete(fcr, ses)) {
714 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
715 			return (EINVAL);
716 		}
717 		break;
718 	case CIOCCRYPT:
719 #ifdef COMPAT_FREEBSD32
720 	case CIOCCRYPT32:
721 		if (cmd == CIOCCRYPT32) {
722 			cop = &copc;
723 			crypt_op_from_32(data, cop);
724 		} else
725 #endif
726 			cop = (struct crypt_op *)data;
727 		cse = csefind(fcr, cop->ses);
728 		if (cse == NULL) {
729 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
730 			return (EINVAL);
731 		}
732 		error = cryptodev_op(cse, cop, active_cred, td);
733 		csefree(cse);
734 #ifdef COMPAT_FREEBSD32
735 		if (error == 0 && cmd == CIOCCRYPT32)
736 			crypt_op_to_32(cop, data);
737 #endif
738 		break;
739 	case CIOCKEY:
740 	case CIOCKEY2:
741 #ifdef COMPAT_FREEBSD32
742 	case CIOCKEY32:
743 	case CIOCKEY232:
744 #endif
745 		if (!crypto_userasymcrypto) {
746 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
747 			return (EPERM);		/* XXX compat? */
748 		}
749 #ifdef COMPAT_FREEBSD32
750 		if (cmd == CIOCKEY32 || cmd == CIOCKEY232) {
751 			kop = &kopc;
752 			crypt_kop_from_32(data, kop);
753 		} else
754 #endif
755 			kop = (struct crypt_kop *)data;
756 		if (cmd == CIOCKEY
757 #ifdef COMPAT_FREEBSD32
758 		    || cmd == CIOCKEY32
759 #endif
760 		    ) {
761 			/* NB: crypto core enforces s/w driver use */
762 			kop->crk_crid =
763 			    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
764 		}
765 		mtx_lock(&Giant);
766 		error = cryptodev_key(kop);
767 		mtx_unlock(&Giant);
768 #ifdef COMPAT_FREEBSD32
769 		if (cmd == CIOCKEY32 || cmd == CIOCKEY232)
770 			crypt_kop_to_32(kop, data);
771 #endif
772 		break;
773 	case CIOCASYMFEAT:
774 		if (!crypto_userasymcrypto) {
775 			/*
776 			 * NB: if user asym crypto operations are
777 			 * not permitted return "no algorithms"
778 			 * so well-behaved applications will just
779 			 * fallback to doing them in software.
780 			 */
781 			*(int *)data = 0;
782 		} else {
783 			error = crypto_getfeat((int *)data);
784 			if (error)
785 				SDT_PROBE1(opencrypto, dev, ioctl, error,
786 				    __LINE__);
787 		}
788 		break;
789 	case CIOCFINDDEV:
790 		error = cryptodev_find((struct crypt_find_op *)data);
791 		break;
792 	case CIOCCRYPTAEAD:
793 		caead = (struct crypt_aead *)data;
794 		cse = csefind(fcr, caead->ses);
795 		if (cse == NULL) {
796 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
797 			return (EINVAL);
798 		}
799 		error = cryptodev_aead(cse, caead, active_cred, td);
800 		csefree(cse);
801 		break;
802 	default:
803 		error = EINVAL;
804 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
805 		break;
806 	}
807 	return (error);
808 #undef SES2
809 }
810 
811 static int cryptodev_cb(struct cryptop *);
812 
813 static struct cryptop_data *
814 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td)
815 {
816 	struct cryptop_data *cod;
817 
818 	cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
819 
820 	cod->cse = cse;
821 	if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
822 		if (aad_len != 0)
823 			cod->aad = malloc(aad_len, M_XDATA, M_WAITOK);
824 		cod->buf = malloc(len, M_XDATA, M_WAITOK);
825 	} else
826 		cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK);
827 	if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
828 		cod->obuf = malloc(len, M_XDATA, M_WAITOK);
829 	return (cod);
830 }
831 
832 static void
833 cod_free(struct cryptop_data *cod)
834 {
835 
836 	free(cod->aad, M_XDATA);
837 	free(cod->obuf, M_XDATA);
838 	free(cod->buf, M_XDATA);
839 	free(cod, M_XDATA);
840 }
841 
842 static int
843 cryptodev_op(
844 	struct csession *cse,
845 	struct crypt_op *cop,
846 	struct ucred *active_cred,
847 	struct thread *td)
848 {
849 	struct cryptop_data *cod = NULL;
850 	struct cryptop *crp = NULL;
851 	int error;
852 
853 	if (cop->len > 256*1024-4) {
854 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
855 		return (E2BIG);
856 	}
857 
858 	if (cse->txform) {
859 		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
860 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
861 			return (EINVAL);
862 		}
863 	}
864 
865 	if (cop->mac && cse->hashsize == 0) {
866 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
867 		return (EINVAL);
868 	}
869 
870 	/*
871 	 * The COP_F_CIPHER_FIRST flag predates explicit session
872 	 * modes, but the only way it was used was for EtA so allow it
873 	 * as long as it is consistent with EtA.
874 	 */
875 	if (cop->flags & COP_F_CIPHER_FIRST) {
876 		if (cop->op != COP_ENCRYPT) {
877 			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
878 			return (EINVAL);
879 		}
880 	}
881 
882 	cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td);
883 
884 	crp = crypto_getreq(cse->cses, M_WAITOK);
885 
886 	error = copyin(cop->src, cod->buf, cop->len);
887 	if (error) {
888 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
889 		goto bail;
890 	}
891 	crp->crp_payload_start = 0;
892 	crp->crp_payload_length = cop->len;
893 	if (cse->hashsize)
894 		crp->crp_digest_start = cop->len;
895 
896 	switch (cse->mode) {
897 	case CSP_MODE_COMPRESS:
898 		switch (cop->op) {
899 		case COP_ENCRYPT:
900 			crp->crp_op = CRYPTO_OP_COMPRESS;
901 			break;
902 		case COP_DECRYPT:
903 			crp->crp_op = CRYPTO_OP_DECOMPRESS;
904 			break;
905 		default:
906 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
907 			error = EINVAL;
908 			goto bail;
909 		}
910 		break;
911 	case CSP_MODE_CIPHER:
912 		switch (cop->op) {
913 		case COP_ENCRYPT:
914 			crp->crp_op = CRYPTO_OP_ENCRYPT;
915 			break;
916 		case COP_DECRYPT:
917 			crp->crp_op = CRYPTO_OP_DECRYPT;
918 			break;
919 		default:
920 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
921 			error = EINVAL;
922 			goto bail;
923 		}
924 		break;
925 	case CSP_MODE_DIGEST:
926 		switch (cop->op) {
927 		case 0:
928 		case COP_ENCRYPT:
929 		case COP_DECRYPT:
930 			crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
931 			if (cod->obuf != NULL)
932 				crp->crp_digest_start = 0;
933 			break;
934 		default:
935 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
936 			error = EINVAL;
937 			goto bail;
938 		}
939 		break;
940 	case CSP_MODE_ETA:
941 		switch (cop->op) {
942 		case COP_ENCRYPT:
943 			crp->crp_op = CRYPTO_OP_ENCRYPT |
944 			    CRYPTO_OP_COMPUTE_DIGEST;
945 			break;
946 		case COP_DECRYPT:
947 			crp->crp_op = CRYPTO_OP_DECRYPT |
948 			    CRYPTO_OP_VERIFY_DIGEST;
949 			break;
950 		default:
951 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
952 			error = EINVAL;
953 			goto bail;
954 		}
955 		break;
956 	default:
957 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
958 		error = EINVAL;
959 		goto bail;
960 	}
961 
962 	crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
963 	crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
964 	if (cod->obuf)
965 		crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
966 	crp->crp_callback = cryptodev_cb;
967 	crp->crp_opaque = cod;
968 
969 	if (cop->iv) {
970 		if (cse->ivsize == 0) {
971 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
972 			error = EINVAL;
973 			goto bail;
974 		}
975 		error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
976 		if (error) {
977 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
978 			goto bail;
979 		}
980 		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
981 	} else if (cse->ivsize != 0) {
982 		crp->crp_iv_start = 0;
983 		crp->crp_payload_start += cse->ivsize;
984 		crp->crp_payload_length -= cse->ivsize;
985 		cop->dst += cse->ivsize;
986 	}
987 
988 	if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
989 		error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
990 		    cse->hashsize);
991 		if (error) {
992 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
993 			goto bail;
994 		}
995 	}
996 again:
997 	/*
998 	 * Let the dispatch run unlocked, then, interlock against the
999 	 * callback before checking if the operation completed and going
1000 	 * to sleep.  This insures drivers don't inherit our lock which
1001 	 * results in a lock order reversal between crypto_dispatch forced
1002 	 * entry and the crypto_done callback into us.
1003 	 */
1004 	error = crypto_dispatch(crp);
1005 	if (error != 0) {
1006 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1007 		goto bail;
1008 	}
1009 
1010 	mtx_lock(&cse->lock);
1011 	while (!cod->done)
1012 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1013 	mtx_unlock(&cse->lock);
1014 
1015 	if (crp->crp_etype == EAGAIN) {
1016 		crp->crp_etype = 0;
1017 		crp->crp_flags &= ~CRYPTO_F_DONE;
1018 		cod->done = false;
1019 		goto again;
1020 	}
1021 
1022 	if (crp->crp_etype != 0) {
1023 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1024 		error = crp->crp_etype;
1025 		goto bail;
1026 	}
1027 
1028 	if (cop->dst != NULL) {
1029 		error = copyout(cod->obuf != NULL ? cod->obuf :
1030 		    cod->buf + crp->crp_payload_start, cop->dst,
1031 		    crp->crp_payload_length);
1032 		if (error) {
1033 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1034 			goto bail;
1035 		}
1036 	}
1037 
1038 	if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1039 		error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1040 		    crp->crp_digest_start, cop->mac, cse->hashsize);
1041 		if (error) {
1042 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1043 			goto bail;
1044 		}
1045 	}
1046 
1047 bail:
1048 	crypto_freereq(crp);
1049 	cod_free(cod);
1050 
1051 	return (error);
1052 }
1053 
1054 static int
1055 cryptodev_aead(
1056 	struct csession *cse,
1057 	struct crypt_aead *caead,
1058 	struct ucred *active_cred,
1059 	struct thread *td)
1060 {
1061 	struct cryptop_data *cod = NULL;
1062 	struct cryptop *crp = NULL;
1063 	int error;
1064 
1065 	if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
1066 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1067 		return (E2BIG);
1068 	}
1069 
1070 	if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
1071 	    (caead->len % cse->txform->blocksize) != 0) {
1072 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1073 		return (EINVAL);
1074 	}
1075 
1076 	/*
1077 	 * The COP_F_CIPHER_FIRST flag predates explicit session
1078 	 * modes, but the only way it was used was for EtA so allow it
1079 	 * as long as it is consistent with EtA.
1080 	 */
1081 	if (caead->flags & COP_F_CIPHER_FIRST) {
1082 		if (caead->op != COP_ENCRYPT) {
1083 			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
1084 			return (EINVAL);
1085 		}
1086 	}
1087 
1088 	cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td);
1089 
1090 	crp = crypto_getreq(cse->cses, M_WAITOK);
1091 
1092 	if (cod->aad != NULL)
1093 		error = copyin(caead->aad, cod->aad, caead->aadlen);
1094 	else
1095 		error = copyin(caead->aad, cod->buf, caead->aadlen);
1096 	if (error) {
1097 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1098 		goto bail;
1099 	}
1100 	crp->crp_aad = cod->aad;
1101 	crp->crp_aad_start = 0;
1102 	crp->crp_aad_length = caead->aadlen;
1103 
1104 	if (cod->aad != NULL)
1105 		crp->crp_payload_start = 0;
1106 	else
1107 		crp->crp_payload_start = caead->aadlen;
1108 	error = copyin(caead->src, cod->buf + crp->crp_payload_start,
1109 	    caead->len);
1110 	if (error) {
1111 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1112 		goto bail;
1113 	}
1114 	crp->crp_payload_length = caead->len;
1115 	if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
1116 		crp->crp_digest_start = crp->crp_payload_output_start +
1117 		    caead->len;
1118 	else
1119 		crp->crp_digest_start = crp->crp_payload_start + caead->len;
1120 
1121 	switch (cse->mode) {
1122 	case CSP_MODE_AEAD:
1123 	case CSP_MODE_ETA:
1124 		switch (caead->op) {
1125 		case COP_ENCRYPT:
1126 			crp->crp_op = CRYPTO_OP_ENCRYPT |
1127 			    CRYPTO_OP_COMPUTE_DIGEST;
1128 			break;
1129 		case COP_DECRYPT:
1130 			crp->crp_op = CRYPTO_OP_DECRYPT |
1131 			    CRYPTO_OP_VERIFY_DIGEST;
1132 			break;
1133 		default:
1134 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1135 			error = EINVAL;
1136 			goto bail;
1137 		}
1138 		break;
1139 	default:
1140 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1141 		error = EINVAL;
1142 		goto bail;
1143 	}
1144 
1145 	crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
1146 	crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
1147 	    cse->hashsize);
1148 	if (cod->obuf != NULL)
1149 		crypto_use_output_buf(crp, cod->obuf, caead->len +
1150 		    cse->hashsize);
1151 	crp->crp_callback = cryptodev_cb;
1152 	crp->crp_opaque = cod;
1153 
1154 	if (caead->iv) {
1155 		/*
1156 		 * Permit a 16-byte IV for AES-XTS, but only use the
1157 		 * first 8 bytes as a block number.
1158 		 */
1159 		if (cse->mode == CSP_MODE_ETA &&
1160 		    caead->ivlen == AES_BLOCK_LEN &&
1161 		    cse->ivsize == AES_XTS_IV_LEN)
1162 			caead->ivlen = AES_XTS_IV_LEN;
1163 
1164 		if (caead->ivlen != cse->ivsize) {
1165 			error = EINVAL;
1166 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1167 			goto bail;
1168 		}
1169 
1170 		error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
1171 		if (error) {
1172 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1173 			goto bail;
1174 		}
1175 		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1176 	} else {
1177 		crp->crp_iv_start = crp->crp_payload_start;
1178 		crp->crp_payload_start += cse->ivsize;
1179 		crp->crp_payload_length -= cse->ivsize;
1180 		caead->dst += cse->ivsize;
1181 	}
1182 
1183 	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1184 		error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1185 		    cse->hashsize);
1186 		if (error) {
1187 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1188 			goto bail;
1189 		}
1190 	}
1191 again:
1192 	/*
1193 	 * Let the dispatch run unlocked, then, interlock against the
1194 	 * callback before checking if the operation completed and going
1195 	 * to sleep.  This insures drivers don't inherit our lock which
1196 	 * results in a lock order reversal between crypto_dispatch forced
1197 	 * entry and the crypto_done callback into us.
1198 	 */
1199 	error = crypto_dispatch(crp);
1200 	if (error != 0) {
1201 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1202 		goto bail;
1203 	}
1204 
1205 	mtx_lock(&cse->lock);
1206 	while (!cod->done)
1207 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1208 	mtx_unlock(&cse->lock);
1209 
1210 	if (crp->crp_etype == EAGAIN) {
1211 		crp->crp_etype = 0;
1212 		crp->crp_flags &= ~CRYPTO_F_DONE;
1213 		cod->done = false;
1214 		goto again;
1215 	}
1216 
1217 	if (crp->crp_etype != 0) {
1218 		error = crp->crp_etype;
1219 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1220 		goto bail;
1221 	}
1222 
1223 	if (caead->dst != NULL) {
1224 		error = copyout(cod->obuf != NULL ? cod->obuf :
1225 		    cod->buf + crp->crp_payload_start, caead->dst,
1226 		    crp->crp_payload_length);
1227 		if (error) {
1228 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1229 			goto bail;
1230 		}
1231 	}
1232 
1233 	if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1234 		error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1235 		    crp->crp_digest_start, caead->tag, cse->hashsize);
1236 		if (error) {
1237 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1238 			goto bail;
1239 		}
1240 	}
1241 
1242 bail:
1243 	crypto_freereq(crp);
1244 	cod_free(cod);
1245 
1246 	return (error);
1247 }
1248 
1249 static int
1250 cryptodev_cb(struct cryptop *crp)
1251 {
1252 	struct cryptop_data *cod = crp->crp_opaque;
1253 
1254 	/*
1255 	 * Lock to ensure the wakeup() is not missed by the loops
1256 	 * waiting on cod->done in cryptodev_op() and
1257 	 * cryptodev_aead().
1258 	 */
1259 	mtx_lock(&cod->cse->lock);
1260 	cod->done = true;
1261 	mtx_unlock(&cod->cse->lock);
1262 	wakeup(cod);
1263 	return (0);
1264 }
1265 
1266 static void
1267 cryptodevkey_cb(struct cryptkop *krp)
1268 {
1269 
1270 	wakeup_one(krp);
1271 }
1272 
1273 static int
1274 cryptodev_key(struct crypt_kop *kop)
1275 {
1276 	struct cryptkop *krp = NULL;
1277 	int error = EINVAL;
1278 	int in, out, size, i;
1279 
1280 	if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) {
1281 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1282 		return (EFBIG);
1283 	}
1284 
1285 	in = kop->crk_iparams;
1286 	out = kop->crk_oparams;
1287 	switch (kop->crk_op) {
1288 	case CRK_MOD_EXP:
1289 		if (in == 3 && out == 1)
1290 			break;
1291 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1292 		return (EINVAL);
1293 	case CRK_MOD_EXP_CRT:
1294 		if (in == 6 && out == 1)
1295 			break;
1296 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1297 		return (EINVAL);
1298 	case CRK_DSA_SIGN:
1299 		if (in == 5 && out == 2)
1300 			break;
1301 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1302 		return (EINVAL);
1303 	case CRK_DSA_VERIFY:
1304 		if (in == 7 && out == 0)
1305 			break;
1306 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1307 		return (EINVAL);
1308 	case CRK_DH_COMPUTE_KEY:
1309 		if (in == 3 && out == 1)
1310 			break;
1311 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1312 		return (EINVAL);
1313 	default:
1314 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1315 		return (EINVAL);
1316 	}
1317 
1318 	krp = (struct cryptkop *)malloc(sizeof *krp, M_XDATA, M_WAITOK|M_ZERO);
1319 	if (!krp) {
1320 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1321 		return (ENOMEM);
1322 	}
1323 	krp->krp_op = kop->crk_op;
1324 	krp->krp_status = kop->crk_status;
1325 	krp->krp_iparams = kop->crk_iparams;
1326 	krp->krp_oparams = kop->crk_oparams;
1327 	krp->krp_crid = kop->crk_crid;
1328 	krp->krp_status = 0;
1329 	krp->krp_callback = cryptodevkey_cb;
1330 
1331 	for (i = 0; i < CRK_MAXPARAM; i++) {
1332 		if (kop->crk_param[i].crp_nbits > 65536) {
1333 			/* Limit is the same as in OpenBSD */
1334 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1335 			goto fail;
1336 		}
1337 		krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
1338 	}
1339 	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
1340 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
1341 		if (size == 0)
1342 			continue;
1343 		krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK);
1344 		if (i >= krp->krp_iparams)
1345 			continue;
1346 		error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size);
1347 		if (error) {
1348 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1349 			goto fail;
1350 		}
1351 	}
1352 
1353 	error = crypto_kdispatch(krp);
1354 	if (error) {
1355 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1356 		goto fail;
1357 	}
1358 	error = tsleep(krp, PSOCK, "crydev", 0);
1359 	if (error) {
1360 		/* XXX can this happen?  if so, how do we recover? */
1361 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1362 		goto fail;
1363 	}
1364 
1365 	kop->crk_crid = krp->krp_hid;		/* device that did the work */
1366 	if (krp->krp_status != 0) {
1367 		error = krp->krp_status;
1368 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1369 		goto fail;
1370 	}
1371 
1372 	for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) {
1373 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
1374 		if (size == 0)
1375 			continue;
1376 		error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size);
1377 		if (error) {
1378 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1379 			goto fail;
1380 		}
1381 	}
1382 
1383 fail:
1384 	if (krp) {
1385 		kop->crk_status = krp->krp_status;
1386 		for (i = 0; i < CRK_MAXPARAM; i++) {
1387 			if (krp->krp_param[i].crp_p)
1388 				free(krp->krp_param[i].crp_p, M_XDATA);
1389 		}
1390 		free(krp, M_XDATA);
1391 	}
1392 	return (error);
1393 }
1394 
1395 static int
1396 cryptodev_find(struct crypt_find_op *find)
1397 {
1398 	device_t dev;
1399 	size_t fnlen = sizeof find->name;
1400 
1401 	if (find->crid != -1) {
1402 		dev = crypto_find_device_byhid(find->crid);
1403 		if (dev == NULL)
1404 			return (ENOENT);
1405 		strncpy(find->name, device_get_nameunit(dev), fnlen);
1406 		find->name[fnlen - 1] = '\x0';
1407 	} else {
1408 		find->name[fnlen - 1] = '\x0';
1409 		find->crid = crypto_find_driver(find->name);
1410 		if (find->crid == -1)
1411 			return (ENOENT);
1412 	}
1413 	return (0);
1414 }
1415 
1416 /* ARGSUSED */
1417 static int
1418 cryptof_stat(
1419 	struct file *fp,
1420 	struct stat *sb,
1421 	struct ucred *active_cred,
1422 	struct thread *td)
1423 {
1424 
1425 	return (EOPNOTSUPP);
1426 }
1427 
1428 /* ARGSUSED */
1429 static int
1430 cryptof_close(struct file *fp, struct thread *td)
1431 {
1432 	struct fcrypt *fcr = fp->f_data;
1433 	struct csession *cse;
1434 
1435 	while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1436 		TAILQ_REMOVE(&fcr->csessions, cse, next);
1437 		KASSERT(cse->refs == 1,
1438 		    ("%s: crypto session %p with %d refs", __func__, cse,
1439 		    cse->refs));
1440 		csefree(cse);
1441 	}
1442 	free(fcr, M_XDATA);
1443 	fp->f_data = NULL;
1444 	return 0;
1445 }
1446 
1447 static int
1448 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1449 {
1450 
1451 	kif->kf_type = KF_TYPE_CRYPTO;
1452 	return (0);
1453 }
1454 
1455 static struct csession *
1456 csefind(struct fcrypt *fcr, u_int ses)
1457 {
1458 	struct csession *cse;
1459 
1460 	mtx_lock(&fcr->lock);
1461 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
1462 		if (cse->ses == ses) {
1463 			refcount_acquire(&cse->refs);
1464 			mtx_unlock(&fcr->lock);
1465 			return (cse);
1466 		}
1467 	}
1468 	mtx_unlock(&fcr->lock);
1469 	return (NULL);
1470 }
1471 
1472 static bool
1473 csedelete(struct fcrypt *fcr, u_int ses)
1474 {
1475 	struct csession *cse;
1476 
1477 	mtx_lock(&fcr->lock);
1478 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
1479 		if (cse->ses == ses) {
1480 			TAILQ_REMOVE(&fcr->csessions, cse, next);
1481 			mtx_unlock(&fcr->lock);
1482 			csefree(cse);
1483 			return (true);
1484 		}
1485 	}
1486 	mtx_unlock(&fcr->lock);
1487 	return (false);
1488 }
1489 
1490 struct csession *
1491 csecreate(struct fcrypt *fcr, crypto_session_t cses,
1492     struct crypto_session_params *csp, struct enc_xform *txform,
1493     void *key, struct auth_hash *thash, void *mackey)
1494 {
1495 	struct csession *cse;
1496 
1497 	cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO);
1498 	if (cse == NULL)
1499 		return NULL;
1500 	mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
1501 	refcount_init(&cse->refs, 1);
1502 	cse->key = key;
1503 	cse->mackey = mackey;
1504 	cse->mode = csp->csp_mode;
1505 	cse->cses = cses;
1506 	cse->txform = txform;
1507 	if (thash != NULL)
1508 		cse->hashsize = thash->hashsize;
1509 	else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
1510 		cse->hashsize = AES_GMAC_HASH_LEN;
1511 	else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
1512 		cse->hashsize = AES_CBC_MAC_HASH_LEN;
1513 	cse->ivsize = csp->csp_ivlen;
1514 	mtx_lock(&fcr->lock);
1515 	TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
1516 	cse->ses = fcr->sesn++;
1517 	mtx_unlock(&fcr->lock);
1518 	return (cse);
1519 }
1520 
1521 static void
1522 csefree(struct csession *cse)
1523 {
1524 
1525 	if (!refcount_release(&cse->refs))
1526 		return;
1527 	crypto_freesession(cse->cses);
1528 	mtx_destroy(&cse->lock);
1529 	if (cse->key)
1530 		free(cse->key, M_XDATA);
1531 	if (cse->mackey)
1532 		free(cse->mackey, M_XDATA);
1533 	free(cse, M_XDATA);
1534 }
1535 
1536 static int
1537 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
1538 {
1539 	struct file *f;
1540 	struct fcrypt *fcr;
1541 	int fd, error;
1542 
1543 	switch (cmd) {
1544 	case CRIOGET:
1545 		error = falloc_noinstall(td, &f);
1546 		if (error)
1547 			break;
1548 
1549 		fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO);
1550 		TAILQ_INIT(&fcr->csessions);
1551 		mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1552 
1553 		finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops);
1554 		error = finstall(td, f, &fd, 0, NULL);
1555 		if (error) {
1556 			mtx_destroy(&fcr->lock);
1557 			free(fcr, M_XDATA);
1558 		} else
1559 			*(uint32_t *)data = fd;
1560 		fdrop(f, td);
1561 		break;
1562 	case CRIOFINDDEV:
1563 		error = cryptodev_find((struct crypt_find_op *)data);
1564 		break;
1565 	case CRIOASYMFEAT:
1566 		error = crypto_getfeat((int *)data);
1567 		break;
1568 	default:
1569 		error = EINVAL;
1570 		break;
1571 	}
1572 	return (error);
1573 }
1574 
1575 static struct cdevsw crypto_cdevsw = {
1576 	.d_version =	D_VERSION,
1577 	.d_ioctl =	cryptoioctl,
1578 	.d_name =	"crypto",
1579 };
1580 static struct cdev *crypto_dev;
1581 
1582 /*
1583  * Initialization code, both for static and dynamic loading.
1584  */
1585 static int
1586 cryptodev_modevent(module_t mod, int type, void *unused)
1587 {
1588 	switch (type) {
1589 	case MOD_LOAD:
1590 		if (bootverbose)
1591 			printf("crypto: <crypto device>\n");
1592 		crypto_dev = make_dev(&crypto_cdevsw, 0,
1593 				      UID_ROOT, GID_WHEEL, 0666,
1594 				      "crypto");
1595 		return 0;
1596 	case MOD_UNLOAD:
1597 		/*XXX disallow if active sessions */
1598 		destroy_dev(crypto_dev);
1599 		return 0;
1600 	}
1601 	return EINVAL;
1602 }
1603 
1604 static moduledata_t cryptodev_mod = {
1605 	"cryptodev",
1606 	cryptodev_modevent,
1607 	0
1608 };
1609 MODULE_VERSION(cryptodev, 1);
1610 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1611 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1612 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
1613