1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. The name of the author may not be used to endorse or promote products 23 * derived from this software without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * Effort sponsored in part by the Defense Advanced Research Projects 37 * Agency (DARPA) and Air Force Research Laboratory, Air Force 38 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 39 */ 40 41 #include <sys/cdefs.h> 42 __FBSDID("$FreeBSD$"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/lock.h> 49 #include <sys/mutex.h> 50 #include <sys/sysctl.h> 51 #include <sys/file.h> 52 #include <sys/filedesc.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/user.h> 61 #include <sys/sdt.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD32 71 #include <sys/mount.h> 72 #include <compat/freebsd32/freebsd32.h> 73 74 struct session_op32 { 75 u_int32_t cipher; 76 u_int32_t mac; 77 u_int32_t keylen; 78 u_int32_t key; 79 int mackeylen; 80 u_int32_t mackey; 81 u_int32_t ses; 82 }; 83 84 struct session2_op32 { 85 u_int32_t cipher; 86 u_int32_t mac; 87 u_int32_t keylen; 88 u_int32_t key; 89 int mackeylen; 90 u_int32_t mackey; 91 u_int32_t ses; 92 int crid; 93 int pad[4]; 94 }; 95 96 struct crypt_op32 { 97 u_int32_t ses; 98 u_int16_t op; 99 u_int16_t flags; 100 u_int len; 101 u_int32_t src, dst; 102 u_int32_t mac; 103 u_int32_t iv; 104 }; 105 106 struct crypt_aead32 { 107 u_int32_t ses; 108 u_int16_t op; 109 u_int16_t flags; 110 u_int len; 111 u_int aadlen; 112 u_int ivlen; 113 u_int32_t src; 114 u_int32_t dst; 115 u_int32_t aad; 116 u_int32_t tag; 117 u_int32_t iv; 118 }; 119 120 struct crparam32 { 121 u_int32_t crp_p; 122 u_int crp_nbits; 123 }; 124 125 struct crypt_kop32 { 126 u_int crk_op; 127 u_int crk_status; 128 u_short crk_iparams; 129 u_short crk_oparams; 130 u_int crk_crid; 131 struct crparam32 crk_param[CRK_MAXPARAM]; 132 }; 133 134 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 135 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 136 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 137 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 138 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 139 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 140 141 static void 142 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 143 { 144 145 memset(to, 0, sizeof(*to)); 146 CP(*from, *to, cipher); 147 CP(*from, *to, mac); 148 CP(*from, *to, keylen); 149 PTRIN_CP(*from, *to, key); 150 CP(*from, *to, mackeylen); 151 PTRIN_CP(*from, *to, mackey); 152 CP(*from, *to, ses); 153 to->crid = CRYPTOCAP_F_HARDWARE; 154 } 155 156 static void 157 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 158 { 159 160 session_op_from_32((const struct session_op32 *)from, to); 161 CP(*from, *to, crid); 162 } 163 164 static void 165 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 166 { 167 168 CP(*from, *to, cipher); 169 CP(*from, *to, mac); 170 CP(*from, *to, keylen); 171 PTROUT_CP(*from, *to, key); 172 CP(*from, *to, mackeylen); 173 PTROUT_CP(*from, *to, mackey); 174 CP(*from, *to, ses); 175 } 176 177 static void 178 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 179 { 180 181 session_op_to_32(from, (struct session_op32 *)to); 182 CP(*from, *to, crid); 183 } 184 185 static void 186 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 187 { 188 189 CP(*from, *to, ses); 190 CP(*from, *to, op); 191 CP(*from, *to, flags); 192 CP(*from, *to, len); 193 PTRIN_CP(*from, *to, src); 194 PTRIN_CP(*from, *to, dst); 195 PTRIN_CP(*from, *to, mac); 196 PTRIN_CP(*from, *to, iv); 197 } 198 199 static void 200 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 201 { 202 203 CP(*from, *to, ses); 204 CP(*from, *to, op); 205 CP(*from, *to, flags); 206 CP(*from, *to, len); 207 PTROUT_CP(*from, *to, src); 208 PTROUT_CP(*from, *to, dst); 209 PTROUT_CP(*from, *to, mac); 210 PTROUT_CP(*from, *to, iv); 211 } 212 213 static void 214 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 215 { 216 217 CP(*from, *to, ses); 218 CP(*from, *to, op); 219 CP(*from, *to, flags); 220 CP(*from, *to, len); 221 CP(*from, *to, aadlen); 222 CP(*from, *to, ivlen); 223 PTRIN_CP(*from, *to, src); 224 PTRIN_CP(*from, *to, dst); 225 PTRIN_CP(*from, *to, aad); 226 PTRIN_CP(*from, *to, tag); 227 PTRIN_CP(*from, *to, iv); 228 } 229 230 static void 231 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 232 { 233 234 CP(*from, *to, ses); 235 CP(*from, *to, op); 236 CP(*from, *to, flags); 237 CP(*from, *to, len); 238 CP(*from, *to, aadlen); 239 CP(*from, *to, ivlen); 240 PTROUT_CP(*from, *to, src); 241 PTROUT_CP(*from, *to, dst); 242 PTROUT_CP(*from, *to, aad); 243 PTROUT_CP(*from, *to, tag); 244 PTROUT_CP(*from, *to, iv); 245 } 246 247 static void 248 crparam_from_32(const struct crparam32 *from, struct crparam *to) 249 { 250 251 PTRIN_CP(*from, *to, crp_p); 252 CP(*from, *to, crp_nbits); 253 } 254 255 static void 256 crparam_to_32(const struct crparam *from, struct crparam32 *to) 257 { 258 259 PTROUT_CP(*from, *to, crp_p); 260 CP(*from, *to, crp_nbits); 261 } 262 263 static void 264 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 265 { 266 int i; 267 268 CP(*from, *to, crk_op); 269 CP(*from, *to, crk_status); 270 CP(*from, *to, crk_iparams); 271 CP(*from, *to, crk_oparams); 272 CP(*from, *to, crk_crid); 273 for (i = 0; i < CRK_MAXPARAM; i++) 274 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 275 } 276 277 static void 278 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 279 { 280 int i; 281 282 CP(*from, *to, crk_op); 283 CP(*from, *to, crk_status); 284 CP(*from, *to, crk_iparams); 285 CP(*from, *to, crk_oparams); 286 CP(*from, *to, crk_crid); 287 for (i = 0; i < CRK_MAXPARAM; i++) 288 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 289 } 290 #endif 291 292 static void 293 session2_op_from_op(const struct session_op *from, struct session2_op *to) 294 { 295 296 memset(to, 0, sizeof(*to)); 297 memcpy(to, from, sizeof(*from)); 298 to->crid = CRYPTOCAP_F_HARDWARE; 299 } 300 301 static void 302 session2_op_to_op(const struct session2_op *from, struct session_op *to) 303 { 304 305 memcpy(to, from, sizeof(*to)); 306 } 307 308 struct csession { 309 TAILQ_ENTRY(csession) next; 310 crypto_session_t cses; 311 volatile u_int refs; 312 u_int32_t ses; 313 struct mtx lock; /* for op submission */ 314 315 struct enc_xform *txform; 316 int hashsize; 317 int ivsize; 318 int mode; 319 320 void *key; 321 void *mackey; 322 }; 323 324 struct cryptop_data { 325 struct csession *cse; 326 327 char *buf; 328 char *obuf; 329 char *aad; 330 bool done; 331 }; 332 333 struct fcrypt { 334 TAILQ_HEAD(csessionlist, csession) csessions; 335 int sesn; 336 struct mtx lock; 337 }; 338 339 static bool use_outputbuffers; 340 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 341 &use_outputbuffers, 0, 342 "Use separate output buffers for /dev/crypto requests."); 343 344 static bool use_separate_aad; 345 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 346 &use_separate_aad, 0, 347 "Use separate AAD buffer for /dev/crypto requests."); 348 349 static int cryptof_ioctl(struct file *, u_long, void *, 350 struct ucred *, struct thread *); 351 static int cryptof_stat(struct file *, struct stat *, 352 struct ucred *, struct thread *); 353 static int cryptof_close(struct file *, struct thread *); 354 static int cryptof_fill_kinfo(struct file *, struct kinfo_file *, 355 struct filedesc *); 356 357 static struct fileops cryptofops = { 358 .fo_read = invfo_rdwr, 359 .fo_write = invfo_rdwr, 360 .fo_truncate = invfo_truncate, 361 .fo_ioctl = cryptof_ioctl, 362 .fo_poll = invfo_poll, 363 .fo_kqfilter = invfo_kqfilter, 364 .fo_stat = cryptof_stat, 365 .fo_close = cryptof_close, 366 .fo_chmod = invfo_chmod, 367 .fo_chown = invfo_chown, 368 .fo_sendfile = invfo_sendfile, 369 .fo_fill_kinfo = cryptof_fill_kinfo, 370 }; 371 372 static struct csession *csefind(struct fcrypt *, u_int); 373 static bool csedelete(struct fcrypt *, u_int); 374 static struct csession *csecreate(struct fcrypt *, crypto_session_t, 375 struct crypto_session_params *, struct enc_xform *, void *, 376 struct auth_hash *, void *); 377 static void csefree(struct csession *); 378 379 static int cryptodev_op(struct csession *, struct crypt_op *, 380 struct ucred *, struct thread *td); 381 static int cryptodev_aead(struct csession *, struct crypt_aead *, 382 struct ucred *, struct thread *); 383 static int cryptodev_key(struct crypt_kop *); 384 static int cryptodev_find(struct crypt_find_op *); 385 386 /* 387 * Check a crypto identifier to see if it requested 388 * a software device/driver. This can be done either 389 * by device name/class or through search constraints. 390 */ 391 static int 392 checkforsoftware(int *cridp) 393 { 394 int crid; 395 396 crid = *cridp; 397 398 if (!crypto_devallowsoft) { 399 if (crid & CRYPTOCAP_F_SOFTWARE) { 400 if (crid & CRYPTOCAP_F_HARDWARE) { 401 *cridp = CRYPTOCAP_F_HARDWARE; 402 return 0; 403 } 404 return EINVAL; 405 } 406 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 407 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 408 return EINVAL; 409 } 410 return 0; 411 } 412 413 /* ARGSUSED */ 414 static int 415 cryptof_ioctl( 416 struct file *fp, 417 u_long cmd, 418 void *data, 419 struct ucred *active_cred, 420 struct thread *td) 421 { 422 struct crypto_session_params csp; 423 struct fcrypt *fcr = fp->f_data; 424 struct csession *cse; 425 struct session2_op *sop; 426 struct crypt_op *cop; 427 struct crypt_aead *caead; 428 struct enc_xform *txform = NULL; 429 struct auth_hash *thash = NULL; 430 void *key = NULL; 431 void *mackey = NULL; 432 struct crypt_kop *kop; 433 crypto_session_t cses; 434 u_int32_t ses; 435 int error = 0, crid; 436 union { 437 struct session2_op sopc; 438 #ifdef COMPAT_FREEBSD32 439 struct crypt_op copc; 440 struct crypt_aead aeadc; 441 struct crypt_kop kopc; 442 #endif 443 } thunk; 444 #ifdef COMPAT_FREEBSD32 445 u_long cmd32; 446 void *data32; 447 448 cmd32 = 0; 449 data32 = NULL; 450 switch (cmd) { 451 case CIOCGSESSION32: 452 cmd32 = cmd; 453 data32 = data; 454 cmd = CIOCGSESSION; 455 data = &thunk.sopc; 456 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 457 break; 458 case CIOCGSESSION232: 459 cmd32 = cmd; 460 data32 = data; 461 cmd = CIOCGSESSION2; 462 data = &thunk.sopc; 463 session2_op_from_32((struct session2_op32 *)data32, 464 &thunk.sopc); 465 break; 466 case CIOCCRYPT32: 467 cmd32 = cmd; 468 data32 = data; 469 cmd = CIOCCRYPT; 470 data = &thunk.copc; 471 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 472 break; 473 case CIOCCRYPTAEAD32: 474 cmd32 = cmd; 475 data32 = data; 476 cmd = CIOCCRYPTAEAD; 477 data = &thunk.aeadc; 478 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 479 break; 480 case CIOCKEY32: 481 case CIOCKEY232: 482 cmd32 = cmd; 483 data32 = data; 484 if (cmd == CIOCKEY32) 485 cmd = CIOCKEY; 486 else 487 cmd = CIOCKEY2; 488 data = &thunk.kopc; 489 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc); 490 break; 491 } 492 #endif 493 494 switch (cmd) { 495 case CIOCGSESSION: 496 case CIOCGSESSION2: 497 if (cmd == CIOCGSESSION) { 498 session2_op_from_op(data, &thunk.sopc); 499 sop = &thunk.sopc; 500 } else 501 sop = (struct session2_op *)data; 502 503 switch (sop->cipher) { 504 case 0: 505 break; 506 case CRYPTO_AES_CBC: 507 txform = &enc_xform_rijndael128; 508 break; 509 case CRYPTO_AES_XTS: 510 txform = &enc_xform_aes_xts; 511 break; 512 case CRYPTO_NULL_CBC: 513 txform = &enc_xform_null; 514 break; 515 case CRYPTO_CAMELLIA_CBC: 516 txform = &enc_xform_camellia; 517 break; 518 case CRYPTO_AES_ICM: 519 txform = &enc_xform_aes_icm; 520 break; 521 case CRYPTO_AES_NIST_GCM_16: 522 txform = &enc_xform_aes_nist_gcm; 523 break; 524 case CRYPTO_CHACHA20: 525 txform = &enc_xform_chacha20; 526 break; 527 case CRYPTO_AES_CCM_16: 528 txform = &enc_xform_ccm; 529 break; 530 531 default: 532 CRYPTDEB("invalid cipher"); 533 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 534 return (EINVAL); 535 } 536 537 switch (sop->mac) { 538 case 0: 539 break; 540 case CRYPTO_POLY1305: 541 thash = &auth_hash_poly1305; 542 break; 543 case CRYPTO_SHA1_HMAC: 544 thash = &auth_hash_hmac_sha1; 545 break; 546 case CRYPTO_SHA2_224_HMAC: 547 thash = &auth_hash_hmac_sha2_224; 548 break; 549 case CRYPTO_SHA2_256_HMAC: 550 thash = &auth_hash_hmac_sha2_256; 551 break; 552 case CRYPTO_SHA2_384_HMAC: 553 thash = &auth_hash_hmac_sha2_384; 554 break; 555 case CRYPTO_SHA2_512_HMAC: 556 thash = &auth_hash_hmac_sha2_512; 557 break; 558 case CRYPTO_RIPEMD160_HMAC: 559 thash = &auth_hash_hmac_ripemd_160; 560 break; 561 #ifdef COMPAT_FREEBSD12 562 case CRYPTO_AES_128_NIST_GMAC: 563 case CRYPTO_AES_192_NIST_GMAC: 564 case CRYPTO_AES_256_NIST_GMAC: 565 /* Should always be paired with GCM. */ 566 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 567 CRYPTDEB("GMAC without GCM"); 568 SDT_PROBE1(opencrypto, dev, ioctl, error, 569 __LINE__); 570 return (EINVAL); 571 } 572 break; 573 #endif 574 case CRYPTO_AES_NIST_GMAC: 575 switch (sop->mackeylen * 8) { 576 case 128: 577 thash = &auth_hash_nist_gmac_aes_128; 578 break; 579 case 192: 580 thash = &auth_hash_nist_gmac_aes_192; 581 break; 582 case 256: 583 thash = &auth_hash_nist_gmac_aes_256; 584 break; 585 default: 586 CRYPTDEB("invalid GMAC key length"); 587 SDT_PROBE1(opencrypto, dev, ioctl, error, 588 __LINE__); 589 return (EINVAL); 590 } 591 break; 592 case CRYPTO_AES_CCM_CBC_MAC: 593 switch (sop->mackeylen) { 594 case 16: 595 thash = &auth_hash_ccm_cbc_mac_128; 596 break; 597 case 24: 598 thash = &auth_hash_ccm_cbc_mac_192; 599 break; 600 case 32: 601 thash = &auth_hash_ccm_cbc_mac_256; 602 break; 603 default: 604 CRYPTDEB("Invalid CBC MAC key size %d", 605 sop->keylen); 606 SDT_PROBE1(opencrypto, dev, ioctl, 607 error, __LINE__); 608 return (EINVAL); 609 } 610 break; 611 case CRYPTO_SHA1: 612 thash = &auth_hash_sha1; 613 break; 614 case CRYPTO_SHA2_224: 615 thash = &auth_hash_sha2_224; 616 break; 617 case CRYPTO_SHA2_256: 618 thash = &auth_hash_sha2_256; 619 break; 620 case CRYPTO_SHA2_384: 621 thash = &auth_hash_sha2_384; 622 break; 623 case CRYPTO_SHA2_512: 624 thash = &auth_hash_sha2_512; 625 break; 626 627 case CRYPTO_NULL_HMAC: 628 thash = &auth_hash_null; 629 break; 630 631 case CRYPTO_BLAKE2B: 632 thash = &auth_hash_blake2b; 633 break; 634 case CRYPTO_BLAKE2S: 635 thash = &auth_hash_blake2s; 636 break; 637 638 default: 639 CRYPTDEB("invalid mac"); 640 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 641 return (EINVAL); 642 } 643 644 if (txform == NULL && thash == NULL) { 645 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 646 return (EINVAL); 647 } 648 649 memset(&csp, 0, sizeof(csp)); 650 if (use_outputbuffers) 651 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 652 653 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 654 switch (sop->mac) { 655 #ifdef COMPAT_FREEBSD12 656 case CRYPTO_AES_128_NIST_GMAC: 657 case CRYPTO_AES_192_NIST_GMAC: 658 case CRYPTO_AES_256_NIST_GMAC: 659 if (sop->keylen != sop->mackeylen) { 660 SDT_PROBE1(opencrypto, dev, ioctl, 661 error, __LINE__); 662 return (EINVAL); 663 } 664 break; 665 #endif 666 case 0: 667 break; 668 default: 669 SDT_PROBE1(opencrypto, dev, ioctl, error, 670 __LINE__); 671 return (EINVAL); 672 } 673 csp.csp_mode = CSP_MODE_AEAD; 674 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 675 switch (sop->mac) { 676 #ifdef COMPAT_FREEBSD12 677 case CRYPTO_AES_CCM_CBC_MAC: 678 if (sop->keylen != sop->mackeylen) { 679 SDT_PROBE1(opencrypto, dev, ioctl, 680 error, __LINE__); 681 return (EINVAL); 682 } 683 thash = NULL; 684 break; 685 #endif 686 case 0: 687 break; 688 default: 689 SDT_PROBE1(opencrypto, dev, ioctl, error, 690 __LINE__); 691 return (EINVAL); 692 } 693 csp.csp_mode = CSP_MODE_AEAD; 694 } else if (txform && thash) 695 csp.csp_mode = CSP_MODE_ETA; 696 else if (txform) 697 csp.csp_mode = CSP_MODE_CIPHER; 698 else 699 csp.csp_mode = CSP_MODE_DIGEST; 700 701 switch (csp.csp_mode) { 702 case CSP_MODE_AEAD: 703 case CSP_MODE_ETA: 704 if (use_separate_aad) 705 csp.csp_flags |= CSP_F_SEPARATE_AAD; 706 break; 707 } 708 709 if (txform) { 710 csp.csp_cipher_alg = txform->type; 711 csp.csp_cipher_klen = sop->keylen; 712 if (sop->keylen > txform->maxkey || 713 sop->keylen < txform->minkey) { 714 CRYPTDEB("invalid cipher parameters"); 715 error = EINVAL; 716 SDT_PROBE1(opencrypto, dev, ioctl, error, 717 __LINE__); 718 goto bail; 719 } 720 721 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 722 error = copyin(sop->key, key, csp.csp_cipher_klen); 723 if (error) { 724 CRYPTDEB("invalid key"); 725 SDT_PROBE1(opencrypto, dev, ioctl, error, 726 __LINE__); 727 goto bail; 728 } 729 csp.csp_cipher_key = key; 730 csp.csp_ivlen = txform->ivsize; 731 } 732 733 if (thash) { 734 csp.csp_auth_alg = thash->type; 735 csp.csp_auth_klen = sop->mackeylen; 736 if (sop->mackeylen > thash->keysize || 737 sop->mackeylen < 0) { 738 CRYPTDEB("invalid mac key length"); 739 error = EINVAL; 740 SDT_PROBE1(opencrypto, dev, ioctl, error, 741 __LINE__); 742 goto bail; 743 } 744 745 if (csp.csp_auth_klen) { 746 mackey = malloc(csp.csp_auth_klen, M_XDATA, 747 M_WAITOK); 748 error = copyin(sop->mackey, mackey, 749 csp.csp_auth_klen); 750 if (error) { 751 CRYPTDEB("invalid mac key"); 752 SDT_PROBE1(opencrypto, dev, ioctl, 753 error, __LINE__); 754 goto bail; 755 } 756 csp.csp_auth_key = mackey; 757 } 758 759 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 760 csp.csp_ivlen = AES_GCM_IV_LEN; 761 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 762 csp.csp_ivlen = AES_CCM_IV_LEN; 763 } 764 765 crid = sop->crid; 766 error = checkforsoftware(&crid); 767 if (error) { 768 CRYPTDEB("checkforsoftware"); 769 SDT_PROBE1(opencrypto, dev, ioctl, error, 770 __LINE__); 771 goto bail; 772 } 773 error = crypto_newsession(&cses, &csp, crid); 774 if (error) { 775 CRYPTDEB("crypto_newsession"); 776 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 777 goto bail; 778 } 779 780 cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey); 781 782 if (cse == NULL) { 783 crypto_freesession(cses); 784 error = EINVAL; 785 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 786 CRYPTDEB("csecreate"); 787 goto bail; 788 } 789 sop->ses = cse->ses; 790 791 /* return hardware/driver id */ 792 sop->crid = crypto_ses2hid(cse->cses); 793 bail: 794 if (error) { 795 free(key, M_XDATA); 796 free(mackey, M_XDATA); 797 } 798 799 if (cmd == CIOCGSESSION && error == 0) 800 session2_op_to_op(sop, data); 801 break; 802 case CIOCFSESSION: 803 ses = *(u_int32_t *)data; 804 if (!csedelete(fcr, ses)) { 805 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 806 return (EINVAL); 807 } 808 break; 809 case CIOCCRYPT: 810 cop = (struct crypt_op *)data; 811 cse = csefind(fcr, cop->ses); 812 if (cse == NULL) { 813 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 814 return (EINVAL); 815 } 816 error = cryptodev_op(cse, cop, active_cred, td); 817 csefree(cse); 818 break; 819 case CIOCKEY: 820 case CIOCKEY2: 821 if (!crypto_userasymcrypto) { 822 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 823 return (EPERM); /* XXX compat? */ 824 } 825 kop = (struct crypt_kop *)data; 826 if (cmd == CIOCKEY) { 827 /* NB: crypto core enforces s/w driver use */ 828 kop->crk_crid = 829 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 830 } 831 mtx_lock(&Giant); 832 error = cryptodev_key(kop); 833 mtx_unlock(&Giant); 834 break; 835 case CIOCASYMFEAT: 836 if (!crypto_userasymcrypto) { 837 /* 838 * NB: if user asym crypto operations are 839 * not permitted return "no algorithms" 840 * so well-behaved applications will just 841 * fallback to doing them in software. 842 */ 843 *(int *)data = 0; 844 } else { 845 error = crypto_getfeat((int *)data); 846 if (error) 847 SDT_PROBE1(opencrypto, dev, ioctl, error, 848 __LINE__); 849 } 850 break; 851 case CIOCFINDDEV: 852 error = cryptodev_find((struct crypt_find_op *)data); 853 break; 854 case CIOCCRYPTAEAD: 855 caead = (struct crypt_aead *)data; 856 cse = csefind(fcr, caead->ses); 857 if (cse == NULL) { 858 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 859 return (EINVAL); 860 } 861 error = cryptodev_aead(cse, caead, active_cred, td); 862 csefree(cse); 863 break; 864 default: 865 error = EINVAL; 866 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 867 break; 868 } 869 870 #ifdef COMPAT_FREEBSD32 871 switch (cmd32) { 872 case CIOCGSESSION32: 873 if (error == 0) 874 session_op_to_32(data, data32); 875 break; 876 case CIOCGSESSION232: 877 if (error == 0) 878 session2_op_to_32(data, data32); 879 break; 880 case CIOCCRYPT32: 881 if (error == 0) 882 crypt_op_to_32(data, data32); 883 break; 884 case CIOCCRYPTAEAD32: 885 if (error == 0) 886 crypt_aead_to_32(data, data32); 887 break; 888 case CIOCKEY32: 889 case CIOCKEY232: 890 crypt_kop_to_32(data, data32); 891 break; 892 } 893 #endif 894 return (error); 895 } 896 897 static int cryptodev_cb(struct cryptop *); 898 899 static struct cryptop_data * 900 cod_alloc(struct csession *cse, size_t aad_len, size_t len, struct thread *td) 901 { 902 struct cryptop_data *cod; 903 904 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 905 906 cod->cse = cse; 907 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 908 if (aad_len != 0) 909 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 910 cod->buf = malloc(len, M_XDATA, M_WAITOK); 911 } else 912 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 913 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 914 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 915 return (cod); 916 } 917 918 static void 919 cod_free(struct cryptop_data *cod) 920 { 921 922 free(cod->aad, M_XDATA); 923 free(cod->obuf, M_XDATA); 924 free(cod->buf, M_XDATA); 925 free(cod, M_XDATA); 926 } 927 928 static int 929 cryptodev_op( 930 struct csession *cse, 931 struct crypt_op *cop, 932 struct ucred *active_cred, 933 struct thread *td) 934 { 935 struct cryptop_data *cod = NULL; 936 struct cryptop *crp = NULL; 937 int error; 938 939 if (cop->len > 256*1024-4) { 940 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 941 return (E2BIG); 942 } 943 944 if (cse->txform) { 945 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) { 946 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 947 return (EINVAL); 948 } 949 } 950 951 if (cop->mac && cse->hashsize == 0) { 952 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 953 return (EINVAL); 954 } 955 956 /* 957 * The COP_F_CIPHER_FIRST flag predates explicit session 958 * modes, but the only way it was used was for EtA so allow it 959 * as long as it is consistent with EtA. 960 */ 961 if (cop->flags & COP_F_CIPHER_FIRST) { 962 if (cop->op != COP_ENCRYPT) { 963 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 964 return (EINVAL); 965 } 966 } 967 968 cod = cod_alloc(cse, 0, cop->len + cse->hashsize, td); 969 970 crp = crypto_getreq(cse->cses, M_WAITOK); 971 972 error = copyin(cop->src, cod->buf, cop->len); 973 if (error) { 974 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 975 goto bail; 976 } 977 crp->crp_payload_start = 0; 978 crp->crp_payload_length = cop->len; 979 if (cse->hashsize) 980 crp->crp_digest_start = cop->len; 981 982 switch (cse->mode) { 983 case CSP_MODE_COMPRESS: 984 switch (cop->op) { 985 case COP_ENCRYPT: 986 crp->crp_op = CRYPTO_OP_COMPRESS; 987 break; 988 case COP_DECRYPT: 989 crp->crp_op = CRYPTO_OP_DECOMPRESS; 990 break; 991 default: 992 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 993 error = EINVAL; 994 goto bail; 995 } 996 break; 997 case CSP_MODE_CIPHER: 998 switch (cop->op) { 999 case COP_ENCRYPT: 1000 crp->crp_op = CRYPTO_OP_ENCRYPT; 1001 break; 1002 case COP_DECRYPT: 1003 crp->crp_op = CRYPTO_OP_DECRYPT; 1004 break; 1005 default: 1006 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1007 error = EINVAL; 1008 goto bail; 1009 } 1010 break; 1011 case CSP_MODE_DIGEST: 1012 switch (cop->op) { 1013 case 0: 1014 case COP_ENCRYPT: 1015 case COP_DECRYPT: 1016 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 1017 if (cod->obuf != NULL) 1018 crp->crp_digest_start = 0; 1019 break; 1020 default: 1021 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1022 error = EINVAL; 1023 goto bail; 1024 } 1025 break; 1026 case CSP_MODE_ETA: 1027 switch (cop->op) { 1028 case COP_ENCRYPT: 1029 crp->crp_op = CRYPTO_OP_ENCRYPT | 1030 CRYPTO_OP_COMPUTE_DIGEST; 1031 break; 1032 case COP_DECRYPT: 1033 crp->crp_op = CRYPTO_OP_DECRYPT | 1034 CRYPTO_OP_VERIFY_DIGEST; 1035 break; 1036 default: 1037 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1038 error = EINVAL; 1039 goto bail; 1040 } 1041 break; 1042 default: 1043 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1044 error = EINVAL; 1045 goto bail; 1046 } 1047 1048 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 1049 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 1050 if (cod->obuf) 1051 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 1052 crp->crp_callback = cryptodev_cb; 1053 crp->crp_opaque = cod; 1054 1055 if (cop->iv) { 1056 if (cse->ivsize == 0) { 1057 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1058 error = EINVAL; 1059 goto bail; 1060 } 1061 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 1062 if (error) { 1063 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1064 goto bail; 1065 } 1066 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1067 } else if (cse->ivsize != 0) { 1068 crp->crp_iv_start = 0; 1069 crp->crp_payload_start += cse->ivsize; 1070 crp->crp_payload_length -= cse->ivsize; 1071 cop->dst += cse->ivsize; 1072 } 1073 1074 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1075 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 1076 cse->hashsize); 1077 if (error) { 1078 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1079 goto bail; 1080 } 1081 } 1082 again: 1083 /* 1084 * Let the dispatch run unlocked, then, interlock against the 1085 * callback before checking if the operation completed and going 1086 * to sleep. This insures drivers don't inherit our lock which 1087 * results in a lock order reversal between crypto_dispatch forced 1088 * entry and the crypto_done callback into us. 1089 */ 1090 error = crypto_dispatch(crp); 1091 if (error != 0) { 1092 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1093 goto bail; 1094 } 1095 1096 mtx_lock(&cse->lock); 1097 while (!cod->done) 1098 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1099 mtx_unlock(&cse->lock); 1100 1101 if (crp->crp_etype == EAGAIN) { 1102 crp->crp_etype = 0; 1103 crp->crp_flags &= ~CRYPTO_F_DONE; 1104 cod->done = false; 1105 goto again; 1106 } 1107 1108 if (crp->crp_etype != 0) { 1109 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1110 error = crp->crp_etype; 1111 goto bail; 1112 } 1113 1114 if (cop->dst != NULL) { 1115 error = copyout(cod->obuf != NULL ? cod->obuf : 1116 cod->buf + crp->crp_payload_start, cop->dst, 1117 crp->crp_payload_length); 1118 if (error) { 1119 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1120 goto bail; 1121 } 1122 } 1123 1124 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1125 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1126 crp->crp_digest_start, cop->mac, cse->hashsize); 1127 if (error) { 1128 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1129 goto bail; 1130 } 1131 } 1132 1133 bail: 1134 crypto_freereq(crp); 1135 cod_free(cod); 1136 1137 return (error); 1138 } 1139 1140 static int 1141 cryptodev_aead( 1142 struct csession *cse, 1143 struct crypt_aead *caead, 1144 struct ucred *active_cred, 1145 struct thread *td) 1146 { 1147 struct cryptop_data *cod = NULL; 1148 struct cryptop *crp = NULL; 1149 int error; 1150 1151 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1152 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1153 return (E2BIG); 1154 } 1155 1156 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1157 (caead->len % cse->txform->blocksize) != 0) { 1158 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1159 return (EINVAL); 1160 } 1161 1162 /* 1163 * The COP_F_CIPHER_FIRST flag predates explicit session 1164 * modes, but the only way it was used was for EtA so allow it 1165 * as long as it is consistent with EtA. 1166 */ 1167 if (caead->flags & COP_F_CIPHER_FIRST) { 1168 if (caead->op != COP_ENCRYPT) { 1169 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1170 return (EINVAL); 1171 } 1172 } 1173 1174 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize, td); 1175 1176 crp = crypto_getreq(cse->cses, M_WAITOK); 1177 1178 if (cod->aad != NULL) 1179 error = copyin(caead->aad, cod->aad, caead->aadlen); 1180 else 1181 error = copyin(caead->aad, cod->buf, caead->aadlen); 1182 if (error) { 1183 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1184 goto bail; 1185 } 1186 crp->crp_aad = cod->aad; 1187 crp->crp_aad_start = 0; 1188 crp->crp_aad_length = caead->aadlen; 1189 1190 if (cod->aad != NULL) 1191 crp->crp_payload_start = 0; 1192 else 1193 crp->crp_payload_start = caead->aadlen; 1194 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1195 caead->len); 1196 if (error) { 1197 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1198 goto bail; 1199 } 1200 crp->crp_payload_length = caead->len; 1201 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1202 crp->crp_digest_start = crp->crp_payload_output_start + 1203 caead->len; 1204 else 1205 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1206 1207 switch (cse->mode) { 1208 case CSP_MODE_AEAD: 1209 case CSP_MODE_ETA: 1210 switch (caead->op) { 1211 case COP_ENCRYPT: 1212 crp->crp_op = CRYPTO_OP_ENCRYPT | 1213 CRYPTO_OP_COMPUTE_DIGEST; 1214 break; 1215 case COP_DECRYPT: 1216 crp->crp_op = CRYPTO_OP_DECRYPT | 1217 CRYPTO_OP_VERIFY_DIGEST; 1218 break; 1219 default: 1220 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1221 error = EINVAL; 1222 goto bail; 1223 } 1224 break; 1225 default: 1226 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1227 error = EINVAL; 1228 goto bail; 1229 } 1230 1231 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1232 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1233 cse->hashsize); 1234 if (cod->obuf != NULL) 1235 crypto_use_output_buf(crp, cod->obuf, caead->len + 1236 cse->hashsize); 1237 crp->crp_callback = cryptodev_cb; 1238 crp->crp_opaque = cod; 1239 1240 if (caead->iv) { 1241 /* 1242 * Permit a 16-byte IV for AES-XTS, but only use the 1243 * first 8 bytes as a block number. 1244 */ 1245 if (cse->mode == CSP_MODE_ETA && 1246 caead->ivlen == AES_BLOCK_LEN && 1247 cse->ivsize == AES_XTS_IV_LEN) 1248 caead->ivlen = AES_XTS_IV_LEN; 1249 1250 if (caead->ivlen != cse->ivsize) { 1251 error = EINVAL; 1252 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1253 goto bail; 1254 } 1255 1256 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1257 if (error) { 1258 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1259 goto bail; 1260 } 1261 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1262 } else { 1263 crp->crp_iv_start = crp->crp_payload_start; 1264 crp->crp_payload_start += cse->ivsize; 1265 crp->crp_payload_length -= cse->ivsize; 1266 caead->dst += cse->ivsize; 1267 } 1268 1269 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1270 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1271 cse->hashsize); 1272 if (error) { 1273 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1274 goto bail; 1275 } 1276 } 1277 again: 1278 /* 1279 * Let the dispatch run unlocked, then, interlock against the 1280 * callback before checking if the operation completed and going 1281 * to sleep. This insures drivers don't inherit our lock which 1282 * results in a lock order reversal between crypto_dispatch forced 1283 * entry and the crypto_done callback into us. 1284 */ 1285 error = crypto_dispatch(crp); 1286 if (error != 0) { 1287 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1288 goto bail; 1289 } 1290 1291 mtx_lock(&cse->lock); 1292 while (!cod->done) 1293 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1294 mtx_unlock(&cse->lock); 1295 1296 if (crp->crp_etype == EAGAIN) { 1297 crp->crp_etype = 0; 1298 crp->crp_flags &= ~CRYPTO_F_DONE; 1299 cod->done = false; 1300 goto again; 1301 } 1302 1303 if (crp->crp_etype != 0) { 1304 error = crp->crp_etype; 1305 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1306 goto bail; 1307 } 1308 1309 if (caead->dst != NULL) { 1310 error = copyout(cod->obuf != NULL ? cod->obuf : 1311 cod->buf + crp->crp_payload_start, caead->dst, 1312 crp->crp_payload_length); 1313 if (error) { 1314 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1315 goto bail; 1316 } 1317 } 1318 1319 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1320 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1321 crp->crp_digest_start, caead->tag, cse->hashsize); 1322 if (error) { 1323 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1324 goto bail; 1325 } 1326 } 1327 1328 bail: 1329 crypto_freereq(crp); 1330 cod_free(cod); 1331 1332 return (error); 1333 } 1334 1335 static int 1336 cryptodev_cb(struct cryptop *crp) 1337 { 1338 struct cryptop_data *cod = crp->crp_opaque; 1339 1340 /* 1341 * Lock to ensure the wakeup() is not missed by the loops 1342 * waiting on cod->done in cryptodev_op() and 1343 * cryptodev_aead(). 1344 */ 1345 mtx_lock(&cod->cse->lock); 1346 cod->done = true; 1347 mtx_unlock(&cod->cse->lock); 1348 wakeup(cod); 1349 return (0); 1350 } 1351 1352 static void 1353 cryptodevkey_cb(struct cryptkop *krp) 1354 { 1355 1356 wakeup_one(krp); 1357 } 1358 1359 static int 1360 cryptodev_key(struct crypt_kop *kop) 1361 { 1362 struct cryptkop *krp = NULL; 1363 int error = EINVAL; 1364 int in, out, size, i; 1365 1366 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1367 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1368 return (EFBIG); 1369 } 1370 1371 in = kop->crk_iparams; 1372 out = kop->crk_oparams; 1373 switch (kop->crk_op) { 1374 case CRK_MOD_EXP: 1375 if (in == 3 && out == 1) 1376 break; 1377 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1378 return (EINVAL); 1379 case CRK_MOD_EXP_CRT: 1380 if (in == 6 && out == 1) 1381 break; 1382 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1383 return (EINVAL); 1384 case CRK_DSA_SIGN: 1385 if (in == 5 && out == 2) 1386 break; 1387 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1388 return (EINVAL); 1389 case CRK_DSA_VERIFY: 1390 if (in == 7 && out == 0) 1391 break; 1392 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1393 return (EINVAL); 1394 case CRK_DH_COMPUTE_KEY: 1395 if (in == 3 && out == 1) 1396 break; 1397 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1398 return (EINVAL); 1399 default: 1400 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1401 return (EINVAL); 1402 } 1403 1404 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1405 krp->krp_op = kop->crk_op; 1406 krp->krp_status = kop->crk_status; 1407 krp->krp_iparams = kop->crk_iparams; 1408 krp->krp_oparams = kop->crk_oparams; 1409 krp->krp_crid = kop->crk_crid; 1410 krp->krp_status = 0; 1411 krp->krp_callback = cryptodevkey_cb; 1412 1413 for (i = 0; i < CRK_MAXPARAM; i++) { 1414 if (kop->crk_param[i].crp_nbits > 65536) { 1415 /* Limit is the same as in OpenBSD */ 1416 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1417 goto fail; 1418 } 1419 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1420 } 1421 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1422 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1423 if (size == 0) 1424 continue; 1425 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1426 if (i >= krp->krp_iparams) 1427 continue; 1428 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1429 if (error) { 1430 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1431 goto fail; 1432 } 1433 } 1434 1435 error = crypto_kdispatch(krp); 1436 if (error) { 1437 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1438 goto fail; 1439 } 1440 error = tsleep(krp, PSOCK, "crydev", 0); 1441 if (error) { 1442 /* XXX can this happen? if so, how do we recover? */ 1443 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1444 goto fail; 1445 } 1446 1447 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1448 if (krp->krp_status != 0) { 1449 error = krp->krp_status; 1450 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1451 goto fail; 1452 } 1453 1454 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1455 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1456 if (size == 0) 1457 continue; 1458 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1459 if (error) { 1460 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1461 goto fail; 1462 } 1463 } 1464 1465 fail: 1466 if (krp) { 1467 kop->crk_status = krp->krp_status; 1468 for (i = 0; i < CRK_MAXPARAM; i++) { 1469 if (krp->krp_param[i].crp_p) 1470 free(krp->krp_param[i].crp_p, M_XDATA); 1471 } 1472 free(krp, M_XDATA); 1473 } 1474 return (error); 1475 } 1476 1477 static int 1478 cryptodev_find(struct crypt_find_op *find) 1479 { 1480 device_t dev; 1481 size_t fnlen = sizeof find->name; 1482 1483 if (find->crid != -1) { 1484 dev = crypto_find_device_byhid(find->crid); 1485 if (dev == NULL) 1486 return (ENOENT); 1487 strncpy(find->name, device_get_nameunit(dev), fnlen); 1488 find->name[fnlen - 1] = '\x0'; 1489 } else { 1490 find->name[fnlen - 1] = '\x0'; 1491 find->crid = crypto_find_driver(find->name); 1492 if (find->crid == -1) 1493 return (ENOENT); 1494 } 1495 return (0); 1496 } 1497 1498 /* ARGSUSED */ 1499 static int 1500 cryptof_stat( 1501 struct file *fp, 1502 struct stat *sb, 1503 struct ucred *active_cred, 1504 struct thread *td) 1505 { 1506 1507 return (EOPNOTSUPP); 1508 } 1509 1510 /* ARGSUSED */ 1511 static int 1512 cryptof_close(struct file *fp, struct thread *td) 1513 { 1514 struct fcrypt *fcr = fp->f_data; 1515 struct csession *cse; 1516 1517 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1518 TAILQ_REMOVE(&fcr->csessions, cse, next); 1519 KASSERT(cse->refs == 1, 1520 ("%s: crypto session %p with %d refs", __func__, cse, 1521 cse->refs)); 1522 csefree(cse); 1523 } 1524 free(fcr, M_XDATA); 1525 fp->f_data = NULL; 1526 return 0; 1527 } 1528 1529 static int 1530 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) 1531 { 1532 1533 kif->kf_type = KF_TYPE_CRYPTO; 1534 return (0); 1535 } 1536 1537 static struct csession * 1538 csefind(struct fcrypt *fcr, u_int ses) 1539 { 1540 struct csession *cse; 1541 1542 mtx_lock(&fcr->lock); 1543 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1544 if (cse->ses == ses) { 1545 refcount_acquire(&cse->refs); 1546 mtx_unlock(&fcr->lock); 1547 return (cse); 1548 } 1549 } 1550 mtx_unlock(&fcr->lock); 1551 return (NULL); 1552 } 1553 1554 static bool 1555 csedelete(struct fcrypt *fcr, u_int ses) 1556 { 1557 struct csession *cse; 1558 1559 mtx_lock(&fcr->lock); 1560 TAILQ_FOREACH(cse, &fcr->csessions, next) { 1561 if (cse->ses == ses) { 1562 TAILQ_REMOVE(&fcr->csessions, cse, next); 1563 mtx_unlock(&fcr->lock); 1564 csefree(cse); 1565 return (true); 1566 } 1567 } 1568 mtx_unlock(&fcr->lock); 1569 return (false); 1570 } 1571 1572 struct csession * 1573 csecreate(struct fcrypt *fcr, crypto_session_t cses, 1574 struct crypto_session_params *csp, struct enc_xform *txform, 1575 void *key, struct auth_hash *thash, void *mackey) 1576 { 1577 struct csession *cse; 1578 1579 cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO); 1580 if (cse == NULL) 1581 return NULL; 1582 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 1583 refcount_init(&cse->refs, 1); 1584 cse->key = key; 1585 cse->mackey = mackey; 1586 cse->mode = csp->csp_mode; 1587 cse->cses = cses; 1588 cse->txform = txform; 1589 if (thash != NULL) 1590 cse->hashsize = thash->hashsize; 1591 else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 1592 cse->hashsize = AES_GMAC_HASH_LEN; 1593 else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16) 1594 cse->hashsize = AES_CBC_MAC_HASH_LEN; 1595 cse->ivsize = csp->csp_ivlen; 1596 mtx_lock(&fcr->lock); 1597 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 1598 cse->ses = fcr->sesn++; 1599 mtx_unlock(&fcr->lock); 1600 return (cse); 1601 } 1602 1603 static void 1604 csefree(struct csession *cse) 1605 { 1606 1607 if (!refcount_release(&cse->refs)) 1608 return; 1609 crypto_freesession(cse->cses); 1610 mtx_destroy(&cse->lock); 1611 if (cse->key) 1612 free(cse->key, M_XDATA); 1613 if (cse->mackey) 1614 free(cse->mackey, M_XDATA); 1615 free(cse, M_XDATA); 1616 } 1617 1618 static int 1619 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td) 1620 { 1621 struct file *f; 1622 struct fcrypt *fcr; 1623 int fd, error; 1624 1625 switch (cmd) { 1626 case CRIOGET: 1627 error = falloc_noinstall(td, &f); 1628 if (error) 1629 break; 1630 1631 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1632 TAILQ_INIT(&fcr->csessions); 1633 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1634 1635 finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops); 1636 error = finstall(td, f, &fd, 0, NULL); 1637 if (error) { 1638 mtx_destroy(&fcr->lock); 1639 free(fcr, M_XDATA); 1640 } else 1641 *(uint32_t *)data = fd; 1642 fdrop(f, td); 1643 break; 1644 case CRIOFINDDEV: 1645 error = cryptodev_find((struct crypt_find_op *)data); 1646 break; 1647 case CRIOASYMFEAT: 1648 error = crypto_getfeat((int *)data); 1649 break; 1650 default: 1651 error = EINVAL; 1652 break; 1653 } 1654 return (error); 1655 } 1656 1657 static struct cdevsw crypto_cdevsw = { 1658 .d_version = D_VERSION, 1659 .d_ioctl = cryptoioctl, 1660 .d_name = "crypto", 1661 }; 1662 static struct cdev *crypto_dev; 1663 1664 /* 1665 * Initialization code, both for static and dynamic loading. 1666 */ 1667 static int 1668 cryptodev_modevent(module_t mod, int type, void *unused) 1669 { 1670 switch (type) { 1671 case MOD_LOAD: 1672 if (bootverbose) 1673 printf("crypto: <crypto device>\n"); 1674 crypto_dev = make_dev(&crypto_cdevsw, 0, 1675 UID_ROOT, GID_WHEEL, 0666, 1676 "crypto"); 1677 return 0; 1678 case MOD_UNLOAD: 1679 /*XXX disallow if active sessions */ 1680 destroy_dev(crypto_dev); 1681 return 0; 1682 } 1683 return EINVAL; 1684 } 1685 1686 static moduledata_t cryptodev_mod = { 1687 "cryptodev", 1688 cryptodev_modevent, 1689 0 1690 }; 1691 MODULE_VERSION(cryptodev, 1); 1692 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1693 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1694 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1695