1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014-2021 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Portions of this software were developed by Ararat River 14 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. The name of the author may not be used to endorse or promote products 26 * derived from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Effort sponsored in part by the Defense Advanced Research Projects 40 * Agency (DARPA) and Air Force Research Laboratory, Air Force 41 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 42 */ 43 44 #include <sys/cdefs.h> 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/lock.h> 50 #include <sys/mutex.h> 51 #include <sys/proc.h> 52 #include <sys/sysctl.h> 53 #include <sys/errno.h> 54 #include <sys/random.h> 55 #include <sys/conf.h> 56 #include <sys/kernel.h> 57 #include <sys/module.h> 58 #include <sys/fcntl.h> 59 #include <sys/bus.h> 60 #include <sys/sdt.h> 61 #include <sys/syscallsubr.h> 62 63 #include <opencrypto/cryptodev.h> 64 #include <opencrypto/xform.h> 65 66 SDT_PROVIDER_DECLARE(opencrypto); 67 68 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 69 70 #ifdef COMPAT_FREEBSD12 71 /* 72 * Previously, most ioctls were performed against a cloned descriptor 73 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 74 * against /dev/crypto directly. 75 */ 76 #define CRIOGET _IOWR('c', 100, uint32_t) 77 #endif 78 79 /* the following are done against the cloned descriptor */ 80 81 #ifdef COMPAT_FREEBSD32 82 #include <sys/mount.h> 83 #include <compat/freebsd32/freebsd32.h> 84 85 struct session_op32 { 86 uint32_t cipher; 87 uint32_t mac; 88 uint32_t keylen; 89 uint32_t key; 90 int mackeylen; 91 uint32_t mackey; 92 uint32_t ses; 93 }; 94 95 struct session2_op32 { 96 uint32_t cipher; 97 uint32_t mac; 98 uint32_t keylen; 99 uint32_t key; 100 int mackeylen; 101 uint32_t mackey; 102 uint32_t ses; 103 int crid; 104 int ivlen; 105 int maclen; 106 int pad[2]; 107 }; 108 109 struct crypt_op32 { 110 uint32_t ses; 111 uint16_t op; 112 uint16_t flags; 113 u_int len; 114 uint32_t src, dst; 115 uint32_t mac; 116 uint32_t iv; 117 }; 118 119 struct crypt_aead32 { 120 uint32_t ses; 121 uint16_t op; 122 uint16_t flags; 123 u_int len; 124 u_int aadlen; 125 u_int ivlen; 126 uint32_t src; 127 uint32_t dst; 128 uint32_t aad; 129 uint32_t tag; 130 uint32_t iv; 131 }; 132 133 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 134 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 135 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 136 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 137 138 static void 139 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 140 { 141 142 memset(to, 0, sizeof(*to)); 143 CP(*from, *to, cipher); 144 CP(*from, *to, mac); 145 CP(*from, *to, keylen); 146 PTRIN_CP(*from, *to, key); 147 CP(*from, *to, mackeylen); 148 PTRIN_CP(*from, *to, mackey); 149 CP(*from, *to, ses); 150 to->crid = CRYPTOCAP_F_HARDWARE; 151 } 152 153 static void 154 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 155 { 156 157 session_op_from_32((const struct session_op32 *)from, to); 158 CP(*from, *to, crid); 159 CP(*from, *to, ivlen); 160 CP(*from, *to, maclen); 161 } 162 163 static void 164 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 165 { 166 167 CP(*from, *to, cipher); 168 CP(*from, *to, mac); 169 CP(*from, *to, keylen); 170 PTROUT_CP(*from, *to, key); 171 CP(*from, *to, mackeylen); 172 PTROUT_CP(*from, *to, mackey); 173 CP(*from, *to, ses); 174 } 175 176 static void 177 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 178 { 179 180 session_op_to_32(from, (struct session_op32 *)to); 181 CP(*from, *to, crid); 182 } 183 184 static void 185 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 186 { 187 188 CP(*from, *to, ses); 189 CP(*from, *to, op); 190 CP(*from, *to, flags); 191 CP(*from, *to, len); 192 PTRIN_CP(*from, *to, src); 193 PTRIN_CP(*from, *to, dst); 194 PTRIN_CP(*from, *to, mac); 195 PTRIN_CP(*from, *to, iv); 196 } 197 198 static void 199 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 200 { 201 202 CP(*from, *to, ses); 203 CP(*from, *to, op); 204 CP(*from, *to, flags); 205 CP(*from, *to, len); 206 PTROUT_CP(*from, *to, src); 207 PTROUT_CP(*from, *to, dst); 208 PTROUT_CP(*from, *to, mac); 209 PTROUT_CP(*from, *to, iv); 210 } 211 212 static void 213 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 214 { 215 216 CP(*from, *to, ses); 217 CP(*from, *to, op); 218 CP(*from, *to, flags); 219 CP(*from, *to, len); 220 CP(*from, *to, aadlen); 221 CP(*from, *to, ivlen); 222 PTRIN_CP(*from, *to, src); 223 PTRIN_CP(*from, *to, dst); 224 PTRIN_CP(*from, *to, aad); 225 PTRIN_CP(*from, *to, tag); 226 PTRIN_CP(*from, *to, iv); 227 } 228 229 static void 230 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 231 { 232 233 CP(*from, *to, ses); 234 CP(*from, *to, op); 235 CP(*from, *to, flags); 236 CP(*from, *to, len); 237 CP(*from, *to, aadlen); 238 CP(*from, *to, ivlen); 239 PTROUT_CP(*from, *to, src); 240 PTROUT_CP(*from, *to, dst); 241 PTROUT_CP(*from, *to, aad); 242 PTROUT_CP(*from, *to, tag); 243 PTROUT_CP(*from, *to, iv); 244 } 245 #endif 246 247 static void 248 session2_op_from_op(const struct session_op *from, struct session2_op *to) 249 { 250 251 memset(to, 0, sizeof(*to)); 252 memcpy(to, from, sizeof(*from)); 253 to->crid = CRYPTOCAP_F_HARDWARE; 254 } 255 256 static void 257 session2_op_to_op(const struct session2_op *from, struct session_op *to) 258 { 259 260 memcpy(to, from, sizeof(*to)); 261 } 262 263 struct csession { 264 TAILQ_ENTRY(csession) next; 265 crypto_session_t cses; 266 volatile u_int refs; 267 uint32_t ses; 268 struct mtx lock; /* for op submission */ 269 270 u_int blocksize; 271 int hashsize; 272 int ivsize; 273 274 void *key; 275 void *mackey; 276 }; 277 278 struct cryptop_data { 279 struct csession *cse; 280 281 char *buf; 282 char *obuf; 283 char *aad; 284 bool done; 285 }; 286 287 struct fcrypt { 288 TAILQ_HEAD(csessionlist, csession) csessions; 289 int sesn; 290 struct mtx lock; 291 }; 292 293 static bool use_outputbuffers; 294 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 295 &use_outputbuffers, 0, 296 "Use separate output buffers for /dev/crypto requests."); 297 298 static bool use_separate_aad; 299 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 300 &use_separate_aad, 0, 301 "Use separate AAD buffer for /dev/crypto requests."); 302 303 static MALLOC_DEFINE(M_CRYPTODEV, "cryptodev", "/dev/crypto data buffers"); 304 305 /* 306 * Check a crypto identifier to see if it requested 307 * a software device/driver. This can be done either 308 * by device name/class or through search constraints. 309 */ 310 static int 311 checkforsoftware(int *cridp) 312 { 313 int crid; 314 315 crid = *cridp; 316 317 if (!crypto_devallowsoft) { 318 if (crid & CRYPTOCAP_F_SOFTWARE) { 319 if (crid & CRYPTOCAP_F_HARDWARE) { 320 *cridp = CRYPTOCAP_F_HARDWARE; 321 return 0; 322 } 323 return EINVAL; 324 } 325 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 326 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 327 return EINVAL; 328 } 329 return 0; 330 } 331 332 static int 333 cse_create(struct fcrypt *fcr, struct session2_op *sop) 334 { 335 struct crypto_session_params csp; 336 struct csession *cse; 337 const struct enc_xform *txform; 338 const struct auth_hash *thash; 339 void *key = NULL; 340 void *mackey = NULL; 341 crypto_session_t cses; 342 int crid, error, mac; 343 344 mac = sop->mac; 345 #ifdef COMPAT_FREEBSD12 346 switch (sop->mac) { 347 case CRYPTO_AES_128_NIST_GMAC: 348 case CRYPTO_AES_192_NIST_GMAC: 349 case CRYPTO_AES_256_NIST_GMAC: 350 /* Should always be paired with GCM. */ 351 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 352 CRYPTDEB("GMAC without GCM"); 353 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 354 return (EINVAL); 355 } 356 if (sop->keylen != sop->mackeylen) { 357 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 358 return (EINVAL); 359 } 360 mac = 0; 361 break; 362 case CRYPTO_AES_CCM_CBC_MAC: 363 /* Should always be paired with CCM. */ 364 if (sop->cipher != CRYPTO_AES_CCM_16) { 365 CRYPTDEB("CBC-MAC without CCM"); 366 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 367 return (EINVAL); 368 } 369 if (sop->keylen != sop->mackeylen) { 370 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 371 return (EINVAL); 372 } 373 mac = 0; 374 break; 375 } 376 #endif 377 378 memset(&csp, 0, sizeof(csp)); 379 if (use_outputbuffers) 380 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 381 if (mac != 0) { 382 csp.csp_auth_alg = mac; 383 csp.csp_auth_klen = sop->mackeylen; 384 } 385 if (sop->cipher != 0) { 386 csp.csp_cipher_alg = sop->cipher; 387 csp.csp_cipher_klen = sop->keylen; 388 } 389 thash = crypto_auth_hash(&csp); 390 txform = crypto_cipher(&csp); 391 392 if (txform != NULL && txform->macsize != 0) { 393 if (mac != 0) { 394 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 395 return (EINVAL); 396 } 397 csp.csp_mode = CSP_MODE_AEAD; 398 } else if (txform != NULL && thash != NULL) { 399 csp.csp_mode = CSP_MODE_ETA; 400 } else if (txform != NULL) { 401 csp.csp_mode = CSP_MODE_CIPHER; 402 } else if (thash != NULL) { 403 csp.csp_mode = CSP_MODE_DIGEST; 404 } else { 405 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 406 return (EINVAL); 407 } 408 409 switch (csp.csp_mode) { 410 case CSP_MODE_AEAD: 411 case CSP_MODE_ETA: 412 if (use_separate_aad) 413 csp.csp_flags |= CSP_F_SEPARATE_AAD; 414 break; 415 } 416 417 if (txform != NULL) { 418 if (sop->keylen > txform->maxkey || 419 sop->keylen < txform->minkey) { 420 CRYPTDEB("invalid cipher parameters"); 421 error = EINVAL; 422 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 423 goto bail; 424 } 425 426 key = malloc(csp.csp_cipher_klen, M_CRYPTODEV, M_WAITOK); 427 error = copyin(sop->key, key, csp.csp_cipher_klen); 428 if (error) { 429 CRYPTDEB("invalid key"); 430 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 431 goto bail; 432 } 433 csp.csp_cipher_key = key; 434 csp.csp_ivlen = txform->ivsize; 435 } 436 437 if (thash != NULL) { 438 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 439 CRYPTDEB("invalid mac key length"); 440 error = EINVAL; 441 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 442 goto bail; 443 } 444 445 if (csp.csp_auth_klen != 0) { 446 mackey = malloc(csp.csp_auth_klen, M_CRYPTODEV, 447 M_WAITOK); 448 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 449 if (error) { 450 CRYPTDEB("invalid mac key"); 451 SDT_PROBE1(opencrypto, dev, ioctl, error, 452 __LINE__); 453 goto bail; 454 } 455 csp.csp_auth_key = mackey; 456 } 457 458 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 459 csp.csp_ivlen = AES_GCM_IV_LEN; 460 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 461 csp.csp_ivlen = AES_CCM_IV_LEN; 462 } 463 464 if (sop->ivlen != 0) { 465 if (csp.csp_ivlen == 0) { 466 CRYPTDEB("does not support an IV"); 467 error = EINVAL; 468 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 469 goto bail; 470 } 471 csp.csp_ivlen = sop->ivlen; 472 } 473 if (sop->maclen != 0) { 474 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) { 475 CRYPTDEB("does not support a MAC"); 476 error = EINVAL; 477 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 478 goto bail; 479 } 480 csp.csp_auth_mlen = sop->maclen; 481 } 482 483 crid = sop->crid; 484 error = checkforsoftware(&crid); 485 if (error) { 486 CRYPTDEB("checkforsoftware"); 487 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 488 goto bail; 489 } 490 error = crypto_newsession(&cses, &csp, crid); 491 if (error) { 492 CRYPTDEB("crypto_newsession"); 493 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 494 goto bail; 495 } 496 497 cse = malloc(sizeof(struct csession), M_CRYPTODEV, M_WAITOK | M_ZERO); 498 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 499 refcount_init(&cse->refs, 1); 500 cse->key = key; 501 cse->mackey = mackey; 502 cse->cses = cses; 503 if (sop->maclen != 0) 504 cse->hashsize = sop->maclen; 505 else if (thash != NULL) 506 cse->hashsize = thash->hashsize; 507 else if (csp.csp_mode == CSP_MODE_AEAD) 508 cse->hashsize = txform->macsize; 509 cse->ivsize = csp.csp_ivlen; 510 511 /* 512 * NB: This isn't necessarily the block size of the underlying 513 * MAC or cipher but is instead a restriction on valid input 514 * sizes. 515 */ 516 if (txform != NULL) 517 cse->blocksize = txform->blocksize; 518 else 519 cse->blocksize = 1; 520 521 mtx_lock(&fcr->lock); 522 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 523 cse->ses = fcr->sesn++; 524 mtx_unlock(&fcr->lock); 525 526 sop->ses = cse->ses; 527 528 /* return hardware/driver id */ 529 sop->crid = crypto_ses2hid(cse->cses); 530 bail: 531 if (error) { 532 free(key, M_CRYPTODEV); 533 free(mackey, M_CRYPTODEV); 534 } 535 return (error); 536 } 537 538 static struct csession * 539 cse_find(struct fcrypt *fcr, u_int ses) 540 { 541 struct csession *cse; 542 543 mtx_lock(&fcr->lock); 544 TAILQ_FOREACH(cse, &fcr->csessions, next) { 545 if (cse->ses == ses) { 546 refcount_acquire(&cse->refs); 547 mtx_unlock(&fcr->lock); 548 return (cse); 549 } 550 } 551 mtx_unlock(&fcr->lock); 552 return (NULL); 553 } 554 555 static void 556 cse_free(struct csession *cse) 557 { 558 559 if (!refcount_release(&cse->refs)) 560 return; 561 crypto_freesession(cse->cses); 562 mtx_destroy(&cse->lock); 563 if (cse->key) 564 free(cse->key, M_CRYPTODEV); 565 if (cse->mackey) 566 free(cse->mackey, M_CRYPTODEV); 567 free(cse, M_CRYPTODEV); 568 } 569 570 static bool 571 cse_delete(struct fcrypt *fcr, u_int ses) 572 { 573 struct csession *cse; 574 575 mtx_lock(&fcr->lock); 576 TAILQ_FOREACH(cse, &fcr->csessions, next) { 577 if (cse->ses == ses) { 578 TAILQ_REMOVE(&fcr->csessions, cse, next); 579 mtx_unlock(&fcr->lock); 580 cse_free(cse); 581 return (true); 582 } 583 } 584 mtx_unlock(&fcr->lock); 585 return (false); 586 } 587 588 static struct cryptop_data * 589 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 590 { 591 struct cryptop_data *cod; 592 593 cod = malloc(sizeof(struct cryptop_data), M_CRYPTODEV, M_WAITOK | 594 M_ZERO); 595 596 cod->cse = cse; 597 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 598 if (aad_len != 0) 599 cod->aad = malloc(aad_len, M_CRYPTODEV, M_WAITOK); 600 cod->buf = malloc(len, M_CRYPTODEV, M_WAITOK); 601 } else 602 cod->buf = malloc(aad_len + len, M_CRYPTODEV, M_WAITOK); 603 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 604 cod->obuf = malloc(len, M_CRYPTODEV, M_WAITOK); 605 return (cod); 606 } 607 608 static void 609 cod_free(struct cryptop_data *cod) 610 { 611 612 free(cod->aad, M_CRYPTODEV); 613 free(cod->obuf, M_CRYPTODEV); 614 free(cod->buf, M_CRYPTODEV); 615 free(cod, M_CRYPTODEV); 616 } 617 618 static int 619 cryptodev_cb(struct cryptop *crp) 620 { 621 struct cryptop_data *cod = crp->crp_opaque; 622 623 /* 624 * Lock to ensure the wakeup() is not missed by the loops 625 * waiting on cod->done in cryptodev_op() and 626 * cryptodev_aead(). 627 */ 628 mtx_lock(&cod->cse->lock); 629 cod->done = true; 630 mtx_unlock(&cod->cse->lock); 631 wakeup(cod); 632 return (0); 633 } 634 635 static int 636 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 637 { 638 const struct crypto_session_params *csp; 639 struct cryptop_data *cod = NULL; 640 struct cryptop *crp = NULL; 641 char *dst; 642 int error; 643 644 if (cop->len > 256*1024-4) { 645 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 646 return (E2BIG); 647 } 648 649 if ((cop->len % cse->blocksize) != 0) { 650 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 651 return (EINVAL); 652 } 653 654 if (cop->mac && cse->hashsize == 0) { 655 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 656 return (EINVAL); 657 } 658 659 /* 660 * The COP_F_CIPHER_FIRST flag predates explicit session 661 * modes, but the only way it was used was for EtA so allow it 662 * as long as it is consistent with EtA. 663 */ 664 if (cop->flags & COP_F_CIPHER_FIRST) { 665 if (cop->op != COP_ENCRYPT) { 666 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 667 return (EINVAL); 668 } 669 } 670 671 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 672 dst = cop->dst; 673 674 crp = crypto_getreq(cse->cses, M_WAITOK); 675 676 error = copyin(cop->src, cod->buf, cop->len); 677 if (error) { 678 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 679 goto bail; 680 } 681 crp->crp_payload_start = 0; 682 crp->crp_payload_length = cop->len; 683 if (cse->hashsize) 684 crp->crp_digest_start = cop->len; 685 686 csp = crypto_get_params(cse->cses); 687 switch (csp->csp_mode) { 688 case CSP_MODE_COMPRESS: 689 switch (cop->op) { 690 case COP_ENCRYPT: 691 crp->crp_op = CRYPTO_OP_COMPRESS; 692 break; 693 case COP_DECRYPT: 694 crp->crp_op = CRYPTO_OP_DECOMPRESS; 695 break; 696 default: 697 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 698 error = EINVAL; 699 goto bail; 700 } 701 break; 702 case CSP_MODE_CIPHER: 703 if (cop->len == 0 || 704 (cop->iv == NULL && cop->len == cse->ivsize)) { 705 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 706 error = EINVAL; 707 goto bail; 708 } 709 switch (cop->op) { 710 case COP_ENCRYPT: 711 crp->crp_op = CRYPTO_OP_ENCRYPT; 712 break; 713 case COP_DECRYPT: 714 crp->crp_op = CRYPTO_OP_DECRYPT; 715 break; 716 default: 717 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 718 error = EINVAL; 719 goto bail; 720 } 721 break; 722 case CSP_MODE_DIGEST: 723 switch (cop->op) { 724 case 0: 725 case COP_ENCRYPT: 726 case COP_DECRYPT: 727 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 728 if (cod->obuf != NULL) 729 crp->crp_digest_start = 0; 730 break; 731 default: 732 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 733 error = EINVAL; 734 goto bail; 735 } 736 break; 737 case CSP_MODE_AEAD: 738 if (cse->ivsize != 0 && cop->iv == NULL) { 739 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 740 error = EINVAL; 741 goto bail; 742 } 743 /* FALLTHROUGH */ 744 case CSP_MODE_ETA: 745 switch (cop->op) { 746 case COP_ENCRYPT: 747 crp->crp_op = CRYPTO_OP_ENCRYPT | 748 CRYPTO_OP_COMPUTE_DIGEST; 749 break; 750 case COP_DECRYPT: 751 crp->crp_op = CRYPTO_OP_DECRYPT | 752 CRYPTO_OP_VERIFY_DIGEST; 753 break; 754 default: 755 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 756 error = EINVAL; 757 goto bail; 758 } 759 break; 760 default: 761 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 762 error = EINVAL; 763 goto bail; 764 } 765 766 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 767 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 768 if (cod->obuf) 769 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 770 crp->crp_callback = cryptodev_cb; 771 crp->crp_opaque = cod; 772 773 if (cop->iv) { 774 if (cse->ivsize == 0) { 775 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 776 error = EINVAL; 777 goto bail; 778 } 779 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 780 if (error) { 781 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 782 goto bail; 783 } 784 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 785 } else if (cse->ivsize != 0) { 786 if (crp->crp_payload_length < cse->ivsize) { 787 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 788 error = EINVAL; 789 goto bail; 790 } 791 crp->crp_iv_start = 0; 792 crp->crp_payload_length -= cse->ivsize; 793 if (crp->crp_payload_length != 0) 794 crp->crp_payload_start = cse->ivsize; 795 dst += cse->ivsize; 796 } 797 798 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 799 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 800 cse->hashsize); 801 if (error) { 802 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 803 goto bail; 804 } 805 } 806 again: 807 /* 808 * Let the dispatch run unlocked, then, interlock against the 809 * callback before checking if the operation completed and going 810 * to sleep. This insures drivers don't inherit our lock which 811 * results in a lock order reversal between crypto_dispatch forced 812 * entry and the crypto_done callback into us. 813 */ 814 error = crypto_dispatch(crp); 815 if (error != 0) { 816 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 817 goto bail; 818 } 819 820 mtx_lock(&cse->lock); 821 while (!cod->done) 822 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 823 mtx_unlock(&cse->lock); 824 825 if (crp->crp_etype == EAGAIN) { 826 crp->crp_etype = 0; 827 crp->crp_flags &= ~CRYPTO_F_DONE; 828 cod->done = false; 829 goto again; 830 } 831 832 if (crp->crp_etype != 0) { 833 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 834 error = crp->crp_etype; 835 goto bail; 836 } 837 838 if (cop->dst != NULL) { 839 error = copyout(cod->obuf != NULL ? cod->obuf : 840 cod->buf + crp->crp_payload_start, dst, 841 crp->crp_payload_length); 842 if (error) { 843 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 844 goto bail; 845 } 846 } 847 848 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 849 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 850 crp->crp_digest_start, cop->mac, cse->hashsize); 851 if (error) { 852 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 853 goto bail; 854 } 855 } 856 857 bail: 858 crypto_freereq(crp); 859 cod_free(cod); 860 861 return (error); 862 } 863 864 static int 865 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 866 { 867 const struct crypto_session_params *csp; 868 struct cryptop_data *cod = NULL; 869 struct cryptop *crp = NULL; 870 char *dst; 871 int error; 872 873 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 874 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 875 return (E2BIG); 876 } 877 878 if ((caead->len % cse->blocksize) != 0) { 879 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 880 return (EINVAL); 881 } 882 883 if (cse->hashsize == 0 || caead->tag == NULL) { 884 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 885 return (EINVAL); 886 } 887 888 /* 889 * The COP_F_CIPHER_FIRST flag predates explicit session 890 * modes, but the only way it was used was for EtA so allow it 891 * as long as it is consistent with EtA. 892 */ 893 if (caead->flags & COP_F_CIPHER_FIRST) { 894 if (caead->op != COP_ENCRYPT) { 895 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 896 return (EINVAL); 897 } 898 } 899 900 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 901 dst = caead->dst; 902 903 crp = crypto_getreq(cse->cses, M_WAITOK); 904 905 if (cod->aad != NULL) 906 error = copyin(caead->aad, cod->aad, caead->aadlen); 907 else 908 error = copyin(caead->aad, cod->buf, caead->aadlen); 909 if (error) { 910 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 911 goto bail; 912 } 913 crp->crp_aad = cod->aad; 914 crp->crp_aad_start = 0; 915 crp->crp_aad_length = caead->aadlen; 916 917 if (cod->aad != NULL) 918 crp->crp_payload_start = 0; 919 else 920 crp->crp_payload_start = caead->aadlen; 921 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 922 caead->len); 923 if (error) { 924 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 925 goto bail; 926 } 927 crp->crp_payload_length = caead->len; 928 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 929 crp->crp_digest_start = crp->crp_payload_output_start + 930 caead->len; 931 else 932 crp->crp_digest_start = crp->crp_payload_start + caead->len; 933 934 csp = crypto_get_params(cse->cses); 935 switch (csp->csp_mode) { 936 case CSP_MODE_AEAD: 937 case CSP_MODE_ETA: 938 switch (caead->op) { 939 case COP_ENCRYPT: 940 crp->crp_op = CRYPTO_OP_ENCRYPT | 941 CRYPTO_OP_COMPUTE_DIGEST; 942 break; 943 case COP_DECRYPT: 944 crp->crp_op = CRYPTO_OP_DECRYPT | 945 CRYPTO_OP_VERIFY_DIGEST; 946 break; 947 default: 948 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 949 error = EINVAL; 950 goto bail; 951 } 952 break; 953 default: 954 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 955 error = EINVAL; 956 goto bail; 957 } 958 959 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 960 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 961 cse->hashsize); 962 if (cod->obuf != NULL) 963 crypto_use_output_buf(crp, cod->obuf, caead->len + 964 cse->hashsize); 965 crp->crp_callback = cryptodev_cb; 966 crp->crp_opaque = cod; 967 968 if (caead->iv) { 969 /* 970 * Permit a 16-byte IV for AES-XTS, but only use the 971 * first 8 bytes as a block number. 972 */ 973 if (csp->csp_mode == CSP_MODE_ETA && 974 csp->csp_cipher_alg == CRYPTO_AES_XTS && 975 caead->ivlen == AES_BLOCK_LEN) 976 caead->ivlen = AES_XTS_IV_LEN; 977 978 if (cse->ivsize == 0) { 979 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 980 error = EINVAL; 981 goto bail; 982 } 983 if (caead->ivlen != cse->ivsize) { 984 error = EINVAL; 985 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 986 goto bail; 987 } 988 989 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 990 if (error) { 991 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 992 goto bail; 993 } 994 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 995 } else { 996 error = EINVAL; 997 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 998 goto bail; 999 } 1000 1001 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1002 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1003 cse->hashsize); 1004 if (error) { 1005 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1006 goto bail; 1007 } 1008 } 1009 again: 1010 /* 1011 * Let the dispatch run unlocked, then, interlock against the 1012 * callback before checking if the operation completed and going 1013 * to sleep. This insures drivers don't inherit our lock which 1014 * results in a lock order reversal between crypto_dispatch forced 1015 * entry and the crypto_done callback into us. 1016 */ 1017 error = crypto_dispatch(crp); 1018 if (error != 0) { 1019 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1020 goto bail; 1021 } 1022 1023 mtx_lock(&cse->lock); 1024 while (!cod->done) 1025 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1026 mtx_unlock(&cse->lock); 1027 1028 if (crp->crp_etype == EAGAIN) { 1029 crp->crp_etype = 0; 1030 crp->crp_flags &= ~CRYPTO_F_DONE; 1031 cod->done = false; 1032 goto again; 1033 } 1034 1035 if (crp->crp_etype != 0) { 1036 error = crp->crp_etype; 1037 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1038 goto bail; 1039 } 1040 1041 if (caead->dst != NULL) { 1042 error = copyout(cod->obuf != NULL ? cod->obuf : 1043 cod->buf + crp->crp_payload_start, dst, 1044 crp->crp_payload_length); 1045 if (error) { 1046 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1047 goto bail; 1048 } 1049 } 1050 1051 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1052 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1053 crp->crp_digest_start, caead->tag, cse->hashsize); 1054 if (error) { 1055 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1056 goto bail; 1057 } 1058 } 1059 1060 bail: 1061 crypto_freereq(crp); 1062 cod_free(cod); 1063 1064 return (error); 1065 } 1066 1067 static int 1068 cryptodev_find(struct crypt_find_op *find) 1069 { 1070 device_t dev; 1071 size_t fnlen = sizeof find->name; 1072 1073 if (find->crid != -1) { 1074 dev = crypto_find_device_byhid(find->crid); 1075 if (dev == NULL) 1076 return (ENOENT); 1077 strncpy(find->name, device_get_nameunit(dev), fnlen); 1078 find->name[fnlen - 1] = '\x0'; 1079 } else { 1080 find->name[fnlen - 1] = '\x0'; 1081 find->crid = crypto_find_driver(find->name); 1082 if (find->crid == -1) 1083 return (ENOENT); 1084 } 1085 return (0); 1086 } 1087 1088 static void 1089 fcrypt_dtor(void *data) 1090 { 1091 struct fcrypt *fcr = data; 1092 struct csession *cse; 1093 1094 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1095 TAILQ_REMOVE(&fcr->csessions, cse, next); 1096 KASSERT(refcount_load(&cse->refs) == 1, 1097 ("%s: crypto session %p with %d refs", __func__, cse, 1098 refcount_load(&cse->refs))); 1099 cse_free(cse); 1100 } 1101 mtx_destroy(&fcr->lock); 1102 free(fcr, M_CRYPTODEV); 1103 } 1104 1105 static int 1106 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1107 { 1108 struct fcrypt *fcr; 1109 int error; 1110 1111 fcr = malloc(sizeof(struct fcrypt), M_CRYPTODEV, M_WAITOK | M_ZERO); 1112 TAILQ_INIT(&fcr->csessions); 1113 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1114 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1115 if (error) 1116 fcrypt_dtor(fcr); 1117 return (error); 1118 } 1119 1120 static int 1121 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1122 struct thread *td) 1123 { 1124 struct fcrypt *fcr; 1125 struct csession *cse; 1126 struct session2_op *sop; 1127 struct crypt_op *cop; 1128 struct crypt_aead *caead; 1129 uint32_t ses; 1130 int error = 0; 1131 union { 1132 struct session2_op sopc; 1133 #ifdef COMPAT_FREEBSD32 1134 struct crypt_op copc; 1135 struct crypt_aead aeadc; 1136 #endif 1137 } thunk; 1138 #ifdef COMPAT_FREEBSD32 1139 u_long cmd32; 1140 void *data32; 1141 1142 cmd32 = 0; 1143 data32 = NULL; 1144 switch (cmd) { 1145 case CIOCGSESSION32: 1146 cmd32 = cmd; 1147 data32 = data; 1148 cmd = CIOCGSESSION; 1149 data = (void *)&thunk.sopc; 1150 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1151 break; 1152 case CIOCGSESSION232: 1153 cmd32 = cmd; 1154 data32 = data; 1155 cmd = CIOCGSESSION2; 1156 data = (void *)&thunk.sopc; 1157 session2_op_from_32((struct session2_op32 *)data32, 1158 &thunk.sopc); 1159 break; 1160 case CIOCCRYPT32: 1161 cmd32 = cmd; 1162 data32 = data; 1163 cmd = CIOCCRYPT; 1164 data = (void *)&thunk.copc; 1165 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1166 break; 1167 case CIOCCRYPTAEAD32: 1168 cmd32 = cmd; 1169 data32 = data; 1170 cmd = CIOCCRYPTAEAD; 1171 data = (void *)&thunk.aeadc; 1172 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1173 break; 1174 } 1175 #endif 1176 1177 devfs_get_cdevpriv((void **)&fcr); 1178 1179 switch (cmd) { 1180 #ifdef COMPAT_FREEBSD12 1181 case CRIOGET: 1182 /* 1183 * NB: This may fail in cases that the old 1184 * implementation did not if the current process has 1185 * restricted filesystem access (e.g. running in a 1186 * jail that does not expose /dev/crypto or in 1187 * capability mode). 1188 */ 1189 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1190 O_RDWR, 0); 1191 if (error == 0) 1192 *(uint32_t *)data = td->td_retval[0]; 1193 break; 1194 #endif 1195 case CIOCGSESSION: 1196 case CIOCGSESSION2: 1197 if (cmd == CIOCGSESSION) { 1198 session2_op_from_op((void *)data, &thunk.sopc); 1199 sop = &thunk.sopc; 1200 } else 1201 sop = (struct session2_op *)data; 1202 1203 error = cse_create(fcr, sop); 1204 if (cmd == CIOCGSESSION && error == 0) 1205 session2_op_to_op(sop, (void *)data); 1206 break; 1207 case CIOCFSESSION: 1208 ses = *(uint32_t *)data; 1209 if (!cse_delete(fcr, ses)) { 1210 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1211 return (EINVAL); 1212 } 1213 break; 1214 case CIOCCRYPT: 1215 cop = (struct crypt_op *)data; 1216 cse = cse_find(fcr, cop->ses); 1217 if (cse == NULL) { 1218 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1219 return (EINVAL); 1220 } 1221 error = cryptodev_op(cse, cop); 1222 cse_free(cse); 1223 break; 1224 case CIOCFINDDEV: 1225 error = cryptodev_find((struct crypt_find_op *)data); 1226 break; 1227 case CIOCCRYPTAEAD: 1228 caead = (struct crypt_aead *)data; 1229 cse = cse_find(fcr, caead->ses); 1230 if (cse == NULL) { 1231 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1232 return (EINVAL); 1233 } 1234 error = cryptodev_aead(cse, caead); 1235 cse_free(cse); 1236 break; 1237 default: 1238 error = EINVAL; 1239 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1240 break; 1241 } 1242 1243 #ifdef COMPAT_FREEBSD32 1244 switch (cmd32) { 1245 case CIOCGSESSION32: 1246 if (error == 0) 1247 session_op_to_32((void *)data, data32); 1248 break; 1249 case CIOCGSESSION232: 1250 if (error == 0) 1251 session2_op_to_32((void *)data, data32); 1252 break; 1253 case CIOCCRYPT32: 1254 if (error == 0) 1255 crypt_op_to_32((void *)data, data32); 1256 break; 1257 case CIOCCRYPTAEAD32: 1258 if (error == 0) 1259 crypt_aead_to_32((void *)data, data32); 1260 break; 1261 } 1262 #endif 1263 return (error); 1264 } 1265 1266 static struct cdevsw crypto_cdevsw = { 1267 .d_version = D_VERSION, 1268 .d_open = crypto_open, 1269 .d_ioctl = crypto_ioctl, 1270 .d_name = "crypto", 1271 }; 1272 static struct cdev *crypto_dev; 1273 1274 /* 1275 * Initialization code, both for static and dynamic loading. 1276 */ 1277 static int 1278 cryptodev_modevent(module_t mod, int type, void *unused) 1279 { 1280 switch (type) { 1281 case MOD_LOAD: 1282 if (bootverbose) 1283 printf("crypto: <crypto device>\n"); 1284 crypto_dev = make_dev(&crypto_cdevsw, 0, 1285 UID_ROOT, GID_WHEEL, 0666, 1286 "crypto"); 1287 return 0; 1288 case MOD_UNLOAD: 1289 /*XXX disallow if active sessions */ 1290 destroy_dev(crypto_dev); 1291 return 0; 1292 } 1293 return EINVAL; 1294 } 1295 1296 static moduledata_t cryptodev_mod = { 1297 "cryptodev", 1298 cryptodev_modevent, 1299 0 1300 }; 1301 MODULE_VERSION(cryptodev, 1); 1302 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1303 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1304 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1305