1 /* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 2 /*- 3 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 4 * 5 * This code was written by Angelos D. Keromytis in Athens, Greece, in 6 * February 2000. Network Security Technologies Inc. (NSTI) kindly 7 * supported the development of this code. 8 * 9 * Copyright (c) 2000, 2001 Angelos D. Keromytis 10 * 11 * Permission to use, copy, and modify this software with or without fee 12 * is hereby granted, provided that this entire notice is included in 13 * all source code copies of any software which is or includes a copy or 14 * modification of this software. 15 * 16 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 17 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 18 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 19 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 20 * PURPOSE. 21 */ 22 23 #include <sys/cdefs.h> 24 __FBSDID("$FreeBSD$"); 25 26 #define CRYPTO_TIMING /* enable timing support */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/eventhandler.h> 31 #include <sys/kernel.h> 32 #include <sys/kthread.h> 33 #include <sys/lock.h> 34 #include <sys/module.h> 35 #include <sys/mutex.h> 36 #include <sys/malloc.h> 37 #include <sys/proc.h> 38 #include <sys/sysctl.h> 39 40 #include <vm/uma.h> 41 #include <opencrypto/cryptodev.h> 42 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 43 44 /* 45 * Crypto drivers register themselves by allocating a slot in the 46 * crypto_drivers table with crypto_get_driverid() and then registering 47 * each algorithm they support with crypto_register() and crypto_kregister(). 48 */ 49 static struct mtx crypto_drivers_mtx; /* lock on driver table */ 50 #define CRYPTO_DRIVER_LOCK() mtx_lock(&crypto_drivers_mtx) 51 #define CRYPTO_DRIVER_UNLOCK() mtx_unlock(&crypto_drivers_mtx) 52 static struct cryptocap *crypto_drivers = NULL; 53 static int crypto_drivers_num = 0; 54 55 /* 56 * There are two queues for crypto requests; one for symmetric (e.g. 57 * cipher) operations and one for asymmetric (e.g. MOD)operations. 58 * A single mutex is used to lock access to both queues. We could 59 * have one per-queue but having one simplifies handling of block/unblock 60 * operations. 61 */ 62 static int crp_sleep = 0; 63 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 64 static TAILQ_HEAD(,cryptkop) crp_kq; 65 static struct mtx crypto_q_mtx; 66 #define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx) 67 #define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx) 68 69 /* 70 * There are two queues for processing completed crypto requests; one 71 * for the symmetric and one for the asymmetric ops. We only need one 72 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 73 * mutex is used to lock access to both queues. Note that this lock 74 * must be separate from the lock on request queues to insure driver 75 * callbacks don't generate lock order reversals. 76 */ 77 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 78 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 79 static struct mtx crypto_ret_q_mtx; 80 #define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx) 81 #define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx) 82 #define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) 83 84 static uma_zone_t cryptop_zone; 85 static uma_zone_t cryptodesc_zone; 86 87 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 88 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 89 &crypto_userasymcrypto, 0, 90 "Enable/disable user-mode access to asymmetric crypto support"); 91 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 92 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 93 &crypto_devallowsoft, 0, 94 "Enable/disable use of software asym crypto support"); 95 96 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 97 98 static void crypto_proc(void); 99 static struct proc *cryptoproc; 100 static void crypto_ret_proc(void); 101 static struct proc *cryptoretproc; 102 static void crypto_destroy(void); 103 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 104 static int crypto_kinvoke(struct cryptkop *krp); 105 106 static struct cryptostats cryptostats; 107 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 108 cryptostats, "Crypto system statistics"); 109 110 #ifdef CRYPTO_TIMING 111 static int crypto_timing = 0; 112 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 113 &crypto_timing, 0, "Enable/disable crypto timing support"); 114 #endif 115 116 static int 117 crypto_init(void) 118 { 119 int error; 120 121 mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table", 122 MTX_DEF|MTX_QUIET); 123 124 TAILQ_INIT(&crp_q); 125 TAILQ_INIT(&crp_kq); 126 mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF); 127 128 TAILQ_INIT(&crp_ret_q); 129 TAILQ_INIT(&crp_ret_kq); 130 mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF); 131 132 cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop), 133 0, 0, 0, 0, 134 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 135 cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc), 136 0, 0, 0, 0, 137 UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 138 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 139 printf("crypto_init: cannot setup crypto zones\n"); 140 error = ENOMEM; 141 goto bad; 142 } 143 144 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 145 crypto_drivers = malloc(crypto_drivers_num * 146 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 147 if (crypto_drivers == NULL) { 148 printf("crypto_init: cannot setup crypto drivers\n"); 149 error = ENOMEM; 150 goto bad; 151 } 152 153 error = kthread_create((void (*)(void *)) crypto_proc, NULL, 154 &cryptoproc, 0, 0, "crypto"); 155 if (error) { 156 printf("crypto_init: cannot start crypto thread; error %d", 157 error); 158 goto bad; 159 } 160 161 error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL, 162 &cryptoretproc, 0, 0, "crypto returns"); 163 if (error) { 164 printf("crypto_init: cannot start cryptoret thread; error %d", 165 error); 166 goto bad; 167 } 168 return 0; 169 bad: 170 crypto_destroy(); 171 return error; 172 } 173 174 /* 175 * Signal a crypto thread to terminate. We use the driver 176 * table lock to synchronize the sleep/wakeups so that we 177 * are sure the threads have terminated before we release 178 * the data structures they use. See crypto_finis below 179 * for the other half of this song-and-dance. 180 */ 181 static void 182 crypto_terminate(struct proc **pp, void *q) 183 { 184 struct proc *p; 185 186 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 187 p = *pp; 188 *pp = NULL; 189 if (p) { 190 wakeup_one(q); 191 PROC_LOCK(p); /* NB: insure we don't miss wakeup */ 192 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 193 msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0); 194 PROC_UNLOCK(p); 195 CRYPTO_DRIVER_LOCK(); 196 } 197 } 198 199 static void 200 crypto_destroy(void) 201 { 202 /* 203 * Terminate any crypto threads. 204 */ 205 CRYPTO_DRIVER_LOCK(); 206 crypto_terminate(&cryptoproc, &crp_q); 207 crypto_terminate(&cryptoretproc, &crp_ret_q); 208 CRYPTO_DRIVER_UNLOCK(); 209 210 /* XXX flush queues??? */ 211 212 /* 213 * Reclaim dynamically allocated resources. 214 */ 215 if (crypto_drivers != NULL) 216 free(crypto_drivers, M_CRYPTO_DATA); 217 218 if (cryptodesc_zone != NULL) 219 uma_zdestroy(cryptodesc_zone); 220 if (cryptop_zone != NULL) 221 uma_zdestroy(cryptop_zone); 222 mtx_destroy(&crypto_q_mtx); 223 mtx_destroy(&crypto_ret_q_mtx); 224 mtx_destroy(&crypto_drivers_mtx); 225 } 226 227 /* 228 * Initialization code, both for static and dynamic loading. 229 */ 230 static int 231 crypto_modevent(module_t mod, int type, void *unused) 232 { 233 int error = EINVAL; 234 235 switch (type) { 236 case MOD_LOAD: 237 error = crypto_init(); 238 if (error == 0 && bootverbose) 239 printf("crypto: <crypto core>\n"); 240 break; 241 case MOD_UNLOAD: 242 /*XXX disallow if active sessions */ 243 error = 0; 244 crypto_destroy(); 245 return 0; 246 } 247 return error; 248 } 249 250 static moduledata_t crypto_mod = { 251 "crypto", 252 crypto_modevent, 253 0 254 }; 255 MODULE_VERSION(crypto, 1); 256 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 257 MODULE_DEPEND(crypto, zlib, 1, 1, 1); 258 259 /* 260 * Create a new session. 261 */ 262 int 263 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 264 { 265 struct cryptocap *cap = NULL; 266 struct cryptoini *cr; 267 u_int32_t hid = 0, lid; 268 int err = EINVAL; 269 270 CRYPTO_DRIVER_LOCK(); 271 272 if (crypto_drivers == NULL) 273 goto done; 274 275 /* 276 * The algorithm we use here is pretty stupid; just use the 277 * first driver that supports all the algorithms we need. 278 * 279 * XXX We need more smarts here (in real life too, but that's 280 * XXX another story altogether). 281 */ 282 283 /* 284 * First try to find hardware crypto. 285 */ 286 if (hard >= 0) { 287 for (hid = 0; hid < crypto_drivers_num; hid++) { 288 cap = &crypto_drivers[hid]; 289 /* 290 * If it's not initialized or has remaining sessions 291 * referencing it, skip. 292 */ 293 if (cap->cc_newsession == NULL || 294 (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) 295 continue; 296 297 /* Hardware required -- ignore software drivers. */ 298 if (cap->cc_flags & CRYPTOCAP_F_SOFTWARE) 299 continue; 300 301 /* See if all the algorithms are supported. */ 302 for (cr = cri; cr; cr = cr->cri_next) 303 if (cap->cc_alg[cr->cri_alg] == 0) 304 break; 305 if (cr == NULL) { 306 /* Ok, all algorithms are supported. */ 307 break; 308 } 309 } 310 if (hid == crypto_drivers_num) 311 cap = NULL; 312 } 313 /* 314 * If no hardware crypto, look for software crypto. 315 */ 316 if (cap == NULL && hard <= 0) { 317 for (hid = 0; hid < crypto_drivers_num; hid++) { 318 cap = &crypto_drivers[hid]; 319 /* 320 * If it's not initialized or has remaining sessions 321 * referencing it, skip. 322 */ 323 if (cap->cc_newsession == NULL || 324 (cap->cc_flags & CRYPTOCAP_F_CLEANUP)) 325 continue; 326 327 /* Software required -- ignore hardware drivers. */ 328 if (!(cap->cc_flags & CRYPTOCAP_F_SOFTWARE)) 329 continue; 330 331 /* See if all the algorithms are supported. */ 332 for (cr = cri; cr; cr = cr->cri_next) 333 if (cap->cc_alg[cr->cri_alg] == 0) 334 break; 335 if (cr == NULL) { 336 /* Ok, all algorithms are supported. */ 337 break; 338 } 339 } 340 if (hid == crypto_drivers_num) 341 cap = NULL; 342 } 343 344 if (cap != NULL) { 345 /* 346 * Can't do everything in one session. 347 * 348 * XXX Fix this. We need to inject a "virtual" session layer right 349 * XXX about here. 350 */ 351 352 /* Call the driver initialization routine. */ 353 lid = hid; /* Pass the driver ID. */ 354 err = (*cap->cc_newsession)(cap->cc_arg, &lid, cri); 355 if (err == 0) { 356 /* XXX assert (hid &~ 0xffffff) == 0 */ 357 /* XXX assert (cap->cc_flags &~ 0xff) == 0 */ 358 (*sid) = ((cap->cc_flags & 0xff) << 24) | hid; 359 (*sid) <<= 32; 360 (*sid) |= (lid & 0xffffffff); 361 cap->cc_sessions++; 362 } 363 } 364 done: 365 CRYPTO_DRIVER_UNLOCK(); 366 return err; 367 } 368 369 static void 370 crypto_remove(struct cryptocap *cap) 371 { 372 373 mtx_assert(&crypto_drivers_mtx, MA_OWNED); 374 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 375 bzero(cap, sizeof(*cap)); 376 } 377 378 /* 379 * Delete an existing session (or a reserved session on an unregistered 380 * driver). 381 */ 382 int 383 crypto_freesession(u_int64_t sid) 384 { 385 struct cryptocap *cap; 386 u_int32_t hid; 387 int err; 388 389 CRYPTO_DRIVER_LOCK(); 390 391 if (crypto_drivers == NULL) { 392 err = EINVAL; 393 goto done; 394 } 395 396 /* Determine two IDs. */ 397 hid = CRYPTO_SESID2HID(sid); 398 399 if (hid >= crypto_drivers_num) { 400 err = ENOENT; 401 goto done; 402 } 403 cap = &crypto_drivers[hid]; 404 405 if (cap->cc_sessions) 406 cap->cc_sessions--; 407 408 /* Call the driver cleanup routine, if available. */ 409 if (cap->cc_freesession) 410 err = cap->cc_freesession(cap->cc_arg, sid); 411 else 412 err = 0; 413 414 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 415 crypto_remove(cap); 416 417 done: 418 CRYPTO_DRIVER_UNLOCK(); 419 return err; 420 } 421 422 /* 423 * Return an unused driver id. Used by drivers prior to registering 424 * support for the algorithms they handle. 425 */ 426 int32_t 427 crypto_get_driverid(u_int32_t flags) 428 { 429 struct cryptocap *newdrv; 430 int i; 431 432 CRYPTO_DRIVER_LOCK(); 433 434 for (i = 0; i < crypto_drivers_num; i++) { 435 if (crypto_drivers[i].cc_process == NULL && 436 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 437 break; 438 } 439 } 440 441 /* Out of entries, allocate some more. */ 442 if (i == crypto_drivers_num) { 443 /* Be careful about wrap-around. */ 444 if (2 * crypto_drivers_num <= crypto_drivers_num) { 445 CRYPTO_DRIVER_UNLOCK(); 446 printf("crypto: driver count wraparound!\n"); 447 return -1; 448 } 449 450 newdrv = malloc(2 * crypto_drivers_num * 451 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 452 if (newdrv == NULL) { 453 CRYPTO_DRIVER_UNLOCK(); 454 printf("crypto: no space to expand driver table!\n"); 455 return -1; 456 } 457 458 bcopy(crypto_drivers, newdrv, 459 crypto_drivers_num * sizeof(struct cryptocap)); 460 461 crypto_drivers_num *= 2; 462 463 free(crypto_drivers, M_CRYPTO_DATA); 464 crypto_drivers = newdrv; 465 } 466 467 /* NB: state is zero'd on free */ 468 crypto_drivers[i].cc_sessions = 1; /* Mark */ 469 crypto_drivers[i].cc_flags = flags; 470 if (bootverbose) 471 printf("crypto: assign driver %u, flags %u\n", i, flags); 472 473 CRYPTO_DRIVER_UNLOCK(); 474 475 return i; 476 } 477 478 static struct cryptocap * 479 crypto_checkdriver(u_int32_t hid) 480 { 481 if (crypto_drivers == NULL) 482 return NULL; 483 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 484 } 485 486 /* 487 * Register support for a key-related algorithm. This routine 488 * is called once for each algorithm supported a driver. 489 */ 490 int 491 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 492 int (*kprocess)(void*, struct cryptkop *, int), 493 void *karg) 494 { 495 struct cryptocap *cap; 496 int err; 497 498 CRYPTO_DRIVER_LOCK(); 499 500 cap = crypto_checkdriver(driverid); 501 if (cap != NULL && 502 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 503 /* 504 * XXX Do some performance testing to determine placing. 505 * XXX We probably need an auxiliary data structure that 506 * XXX describes relative performances. 507 */ 508 509 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 510 if (bootverbose) 511 printf("crypto: driver %u registers key alg %u flags %u\n" 512 , driverid 513 , kalg 514 , flags 515 ); 516 517 if (cap->cc_kprocess == NULL) { 518 cap->cc_karg = karg; 519 cap->cc_kprocess = kprocess; 520 } 521 err = 0; 522 } else 523 err = EINVAL; 524 525 CRYPTO_DRIVER_UNLOCK(); 526 return err; 527 } 528 529 /* 530 * Register support for a non-key-related algorithm. This routine 531 * is called once for each such algorithm supported by a driver. 532 */ 533 int 534 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 535 u_int32_t flags, 536 int (*newses)(void*, u_int32_t*, struct cryptoini*), 537 int (*freeses)(void*, u_int64_t), 538 int (*process)(void*, struct cryptop *, int), 539 void *arg) 540 { 541 struct cryptocap *cap; 542 int err; 543 544 CRYPTO_DRIVER_LOCK(); 545 546 cap = crypto_checkdriver(driverid); 547 /* NB: algorithms are in the range [1..max] */ 548 if (cap != NULL && 549 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 550 /* 551 * XXX Do some performance testing to determine placing. 552 * XXX We probably need an auxiliary data structure that 553 * XXX describes relative performances. 554 */ 555 556 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 557 cap->cc_max_op_len[alg] = maxoplen; 558 if (bootverbose) 559 printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 560 , driverid 561 , alg 562 , flags 563 , maxoplen 564 ); 565 566 if (cap->cc_process == NULL) { 567 cap->cc_arg = arg; 568 cap->cc_newsession = newses; 569 cap->cc_process = process; 570 cap->cc_freesession = freeses; 571 cap->cc_sessions = 0; /* Unmark */ 572 } 573 err = 0; 574 } else 575 err = EINVAL; 576 577 CRYPTO_DRIVER_UNLOCK(); 578 return err; 579 } 580 581 /* 582 * Unregister a crypto driver. If there are pending sessions using it, 583 * leave enough information around so that subsequent calls using those 584 * sessions will correctly detect the driver has been unregistered and 585 * reroute requests. 586 */ 587 int 588 crypto_unregister(u_int32_t driverid, int alg) 589 { 590 struct cryptocap *cap; 591 u_int32_t ses, kops; 592 int i, err; 593 594 CRYPTO_DRIVER_LOCK(); 595 596 cap = crypto_checkdriver(driverid); 597 if (cap != NULL && 598 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 599 cap->cc_alg[alg] != 0) { 600 cap->cc_alg[alg] = 0; 601 cap->cc_max_op_len[alg] = 0; 602 603 /* Was this the last algorithm ? */ 604 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 605 if (cap->cc_alg[i] != 0) 606 break; 607 608 if (i == CRYPTO_ALGORITHM_MAX + 1) { 609 ses = cap->cc_sessions; 610 kops = cap->cc_koperations; 611 bzero(cap, sizeof(*cap)); 612 if (ses != 0 || kops != 0) { 613 /* 614 * If there are pending sessions, just mark as invalid. 615 */ 616 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 617 cap->cc_sessions = ses; 618 cap->cc_koperations = kops; 619 } 620 } 621 err = 0; 622 } else 623 err = EINVAL; 624 625 CRYPTO_DRIVER_UNLOCK(); 626 return err; 627 } 628 629 /* 630 * Unregister all algorithms associated with a crypto driver. 631 * If there are pending sessions using it, leave enough information 632 * around so that subsequent calls using those sessions will 633 * correctly detect the driver has been unregistered and reroute 634 * requests. 635 */ 636 int 637 crypto_unregister_all(u_int32_t driverid) 638 { 639 struct cryptocap *cap; 640 u_int32_t ses, kops; 641 int i, err; 642 643 CRYPTO_DRIVER_LOCK(); 644 645 cap = crypto_checkdriver(driverid); 646 if (cap != NULL) { 647 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 648 cap->cc_alg[i] = 0; 649 cap->cc_max_op_len[i] = 0; 650 } 651 ses = cap->cc_sessions; 652 kops = cap->cc_koperations; 653 bzero(cap, sizeof(*cap)); 654 if (ses != 0 || kops != 0) { 655 /* 656 * If there are pending sessions, just mark as invalid. 657 */ 658 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 659 cap->cc_sessions = ses; 660 cap->cc_koperations = kops; 661 } 662 err = 0; 663 } else 664 err = EINVAL; 665 666 CRYPTO_DRIVER_UNLOCK(); 667 return err; 668 } 669 670 /* 671 * Clear blockage on a driver. The what parameter indicates whether 672 * the driver is now ready for cryptop's and/or cryptokop's. 673 */ 674 int 675 crypto_unblock(u_int32_t driverid, int what) 676 { 677 struct cryptocap *cap; 678 int err; 679 680 CRYPTO_Q_LOCK(); 681 cap = crypto_checkdriver(driverid); 682 if (cap != NULL) { 683 if (what & CRYPTO_SYMQ) 684 cap->cc_qblocked = 0; 685 if (what & CRYPTO_ASYMQ) 686 cap->cc_kqblocked = 0; 687 if (crp_sleep) 688 wakeup_one(&crp_q); 689 err = 0; 690 } else 691 err = EINVAL; 692 CRYPTO_Q_UNLOCK(); 693 694 return err; 695 } 696 697 /* 698 * Add a crypto request to a queue, to be processed by the kernel thread. 699 */ 700 int 701 crypto_dispatch(struct cryptop *crp) 702 { 703 struct cryptocap *cap; 704 u_int32_t hid; 705 int result; 706 707 cryptostats.cs_ops++; 708 709 #ifdef CRYPTO_TIMING 710 if (crypto_timing) 711 binuptime(&crp->crp_tstamp); 712 #endif 713 714 hid = CRYPTO_SESID2HID(crp->crp_sid); 715 716 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 717 /* 718 * Caller marked the request to be processed 719 * immediately; dispatch it directly to the 720 * driver unless the driver is currently blocked. 721 */ 722 cap = crypto_checkdriver(hid); 723 /* Driver cannot disappeared when there is an active session. */ 724 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 725 if (!cap->cc_qblocked) { 726 result = crypto_invoke(cap, crp, 0); 727 if (result != ERESTART) 728 return (result); 729 /* 730 * The driver ran out of resources, put the request on 731 * the queue. 732 */ 733 } 734 } 735 CRYPTO_Q_LOCK(); 736 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 737 if (crp_sleep) 738 wakeup_one(&crp_q); 739 CRYPTO_Q_UNLOCK(); 740 return 0; 741 } 742 743 /* 744 * Add an asymetric crypto request to a queue, 745 * to be processed by the kernel thread. 746 */ 747 int 748 crypto_kdispatch(struct cryptkop *krp) 749 { 750 int result; 751 752 cryptostats.cs_kops++; 753 754 result = crypto_kinvoke(krp); 755 if (result != ERESTART) 756 return (result); 757 CRYPTO_Q_LOCK(); 758 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 759 if (crp_sleep) 760 wakeup_one(&crp_q); 761 CRYPTO_Q_UNLOCK(); 762 763 return 0; 764 } 765 766 /* 767 * Dispatch an assymetric crypto request to the appropriate crypto devices. 768 */ 769 static int 770 crypto_kinvoke(struct cryptkop *krp) 771 { 772 struct cryptocap *cap = NULL; 773 u_int32_t hid; 774 int error = 0; 775 776 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 777 KASSERT(krp->krp_callback != NULL, 778 ("%s: krp->crp_callback == NULL", __func__)); 779 780 CRYPTO_DRIVER_LOCK(); 781 for (hid = 0; hid < crypto_drivers_num; hid++) { 782 cap = &crypto_drivers[hid]; 783 if (cap == NULL) 784 continue; 785 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 786 !crypto_devallowsoft) { 787 continue; 788 } 789 if (cap->cc_kprocess == NULL) 790 continue; 791 if (!(cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED)) 792 continue; 793 if (cap->cc_kqblocked) { 794 error = ERESTART; 795 continue; 796 } 797 error = 0; 798 break; 799 } 800 krp->krp_hid = hid; 801 if (hid < crypto_drivers_num) { 802 cap->cc_koperations++; 803 CRYPTO_DRIVER_UNLOCK(); 804 error = cap->cc_kprocess(cap->cc_karg, krp, 0); 805 CRYPTO_DRIVER_LOCK(); 806 if (error == ERESTART) { 807 cap->cc_koperations--; 808 CRYPTO_DRIVER_UNLOCK(); 809 return (error); 810 } 811 } else { 812 error = ENODEV; 813 } 814 CRYPTO_DRIVER_UNLOCK(); 815 816 if (error) { 817 krp->krp_status = error; 818 crypto_kdone(krp); 819 } 820 return 0; 821 } 822 823 #ifdef CRYPTO_TIMING 824 static void 825 crypto_tstat(struct cryptotstat *ts, struct bintime *bt) 826 { 827 struct bintime now, delta; 828 struct timespec t; 829 uint64_t u; 830 831 binuptime(&now); 832 u = now.frac; 833 delta.frac = now.frac - bt->frac; 834 delta.sec = now.sec - bt->sec; 835 if (u < delta.frac) 836 delta.sec--; 837 bintime2timespec(&delta, &t); 838 timespecadd(&ts->acc, &t); 839 if (timespeccmp(&t, &ts->min, <)) 840 ts->min = t; 841 if (timespeccmp(&t, &ts->max, >)) 842 ts->max = t; 843 ts->count++; 844 845 *bt = now; 846 } 847 #endif 848 849 /* 850 * Dispatch a crypto request to the appropriate crypto devices. 851 */ 852 static int 853 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 854 { 855 856 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 857 KASSERT(crp->crp_callback != NULL, 858 ("%s: crp->crp_callback == NULL", __func__)); 859 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 860 861 #ifdef CRYPTO_TIMING 862 if (crypto_timing) 863 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 864 #endif 865 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 866 struct cryptodesc *crd; 867 u_int64_t nid; 868 869 /* 870 * Driver has unregistered; migrate the session and return 871 * an error to the caller so they'll resubmit the op. 872 * 873 * XXX: What if there are more already queued requests for this 874 * session? 875 */ 876 crypto_freesession(crp->crp_sid); 877 878 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 879 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 880 881 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 882 crp->crp_sid = nid; 883 884 crp->crp_etype = EAGAIN; 885 crypto_done(crp); 886 return 0; 887 } else { 888 /* 889 * Invoke the driver to process the request. 890 */ 891 return cap->cc_process(cap->cc_arg, crp, hint); 892 } 893 } 894 895 /* 896 * Release a set of crypto descriptors. 897 */ 898 void 899 crypto_freereq(struct cryptop *crp) 900 { 901 struct cryptodesc *crd; 902 903 if (crp == NULL) 904 return; 905 906 #ifdef DIAGNOSTIC 907 { 908 struct cryptop *crp2; 909 910 CRYPTO_Q_LOCK(); 911 TAILQ_FOREACH(crp2, &crp_q, crp_next) { 912 KASSERT(crp2 != crp, 913 ("Freeing cryptop from the crypto queue (%p).", 914 crp)); 915 } 916 CRYPTO_Q_UNLOCK(); 917 CRYPTO_RETQ_LOCK(); 918 TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { 919 KASSERT(crp2 != crp, 920 ("Freeing cryptop from the return queue (%p).", 921 crp)); 922 } 923 CRYPTO_RETQ_UNLOCK(); 924 } 925 #endif 926 927 while ((crd = crp->crp_desc) != NULL) { 928 crp->crp_desc = crd->crd_next; 929 uma_zfree(cryptodesc_zone, crd); 930 } 931 932 uma_zfree(cryptop_zone, crp); 933 } 934 935 /* 936 * Acquire a set of crypto descriptors. 937 */ 938 struct cryptop * 939 crypto_getreq(int num) 940 { 941 struct cryptodesc *crd; 942 struct cryptop *crp; 943 944 crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO); 945 if (crp != NULL) { 946 while (num--) { 947 crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO); 948 if (crd == NULL) { 949 crypto_freereq(crp); 950 return NULL; 951 } 952 953 crd->crd_next = crp->crp_desc; 954 crp->crp_desc = crd; 955 } 956 } 957 return crp; 958 } 959 960 /* 961 * Invoke the callback on behalf of the driver. 962 */ 963 void 964 crypto_done(struct cryptop *crp) 965 { 966 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 967 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 968 crp->crp_flags |= CRYPTO_F_DONE; 969 if (crp->crp_etype != 0) 970 cryptostats.cs_errs++; 971 #ifdef CRYPTO_TIMING 972 if (crypto_timing) 973 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 974 #endif 975 /* 976 * CBIMM means unconditionally do the callback immediately; 977 * CBIFSYNC means do the callback immediately only if the 978 * operation was done synchronously. Both are used to avoid 979 * doing extraneous context switches; the latter is mostly 980 * used with the software crypto driver. 981 */ 982 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 983 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 984 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 985 /* 986 * Do the callback directly. This is ok when the 987 * callback routine does very little (e.g. the 988 * /dev/crypto callback method just does a wakeup). 989 */ 990 #ifdef CRYPTO_TIMING 991 if (crypto_timing) { 992 /* 993 * NB: We must copy the timestamp before 994 * doing the callback as the cryptop is 995 * likely to be reclaimed. 996 */ 997 struct bintime t = crp->crp_tstamp; 998 crypto_tstat(&cryptostats.cs_cb, &t); 999 crp->crp_callback(crp); 1000 crypto_tstat(&cryptostats.cs_finis, &t); 1001 } else 1002 #endif 1003 crp->crp_callback(crp); 1004 } else { 1005 /* 1006 * Normal case; queue the callback for the thread. 1007 */ 1008 CRYPTO_RETQ_LOCK(); 1009 if (CRYPTO_RETQ_EMPTY()) 1010 wakeup_one(&crp_ret_q); /* shared wait channel */ 1011 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1012 CRYPTO_RETQ_UNLOCK(); 1013 } 1014 } 1015 1016 /* 1017 * Invoke the callback on behalf of the driver. 1018 */ 1019 void 1020 crypto_kdone(struct cryptkop *krp) 1021 { 1022 struct cryptocap *cap; 1023 1024 if (krp->krp_status != 0) 1025 cryptostats.cs_kerrs++; 1026 CRYPTO_DRIVER_LOCK(); 1027 /* XXX: What if driver is loaded in the meantime? */ 1028 if (krp->krp_hid < crypto_drivers_num) { 1029 cap = &crypto_drivers[krp->krp_hid]; 1030 cap->cc_koperations--; 1031 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 1032 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1033 crypto_remove(cap); 1034 } 1035 CRYPTO_DRIVER_UNLOCK(); 1036 CRYPTO_RETQ_LOCK(); 1037 if (CRYPTO_RETQ_EMPTY()) 1038 wakeup_one(&crp_ret_q); /* shared wait channel */ 1039 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1040 CRYPTO_RETQ_UNLOCK(); 1041 } 1042 1043 int 1044 crypto_getfeat(int *featp) 1045 { 1046 int hid, kalg, feat = 0; 1047 1048 if (!crypto_userasymcrypto) 1049 goto out; 1050 1051 CRYPTO_DRIVER_LOCK(); 1052 for (hid = 0; hid < crypto_drivers_num; hid++) { 1053 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 1054 !crypto_devallowsoft) { 1055 continue; 1056 } 1057 if (crypto_drivers[hid].cc_kprocess == NULL) 1058 continue; 1059 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 1060 if ((crypto_drivers[hid].cc_kalg[kalg] & 1061 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 1062 feat |= 1 << kalg; 1063 } 1064 CRYPTO_DRIVER_UNLOCK(); 1065 out: 1066 *featp = feat; 1067 return (0); 1068 } 1069 1070 /* 1071 * Terminate a thread at module unload. The process that 1072 * initiated this is waiting for us to signal that we're gone; 1073 * wake it up and exit. We use the driver table lock to insure 1074 * we don't do the wakeup before they're waiting. There is no 1075 * race here because the waiter sleeps on the proc lock for the 1076 * thread so it gets notified at the right time because of an 1077 * extra wakeup that's done in exit1(). 1078 */ 1079 static void 1080 crypto_finis(void *chan) 1081 { 1082 CRYPTO_DRIVER_LOCK(); 1083 wakeup_one(chan); 1084 CRYPTO_DRIVER_UNLOCK(); 1085 kthread_exit(0); 1086 } 1087 1088 /* 1089 * Crypto thread, dispatches crypto requests. 1090 */ 1091 static void 1092 crypto_proc(void) 1093 { 1094 struct cryptop *crp, *submit; 1095 struct cryptkop *krp; 1096 struct cryptocap *cap; 1097 u_int32_t hid; 1098 int result, hint; 1099 1100 CRYPTO_Q_LOCK(); 1101 for (;;) { 1102 /* 1103 * Find the first element in the queue that can be 1104 * processed and look-ahead to see if multiple ops 1105 * are ready for the same driver. 1106 */ 1107 submit = NULL; 1108 hint = 0; 1109 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1110 hid = CRYPTO_SESID2HID(crp->crp_sid); 1111 cap = crypto_checkdriver(hid); 1112 /* 1113 * Driver cannot disappeared when there is an active 1114 * session. 1115 */ 1116 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1117 __func__, __LINE__)); 1118 if (cap == NULL || cap->cc_process == NULL) { 1119 /* Op needs to be migrated, process it. */ 1120 if (submit == NULL) 1121 submit = crp; 1122 break; 1123 } 1124 if (!cap->cc_qblocked) { 1125 if (submit != NULL) { 1126 /* 1127 * We stop on finding another op, 1128 * regardless whether its for the same 1129 * driver or not. We could keep 1130 * searching the queue but it might be 1131 * better to just use a per-driver 1132 * queue instead. 1133 */ 1134 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1135 hint = CRYPTO_HINT_MORE; 1136 break; 1137 } else { 1138 submit = crp; 1139 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1140 break; 1141 /* keep scanning for more are q'd */ 1142 } 1143 } 1144 } 1145 if (submit != NULL) { 1146 TAILQ_REMOVE(&crp_q, submit, crp_next); 1147 hid = CRYPTO_SESID2HID(submit->crp_sid); 1148 cap = crypto_checkdriver(hid); 1149 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1150 __func__, __LINE__)); 1151 result = crypto_invoke(cap, submit, hint); 1152 if (result == ERESTART) { 1153 /* 1154 * The driver ran out of resources, mark the 1155 * driver ``blocked'' for cryptop's and put 1156 * the request back in the queue. It would 1157 * best to put the request back where we got 1158 * it but that's hard so for now we put it 1159 * at the front. This should be ok; putting 1160 * it at the end does not work. 1161 */ 1162 /* XXX validate sid again? */ 1163 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1164 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1165 cryptostats.cs_blocks++; 1166 } 1167 } 1168 1169 /* As above, but for key ops */ 1170 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1171 cap = crypto_checkdriver(krp->krp_hid); 1172 if (cap == NULL || cap->cc_kprocess == NULL) { 1173 /* Op needs to be migrated, process it. */ 1174 break; 1175 } 1176 if (!cap->cc_kqblocked) 1177 break; 1178 } 1179 if (krp != NULL) { 1180 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1181 result = crypto_kinvoke(krp); 1182 if (result == ERESTART) { 1183 /* 1184 * The driver ran out of resources, mark the 1185 * driver ``blocked'' for cryptkop's and put 1186 * the request back in the queue. It would 1187 * best to put the request back where we got 1188 * it but that's hard so for now we put it 1189 * at the front. This should be ok; putting 1190 * it at the end does not work. 1191 */ 1192 /* XXX validate sid again? */ 1193 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1194 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1195 cryptostats.cs_kblocks++; 1196 } 1197 } 1198 1199 if (submit == NULL && krp == NULL) { 1200 /* 1201 * Nothing more to be processed. Sleep until we're 1202 * woken because there are more ops to process. 1203 * This happens either by submission or by a driver 1204 * becoming unblocked and notifying us through 1205 * crypto_unblock. Note that when we wakeup we 1206 * start processing each queue again from the 1207 * front. It's not clear that it's important to 1208 * preserve this ordering since ops may finish 1209 * out of order if dispatched to different devices 1210 * and some become blocked while others do not. 1211 */ 1212 crp_sleep = 1; 1213 msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0); 1214 crp_sleep = 0; 1215 if (cryptoproc == NULL) 1216 break; 1217 cryptostats.cs_intrs++; 1218 } 1219 } 1220 CRYPTO_Q_UNLOCK(); 1221 1222 crypto_finis(&crp_q); 1223 } 1224 1225 /* 1226 * Crypto returns thread, does callbacks for processed crypto requests. 1227 * Callbacks are done here, rather than in the crypto drivers, because 1228 * callbacks typically are expensive and would slow interrupt handling. 1229 */ 1230 static void 1231 crypto_ret_proc(void) 1232 { 1233 struct cryptop *crpt; 1234 struct cryptkop *krpt; 1235 1236 CRYPTO_RETQ_LOCK(); 1237 for (;;) { 1238 /* Harvest return q's for completed ops */ 1239 crpt = TAILQ_FIRST(&crp_ret_q); 1240 if (crpt != NULL) 1241 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1242 1243 krpt = TAILQ_FIRST(&crp_ret_kq); 1244 if (krpt != NULL) 1245 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1246 1247 if (crpt != NULL || krpt != NULL) { 1248 CRYPTO_RETQ_UNLOCK(); 1249 /* 1250 * Run callbacks unlocked. 1251 */ 1252 if (crpt != NULL) { 1253 #ifdef CRYPTO_TIMING 1254 if (crypto_timing) { 1255 /* 1256 * NB: We must copy the timestamp before 1257 * doing the callback as the cryptop is 1258 * likely to be reclaimed. 1259 */ 1260 struct bintime t = crpt->crp_tstamp; 1261 crypto_tstat(&cryptostats.cs_cb, &t); 1262 crpt->crp_callback(crpt); 1263 crypto_tstat(&cryptostats.cs_finis, &t); 1264 } else 1265 #endif 1266 crpt->crp_callback(crpt); 1267 } 1268 if (krpt != NULL) 1269 krpt->krp_callback(krpt); 1270 CRYPTO_RETQ_LOCK(); 1271 } else { 1272 /* 1273 * Nothing more to be processed. Sleep until we're 1274 * woken because there are more returns to process. 1275 */ 1276 msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT, 1277 "crypto_ret_wait", 0); 1278 if (cryptoretproc == NULL) 1279 break; 1280 cryptostats.cs_rets++; 1281 } 1282 } 1283 CRYPTO_RETQ_UNLOCK(); 1284 1285 crypto_finis(&crp_ret_q); 1286 } 1287