1 /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.28 2007/10/20 23:23:22 julian Exp $ */ 2 /*- 3 * Copyright (c) 2002-2006 Sam Leffler. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 /* 27 * Cryptographic Subsystem. 28 * 29 * This code is derived from the Openbsd Cryptographic Framework (OCF) 30 * that has the copyright shown below. Very little of the original 31 * code remains. 32 */ 33 34 /*- 35 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 36 * 37 * This code was written by Angelos D. Keromytis in Athens, Greece, in 38 * February 2000. Network Security Technologies Inc. (NSTI) kindly 39 * supported the development of this code. 40 * 41 * Copyright (c) 2000, 2001 Angelos D. Keromytis 42 * 43 * Permission to use, copy, and modify this software with or without fee 44 * is hereby granted, provided that this entire notice is included in 45 * all source code copies of any software which is or includes a copy or 46 * modification of this software. 47 * 48 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 49 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 50 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 51 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 52 * PURPOSE. 53 */ 54 55 #define CRYPTO_TIMING /* enable timing support */ 56 57 #include "opt_ddb.h" 58 59 #include <sys/param.h> 60 #include <sys/systm.h> 61 #include <sys/eventhandler.h> 62 #include <sys/kernel.h> 63 #include <sys/kthread.h> 64 #include <sys/lock.h> 65 #include <sys/module.h> 66 #include <sys/malloc.h> 67 #include <sys/proc.h> 68 #include <sys/sysctl.h> 69 70 #include <sys/thread2.h> 71 #include <sys/mplock2.h> 72 73 #include <vm/vm_zone.h> 74 75 #include <ddb/ddb.h> 76 77 #include <opencrypto/cryptodev.h> 78 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 79 80 #include <sys/kobj.h> 81 #include <sys/bus.h> 82 #include "cryptodev_if.h" 83 84 /* 85 * Crypto drivers register themselves by allocating a slot in the 86 * crypto_drivers table with crypto_get_driverid() and then registering 87 * each algorithm they support with crypto_register() and crypto_kregister(). 88 */ 89 static struct lock crypto_drivers_lock; /* lock on driver table */ 90 #define CRYPTO_DRIVER_LOCK() lockmgr(&crypto_drivers_lock, LK_EXCLUSIVE) 91 #define CRYPTO_DRIVER_UNLOCK() lockmgr(&crypto_drivers_lock, LK_RELEASE) 92 #define CRYPTO_DRIVER_ASSERT() KKASSERT(lockstatus(&crypto_drivers_lock, curthread) != 0) 93 94 /* 95 * Crypto device/driver capabilities structure. 96 * 97 * Synchronization: 98 * (d) - protected by CRYPTO_DRIVER_LOCK() 99 * (q) - protected by CRYPTO_Q_LOCK() 100 * Not tagged fields are read-only. 101 */ 102 struct cryptocap { 103 device_t cc_dev; /* (d) device/driver */ 104 u_int32_t cc_sessions; /* (d) # of sessions */ 105 u_int32_t cc_koperations; /* (d) # os asym operations */ 106 /* 107 * Largest possible operator length (in bits) for each type of 108 * encryption algorithm. XXX not used 109 */ 110 u_int16_t cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1]; 111 u_int8_t cc_alg[CRYPTO_ALGORITHM_MAX + 1]; 112 u_int8_t cc_kalg[CRK_ALGORITHM_MAX + 1]; 113 114 int cc_flags; /* (d) flags */ 115 #define CRYPTOCAP_F_CLEANUP 0x80000000 /* needs resource cleanup */ 116 int cc_qblocked; /* (q) symmetric q blocked */ 117 int cc_kqblocked; /* (q) asymmetric q blocked */ 118 }; 119 static struct cryptocap *crypto_drivers = NULL; 120 static int crypto_drivers_num = 0; 121 122 typedef struct crypto_tdinfo { 123 TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 124 TAILQ_HEAD(,cryptkop) crp_kq; 125 thread_t crp_td; 126 struct lock crp_lock; 127 int crp_sleep; 128 } *crypto_tdinfo_t; 129 130 /* 131 * There are two queues for crypto requests; one for symmetric (e.g. 132 * cipher) operations and one for asymmetric (e.g. MOD) operations. 133 * See below for how synchronization is handled. 134 * A single lock is used to lock access to both queues. We could 135 * have one per-queue but having one simplifies handling of block/unblock 136 * operations. 137 */ 138 static struct crypto_tdinfo tdinfo_array[MAXCPU]; 139 140 #define CRYPTO_Q_LOCK(tdinfo) lockmgr(&tdinfo->crp_lock, LK_EXCLUSIVE) 141 #define CRYPTO_Q_UNLOCK(tdinfo) lockmgr(&tdinfo->crp_lock, LK_RELEASE) 142 143 /* 144 * There are two queues for processing completed crypto requests; one 145 * for the symmetric and one for the asymmetric ops. We only need one 146 * but have two to avoid type futzing (cryptop vs. cryptkop). A single 147 * lock is used to lock access to both queues. Note that this lock 148 * must be separate from the lock on request queues to insure driver 149 * callbacks don't generate lock order reversals. 150 */ 151 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 152 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 153 static struct lock crypto_ret_q_lock; 154 #define CRYPTO_RETQ_LOCK() lockmgr(&crypto_ret_q_lock, LK_EXCLUSIVE) 155 #define CRYPTO_RETQ_UNLOCK() lockmgr(&crypto_ret_q_lock, LK_RELEASE) 156 #define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq)) 157 158 /* 159 * Crypto op and desciptor data structures are allocated 160 * from separate private zones. 161 */ 162 static vm_zone_t cryptop_zone; 163 static vm_zone_t cryptodesc_zone; 164 165 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 166 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 167 &crypto_userasymcrypto, 0, 168 "Enable/disable user-mode access to asymmetric crypto support"); 169 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 170 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 171 &crypto_devallowsoft, 0, 172 "Enable/disable use of software asym crypto support"); 173 int crypto_altdispatch = 0; /* dispatch to alternative cpu */ 174 SYSCTL_INT(_kern, OID_AUTO, cryptoaltdispatch, CTLFLAG_RW, 175 &crypto_altdispatch, 0, 176 "Do not queue crypto op on current cpu"); 177 178 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 179 180 static void crypto_proc(void *dummy); 181 static void crypto_ret_proc(void *dummy); 182 static struct thread *cryptoretthread; 183 static void crypto_destroy(void); 184 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint); 185 static int crypto_kinvoke(struct cryptkop *krp, int flags); 186 187 static struct cryptostats cryptostats; 188 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 189 cryptostats, "Crypto system statistics"); 190 191 #ifdef CRYPTO_TIMING 192 static int crypto_timing = 0; 193 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 194 &crypto_timing, 0, "Enable/disable crypto timing support"); 195 #endif 196 197 static int 198 crypto_init(void) 199 { 200 crypto_tdinfo_t tdinfo; 201 int error; 202 int n; 203 204 lockinit(&crypto_drivers_lock, "crypto driver table", 0, LK_CANRECURSE); 205 206 TAILQ_INIT(&crp_ret_q); 207 TAILQ_INIT(&crp_ret_kq); 208 lockinit(&crypto_ret_q_lock, "crypto return queues", 0, LK_CANRECURSE); 209 210 cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1); 211 cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc), 212 0, 0, 1); 213 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 214 kprintf("crypto_init: cannot setup crypto zones\n"); 215 error = ENOMEM; 216 goto bad; 217 } 218 219 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 220 crypto_drivers = kmalloc(crypto_drivers_num * sizeof(struct cryptocap), 221 M_CRYPTO_DATA, M_WAITOK | M_ZERO); 222 if (crypto_drivers == NULL) { 223 kprintf("crypto_init: cannot malloc driver table\n"); 224 error = ENOMEM; 225 goto bad; 226 } 227 228 for (n = 0; n < ncpus; ++n) { 229 tdinfo = &tdinfo_array[n]; 230 TAILQ_INIT(&tdinfo->crp_q); 231 TAILQ_INIT(&tdinfo->crp_kq); 232 lockinit(&tdinfo->crp_lock, "crypto op queues", 233 0, LK_CANRECURSE); 234 kthread_create_cpu(crypto_proc, tdinfo, &tdinfo->crp_td, 235 n, "crypto %d", n); 236 } 237 kthread_create(crypto_ret_proc, NULL, 238 &cryptoretthread, "crypto returns"); 239 return 0; 240 bad: 241 crypto_destroy(); 242 return error; 243 } 244 245 /* 246 * Signal a crypto thread to terminate. We use the driver 247 * table lock to synchronize the sleep/wakeups so that we 248 * are sure the threads have terminated before we release 249 * the data structures they use. See crypto_finis below 250 * for the other half of this song-and-dance. 251 */ 252 static void 253 crypto_terminate(struct thread **tp, void *q) 254 { 255 struct thread *t; 256 257 KKASSERT(lockstatus(&crypto_drivers_lock, curthread) != 0); 258 t = *tp; 259 *tp = NULL; 260 if (t) { 261 kprintf("crypto_terminate: start\n"); 262 wakeup_one(q); 263 crit_enter(); 264 tsleep_interlock(t, 0); 265 CRYPTO_DRIVER_UNLOCK(); /* let crypto_finis progress */ 266 crit_exit(); 267 tsleep(t, PINTERLOCKED, "crypto_destroy", 0); 268 CRYPTO_DRIVER_LOCK(); 269 kprintf("crypto_terminate: end\n"); 270 } 271 } 272 273 static void 274 crypto_destroy(void) 275 { 276 crypto_tdinfo_t tdinfo; 277 int n; 278 279 /* 280 * Terminate any crypto threads. 281 */ 282 CRYPTO_DRIVER_LOCK(); 283 for (n = 0; n < ncpus; ++n) { 284 tdinfo = &tdinfo_array[n]; 285 crypto_terminate(&tdinfo->crp_td, &tdinfo->crp_q); 286 lockuninit(&tdinfo->crp_lock); 287 } 288 crypto_terminate(&cryptoretthread, &crp_ret_q); 289 CRYPTO_DRIVER_UNLOCK(); 290 291 /* XXX flush queues??? */ 292 293 /* 294 * Reclaim dynamically allocated resources. 295 */ 296 if (crypto_drivers != NULL) 297 kfree(crypto_drivers, M_CRYPTO_DATA); 298 299 if (cryptodesc_zone != NULL) 300 zdestroy(cryptodesc_zone); 301 if (cryptop_zone != NULL) 302 zdestroy(cryptop_zone); 303 lockuninit(&crypto_ret_q_lock); 304 lockuninit(&crypto_drivers_lock); 305 } 306 307 static struct cryptocap * 308 crypto_checkdriver(u_int32_t hid) 309 { 310 if (crypto_drivers == NULL) 311 return NULL; 312 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 313 } 314 315 /* 316 * Compare a driver's list of supported algorithms against another 317 * list; return non-zero if all algorithms are supported. 318 */ 319 static int 320 driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri) 321 { 322 const struct cryptoini *cr; 323 324 /* See if all the algorithms are supported. */ 325 for (cr = cri; cr; cr = cr->cri_next) 326 if (cap->cc_alg[cr->cri_alg] == 0) 327 return 0; 328 return 1; 329 } 330 331 /* 332 * Select a driver for a new session that supports the specified 333 * algorithms and, optionally, is constrained according to the flags. 334 * The algorithm we use here is pretty stupid; just use the 335 * first driver that supports all the algorithms we need. If there 336 * are multiple drivers we choose the driver with the fewest active 337 * sessions. We prefer hardware-backed drivers to software ones. 338 * 339 * XXX We need more smarts here (in real life too, but that's 340 * XXX another story altogether). 341 */ 342 static struct cryptocap * 343 crypto_select_driver(const struct cryptoini *cri, int flags) 344 { 345 struct cryptocap *cap, *best; 346 int match, hid; 347 348 CRYPTO_DRIVER_ASSERT(); 349 350 /* 351 * Look first for hardware crypto devices if permitted. 352 */ 353 if (flags & CRYPTOCAP_F_HARDWARE) 354 match = CRYPTOCAP_F_HARDWARE; 355 else 356 match = CRYPTOCAP_F_SOFTWARE; 357 best = NULL; 358 again: 359 for (hid = 0; hid < crypto_drivers_num; hid++) { 360 cap = &crypto_drivers[hid]; 361 /* 362 * If it's not initialized, is in the process of 363 * going away, or is not appropriate (hardware 364 * or software based on match), then skip. 365 */ 366 if (cap->cc_dev == NULL || 367 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 368 (cap->cc_flags & match) == 0) 369 continue; 370 371 /* verify all the algorithms are supported. */ 372 if (driver_suitable(cap, cri)) { 373 if (best == NULL || 374 cap->cc_sessions < best->cc_sessions) 375 best = cap; 376 } 377 } 378 if (best != NULL) 379 return best; 380 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 381 /* sort of an Algol 68-style for loop */ 382 match = CRYPTOCAP_F_SOFTWARE; 383 goto again; 384 } 385 return best; 386 } 387 388 /* 389 * Create a new session. The crid argument specifies a crypto 390 * driver to use or constraints on a driver to select (hardware 391 * only, software only, either). Whatever driver is selected 392 * must be capable of the requested crypto algorithms. 393 */ 394 int 395 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int crid) 396 { 397 struct cryptocap *cap; 398 u_int32_t hid, lid; 399 int err; 400 401 CRYPTO_DRIVER_LOCK(); 402 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 403 /* 404 * Use specified driver; verify it is capable. 405 */ 406 cap = crypto_checkdriver(crid); 407 if (cap != NULL && !driver_suitable(cap, cri)) 408 cap = NULL; 409 } else { 410 /* 411 * No requested driver; select based on crid flags. 412 */ 413 cap = crypto_select_driver(cri, crid); 414 /* 415 * if NULL then can't do everything in one session. 416 * XXX Fix this. We need to inject a "virtual" session 417 * XXX layer right about here. 418 */ 419 } 420 if (cap != NULL) { 421 /* Call the driver initialization routine. */ 422 hid = cap - crypto_drivers; 423 lid = hid; /* Pass the driver ID. */ 424 err = CRYPTODEV_NEWSESSION(cap->cc_dev, &lid, cri); 425 if (err == 0) { 426 (*sid) = (cap->cc_flags & 0xff000000) 427 | (hid & 0x00ffffff); 428 (*sid) <<= 32; 429 (*sid) |= (lid & 0xffffffff); 430 cap->cc_sessions++; 431 } 432 } else 433 err = EINVAL; 434 CRYPTO_DRIVER_UNLOCK(); 435 return err; 436 } 437 438 static void 439 crypto_remove(struct cryptocap *cap) 440 { 441 442 KKASSERT(lockstatus(&crypto_drivers_lock, curthread) != 0); 443 if (cap->cc_sessions == 0 && cap->cc_koperations == 0) 444 bzero(cap, sizeof(*cap)); 445 } 446 447 /* 448 * Delete an existing session (or a reserved session on an unregistered 449 * driver). 450 */ 451 int 452 crypto_freesession(u_int64_t sid) 453 { 454 struct cryptocap *cap; 455 u_int32_t hid; 456 int err; 457 458 CRYPTO_DRIVER_LOCK(); 459 460 if (crypto_drivers == NULL) { 461 err = EINVAL; 462 goto done; 463 } 464 465 /* Determine two IDs. */ 466 hid = CRYPTO_SESID2HID(sid); 467 468 if (hid >= crypto_drivers_num) { 469 err = ENOENT; 470 goto done; 471 } 472 cap = &crypto_drivers[hid]; 473 474 if (cap->cc_sessions) 475 cap->cc_sessions--; 476 477 /* Call the driver cleanup routine, if available. */ 478 err = CRYPTODEV_FREESESSION(cap->cc_dev, sid); 479 480 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 481 crypto_remove(cap); 482 483 done: 484 CRYPTO_DRIVER_UNLOCK(); 485 return err; 486 } 487 488 /* 489 * Return an unused driver id. Used by drivers prior to registering 490 * support for the algorithms they handle. 491 */ 492 int32_t 493 crypto_get_driverid(device_t dev, int flags) 494 { 495 struct cryptocap *newdrv; 496 int i; 497 498 if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 499 kprintf("%s: no flags specified when registering driver\n", 500 device_get_nameunit(dev)); 501 return -1; 502 } 503 504 CRYPTO_DRIVER_LOCK(); 505 506 for (i = 0; i < crypto_drivers_num; i++) { 507 if (crypto_drivers[i].cc_dev == NULL && 508 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) { 509 break; 510 } 511 } 512 513 /* Out of entries, allocate some more. */ 514 if (i == crypto_drivers_num) { 515 /* Be careful about wrap-around. */ 516 if (2 * crypto_drivers_num <= crypto_drivers_num) { 517 CRYPTO_DRIVER_UNLOCK(); 518 kprintf("crypto: driver count wraparound!\n"); 519 return -1; 520 } 521 522 newdrv = kmalloc(2 * crypto_drivers_num * 523 sizeof(struct cryptocap), 524 M_CRYPTO_DATA, M_WAITOK|M_ZERO); 525 if (newdrv == NULL) { 526 CRYPTO_DRIVER_UNLOCK(); 527 kprintf("crypto: no space to expand driver table!\n"); 528 return -1; 529 } 530 531 bcopy(crypto_drivers, newdrv, 532 crypto_drivers_num * sizeof(struct cryptocap)); 533 534 crypto_drivers_num *= 2; 535 536 kfree(crypto_drivers, M_CRYPTO_DATA); 537 crypto_drivers = newdrv; 538 } 539 540 /* NB: state is zero'd on free */ 541 crypto_drivers[i].cc_sessions = 1; /* Mark */ 542 crypto_drivers[i].cc_dev = dev; 543 crypto_drivers[i].cc_flags = flags; 544 if (bootverbose) 545 kprintf("crypto: assign %s driver id %u, flags %u\n", 546 device_get_nameunit(dev), i, flags); 547 548 CRYPTO_DRIVER_UNLOCK(); 549 550 return i; 551 } 552 553 /* 554 * Lookup a driver by name. We match against the full device 555 * name and unit, and against just the name. The latter gives 556 * us a simple widlcarding by device name. On success return the 557 * driver/hardware identifier; otherwise return -1. 558 */ 559 int 560 crypto_find_driver(const char *match) 561 { 562 int i, len = strlen(match); 563 564 CRYPTO_DRIVER_LOCK(); 565 for (i = 0; i < crypto_drivers_num; i++) { 566 device_t dev = crypto_drivers[i].cc_dev; 567 if (dev == NULL || 568 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP)) 569 continue; 570 if (strncmp(match, device_get_nameunit(dev), len) == 0 || 571 strncmp(match, device_get_name(dev), len) == 0) 572 break; 573 } 574 CRYPTO_DRIVER_UNLOCK(); 575 return i < crypto_drivers_num ? i : -1; 576 } 577 578 /* 579 * Return the device_t for the specified driver or NULL 580 * if the driver identifier is invalid. 581 */ 582 device_t 583 crypto_find_device_byhid(int hid) 584 { 585 struct cryptocap *cap = crypto_checkdriver(hid); 586 return cap != NULL ? cap->cc_dev : NULL; 587 } 588 589 /* 590 * Return the device/driver capabilities. 591 */ 592 int 593 crypto_getcaps(int hid) 594 { 595 struct cryptocap *cap = crypto_checkdriver(hid); 596 return cap != NULL ? cap->cc_flags : 0; 597 } 598 599 /* 600 * Register support for a key-related algorithm. This routine 601 * is called once for each algorithm supported a driver. 602 */ 603 int 604 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags) 605 { 606 struct cryptocap *cap; 607 int err; 608 609 CRYPTO_DRIVER_LOCK(); 610 611 cap = crypto_checkdriver(driverid); 612 if (cap != NULL && 613 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 614 /* 615 * XXX Do some performance testing to determine placing. 616 * XXX We probably need an auxiliary data structure that 617 * XXX describes relative performances. 618 */ 619 620 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 621 if (bootverbose) 622 kprintf("crypto: %s registers key alg %u flags %u\n" 623 , device_get_nameunit(cap->cc_dev) 624 , kalg 625 , flags 626 ); 627 628 err = 0; 629 } else 630 err = EINVAL; 631 632 CRYPTO_DRIVER_UNLOCK(); 633 return err; 634 } 635 636 /* 637 * Register support for a non-key-related algorithm. This routine 638 * is called once for each such algorithm supported by a driver. 639 */ 640 int 641 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 642 u_int32_t flags) 643 { 644 struct cryptocap *cap; 645 int err; 646 647 CRYPTO_DRIVER_LOCK(); 648 649 cap = crypto_checkdriver(driverid); 650 /* NB: algorithms are in the range [1..max] */ 651 if (cap != NULL && 652 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 653 /* 654 * XXX Do some performance testing to determine placing. 655 * XXX We probably need an auxiliary data structure that 656 * XXX describes relative performances. 657 */ 658 659 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 660 cap->cc_max_op_len[alg] = maxoplen; 661 if (bootverbose) 662 kprintf("crypto: %s registers alg %u flags %u maxoplen %u\n" 663 , device_get_nameunit(cap->cc_dev) 664 , alg 665 , flags 666 , maxoplen 667 ); 668 cap->cc_sessions = 0; /* Unmark */ 669 err = 0; 670 } else 671 err = EINVAL; 672 673 CRYPTO_DRIVER_UNLOCK(); 674 return err; 675 } 676 677 static void 678 driver_finis(struct cryptocap *cap) 679 { 680 u_int32_t ses, kops; 681 682 CRYPTO_DRIVER_ASSERT(); 683 684 ses = cap->cc_sessions; 685 kops = cap->cc_koperations; 686 bzero(cap, sizeof(*cap)); 687 if (ses != 0 || kops != 0) { 688 /* 689 * If there are pending sessions, 690 * just mark as invalid. 691 */ 692 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 693 cap->cc_sessions = ses; 694 cap->cc_koperations = kops; 695 } 696 } 697 698 /* 699 * Unregister a crypto driver. If there are pending sessions using it, 700 * leave enough information around so that subsequent calls using those 701 * sessions will correctly detect the driver has been unregistered and 702 * reroute requests. 703 */ 704 int 705 crypto_unregister(u_int32_t driverid, int alg) 706 { 707 struct cryptocap *cap; 708 int i, err; 709 710 CRYPTO_DRIVER_LOCK(); 711 cap = crypto_checkdriver(driverid); 712 if (cap != NULL && 713 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 714 cap->cc_alg[alg] != 0) { 715 cap->cc_alg[alg] = 0; 716 cap->cc_max_op_len[alg] = 0; 717 718 /* Was this the last algorithm ? */ 719 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) { 720 if (cap->cc_alg[i] != 0) 721 break; 722 } 723 724 if (i == CRYPTO_ALGORITHM_MAX + 1) 725 driver_finis(cap); 726 err = 0; 727 } else { 728 err = EINVAL; 729 } 730 CRYPTO_DRIVER_UNLOCK(); 731 732 return err; 733 } 734 735 /* 736 * Unregister all algorithms associated with a crypto driver. 737 * If there are pending sessions using it, leave enough information 738 * around so that subsequent calls using those sessions will 739 * correctly detect the driver has been unregistered and reroute 740 * requests. 741 */ 742 int 743 crypto_unregister_all(u_int32_t driverid) 744 { 745 struct cryptocap *cap; 746 int err; 747 748 CRYPTO_DRIVER_LOCK(); 749 cap = crypto_checkdriver(driverid); 750 if (cap != NULL) { 751 driver_finis(cap); 752 err = 0; 753 } else { 754 err = EINVAL; 755 } 756 CRYPTO_DRIVER_UNLOCK(); 757 758 return err; 759 } 760 761 /* 762 * Clear blockage on a driver. The what parameter indicates whether 763 * the driver is now ready for cryptop's and/or cryptokop's. 764 */ 765 int 766 crypto_unblock(u_int32_t driverid, int what) 767 { 768 crypto_tdinfo_t tdinfo; 769 struct cryptocap *cap; 770 int err; 771 int n; 772 773 CRYPTO_DRIVER_LOCK(); 774 cap = crypto_checkdriver(driverid); 775 if (cap != NULL) { 776 if (what & CRYPTO_SYMQ) 777 cap->cc_qblocked = 0; 778 if (what & CRYPTO_ASYMQ) 779 cap->cc_kqblocked = 0; 780 for (n = 0; n < ncpus; ++n) { 781 tdinfo = &tdinfo_array[n]; 782 CRYPTO_Q_LOCK(tdinfo); 783 if (tdinfo[n].crp_sleep) 784 wakeup_one(&tdinfo->crp_q); 785 CRYPTO_Q_UNLOCK(tdinfo); 786 } 787 err = 0; 788 } else { 789 err = EINVAL; 790 } 791 CRYPTO_DRIVER_UNLOCK(); 792 793 return err; 794 } 795 796 static volatile int dispatch_rover; 797 798 /* 799 * Add a crypto request to a queue, to be processed by the kernel thread. 800 */ 801 int 802 crypto_dispatch(struct cryptop *crp) 803 { 804 crypto_tdinfo_t tdinfo; 805 struct cryptocap *cap; 806 u_int32_t hid; 807 int result; 808 int n; 809 810 cryptostats.cs_ops++; 811 812 #ifdef CRYPTO_TIMING 813 if (crypto_timing) 814 nanouptime(&crp->crp_tstamp); 815 #endif 816 817 hid = CRYPTO_SESID2HID(crp->crp_sid); 818 819 /* 820 * Dispatch the crypto op directly to the driver if the caller 821 * marked the request to be processed immediately or this is 822 * a synchronous callback chain occuring from within a crypto 823 * processing thread. 824 * 825 * Fall through to queueing the driver is blocked. 826 */ 827 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0 || 828 (curthread->td_flags & TDF_CRYPTO)) { 829 cap = crypto_checkdriver(hid); 830 /* Driver cannot disappeared when there is an active session. */ 831 KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__)); 832 if (!cap->cc_qblocked) { 833 result = crypto_invoke(cap, crp, 0); 834 if (result != ERESTART) 835 return (result); 836 /* 837 * The driver ran out of resources, put the request on 838 * the queue. 839 */ 840 } 841 } 842 843 /* 844 * Dispatch to a cpu for action if possible. Dispatch to a different 845 * cpu than the current cpu. 846 */ 847 if (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SMP) { 848 n = atomic_fetchadd_int(&dispatch_rover, 1) & 255; 849 if (crypto_altdispatch && mycpu->gd_cpuid == n) 850 ++n; 851 n = n % ncpus; 852 } else { 853 n = 0; 854 } 855 tdinfo = &tdinfo_array[n]; 856 857 CRYPTO_Q_LOCK(tdinfo); 858 TAILQ_INSERT_TAIL(&tdinfo->crp_q, crp, crp_next); 859 if (tdinfo->crp_sleep) 860 wakeup_one(&tdinfo->crp_q); 861 CRYPTO_Q_UNLOCK(tdinfo); 862 return 0; 863 } 864 865 /* 866 * Add an asymetric crypto request to a queue, 867 * to be processed by the kernel thread. 868 */ 869 int 870 crypto_kdispatch(struct cryptkop *krp) 871 { 872 crypto_tdinfo_t tdinfo; 873 int error; 874 int n; 875 876 cryptostats.cs_kops++; 877 878 #if 0 879 /* not sure how to test F_SMP here */ 880 n = atomic_fetchadd_int(&dispatch_rover, 1) & 255; 881 n = n % ncpus; 882 #endif 883 n = 0; 884 tdinfo = &tdinfo_array[n]; 885 886 error = crypto_kinvoke(krp, krp->krp_crid); 887 888 if (error == ERESTART) { 889 CRYPTO_Q_LOCK(tdinfo); 890 TAILQ_INSERT_TAIL(&tdinfo->crp_kq, krp, krp_next); 891 if (tdinfo->crp_sleep) 892 wakeup_one(&tdinfo->crp_q); 893 CRYPTO_Q_UNLOCK(tdinfo); 894 error = 0; 895 } 896 return error; 897 } 898 899 /* 900 * Verify a driver is suitable for the specified operation. 901 */ 902 static __inline int 903 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp) 904 { 905 return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0; 906 } 907 908 /* 909 * Select a driver for an asym operation. The driver must 910 * support the necessary algorithm. The caller can constrain 911 * which device is selected with the flags parameter. The 912 * algorithm we use here is pretty stupid; just use the first 913 * driver that supports the algorithms we need. If there are 914 * multiple suitable drivers we choose the driver with the 915 * fewest active operations. We prefer hardware-backed 916 * drivers to software ones when either may be used. 917 */ 918 static struct cryptocap * 919 crypto_select_kdriver(const struct cryptkop *krp, int flags) 920 { 921 struct cryptocap *cap, *best, *blocked; 922 int match, hid; 923 924 CRYPTO_DRIVER_ASSERT(); 925 926 /* 927 * Look first for hardware crypto devices if permitted. 928 */ 929 if (flags & CRYPTOCAP_F_HARDWARE) 930 match = CRYPTOCAP_F_HARDWARE; 931 else 932 match = CRYPTOCAP_F_SOFTWARE; 933 best = NULL; 934 blocked = NULL; 935 again: 936 for (hid = 0; hid < crypto_drivers_num; hid++) { 937 cap = &crypto_drivers[hid]; 938 /* 939 * If it's not initialized, is in the process of 940 * going away, or is not appropriate (hardware 941 * or software based on match), then skip. 942 */ 943 if (cap->cc_dev == NULL || 944 (cap->cc_flags & CRYPTOCAP_F_CLEANUP) || 945 (cap->cc_flags & match) == 0) 946 continue; 947 948 /* verify all the algorithms are supported. */ 949 if (kdriver_suitable(cap, krp)) { 950 if (best == NULL || 951 cap->cc_koperations < best->cc_koperations) 952 best = cap; 953 } 954 } 955 if (best != NULL) 956 return best; 957 if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) { 958 /* sort of an Algol 68-style for loop */ 959 match = CRYPTOCAP_F_SOFTWARE; 960 goto again; 961 } 962 return best; 963 } 964 965 /* 966 * Dispatch an assymetric crypto request. 967 */ 968 static int 969 crypto_kinvoke(struct cryptkop *krp, int crid) 970 { 971 struct cryptocap *cap = NULL; 972 int error; 973 974 KASSERT(krp != NULL, ("%s: krp == NULL", __func__)); 975 KASSERT(krp->krp_callback != NULL, 976 ("%s: krp->crp_callback == NULL", __func__)); 977 978 CRYPTO_DRIVER_LOCK(); 979 if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) { 980 cap = crypto_checkdriver(crid); 981 if (cap != NULL) { 982 /* 983 * Driver present, it must support the necessary 984 * algorithm and, if s/w drivers are excluded, 985 * it must be registered as hardware-backed. 986 */ 987 if (!kdriver_suitable(cap, krp) || 988 (!crypto_devallowsoft && 989 (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0)) 990 cap = NULL; 991 } 992 } else { 993 /* 994 * No requested driver; select based on crid flags. 995 */ 996 if (!crypto_devallowsoft) /* NB: disallow s/w drivers */ 997 crid &= ~CRYPTOCAP_F_SOFTWARE; 998 cap = crypto_select_kdriver(krp, crid); 999 } 1000 if (cap != NULL && !cap->cc_kqblocked) { 1001 krp->krp_hid = cap - crypto_drivers; 1002 cap->cc_koperations++; 1003 CRYPTO_DRIVER_UNLOCK(); 1004 error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0); 1005 CRYPTO_DRIVER_LOCK(); 1006 if (error == ERESTART) { 1007 cap->cc_koperations--; 1008 CRYPTO_DRIVER_UNLOCK(); 1009 return (error); 1010 } 1011 } else { 1012 /* 1013 * NB: cap is !NULL if device is blocked; in 1014 * that case return ERESTART so the operation 1015 * is resubmitted if possible. 1016 */ 1017 error = (cap == NULL) ? ENODEV : ERESTART; 1018 } 1019 CRYPTO_DRIVER_UNLOCK(); 1020 1021 if (error) { 1022 krp->krp_status = error; 1023 crypto_kdone(krp); 1024 } 1025 return 0; 1026 } 1027 1028 #ifdef CRYPTO_TIMING 1029 static void 1030 crypto_tstat(struct cryptotstat *ts, struct timespec *tv) 1031 { 1032 struct timespec now, t; 1033 1034 nanouptime(&now); 1035 t.tv_sec = now.tv_sec - tv->tv_sec; 1036 t.tv_nsec = now.tv_nsec - tv->tv_nsec; 1037 if (t.tv_nsec < 0) { 1038 t.tv_sec--; 1039 t.tv_nsec += 1000000000; 1040 } 1041 timespecadd(&ts->acc, &t); 1042 if (timespeccmp(&t, &ts->min, <)) 1043 ts->min = t; 1044 if (timespeccmp(&t, &ts->max, >)) 1045 ts->max = t; 1046 ts->count++; 1047 1048 *tv = now; 1049 } 1050 #endif 1051 1052 /* 1053 * Dispatch a crypto request to the appropriate crypto devices. 1054 */ 1055 static int 1056 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint) 1057 { 1058 1059 KASSERT(crp != NULL, ("%s: crp == NULL", __func__)); 1060 KASSERT(crp->crp_callback != NULL, 1061 ("%s: crp->crp_callback == NULL", __func__)); 1062 KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__)); 1063 1064 #ifdef CRYPTO_TIMING 1065 if (crypto_timing) 1066 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 1067 #endif 1068 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) { 1069 struct cryptodesc *crd; 1070 u_int64_t nid; 1071 1072 /* 1073 * Driver has unregistered; migrate the session and return 1074 * an error to the caller so they'll resubmit the op. 1075 * 1076 * XXX: What if there are more already queued requests for this 1077 * session? 1078 */ 1079 crypto_freesession(crp->crp_sid); 1080 1081 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 1082 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 1083 1084 /* XXX propagate flags from initial session? */ 1085 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 1086 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0) 1087 crp->crp_sid = nid; 1088 1089 crp->crp_etype = EAGAIN; 1090 crypto_done(crp); 1091 return 0; 1092 } else { 1093 /* 1094 * Invoke the driver to process the request. 1095 */ 1096 return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint); 1097 } 1098 } 1099 1100 /* 1101 * Release a set of crypto descriptors. 1102 */ 1103 void 1104 crypto_freereq(struct cryptop *crp) 1105 { 1106 struct cryptodesc *crd; 1107 #ifdef DIAGNOSTIC 1108 crypto_tdinfo_t tdinfo; 1109 struct cryptop *crp2; 1110 int n; 1111 #endif 1112 1113 if (crp == NULL) 1114 return; 1115 1116 #ifdef DIAGNOSTIC 1117 for (n = 0; n < ncpus; ++n) { 1118 tdinfo = &tdinfo_array[n]; 1119 1120 CRYPTO_Q_LOCK(tdinfo); 1121 TAILQ_FOREACH(crp2, &tdinfo->crp_q, crp_next) { 1122 KASSERT(crp2 != crp, 1123 ("Freeing cryptop from the crypto queue (%p).", 1124 crp)); 1125 } 1126 CRYPTO_Q_UNLOCK(tdinfo); 1127 } 1128 CRYPTO_RETQ_LOCK(); 1129 TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) { 1130 KASSERT(crp2 != crp, 1131 ("Freeing cryptop from the return queue (%p).", 1132 crp)); 1133 } 1134 CRYPTO_RETQ_UNLOCK(); 1135 #endif 1136 1137 while ((crd = crp->crp_desc) != NULL) { 1138 crp->crp_desc = crd->crd_next; 1139 zfree(cryptodesc_zone, crd); 1140 } 1141 zfree(cryptop_zone, crp); 1142 } 1143 1144 /* 1145 * Acquire a set of crypto descriptors. 1146 */ 1147 struct cryptop * 1148 crypto_getreq(int num) 1149 { 1150 struct cryptodesc *crd; 1151 struct cryptop *crp; 1152 1153 crp = zalloc(cryptop_zone); 1154 if (crp != NULL) { 1155 bzero(crp, sizeof (*crp)); 1156 while (num--) { 1157 crd = zalloc(cryptodesc_zone); 1158 if (crd == NULL) { 1159 crypto_freereq(crp); 1160 return NULL; 1161 } 1162 bzero(crd, sizeof (*crd)); 1163 1164 crd->crd_next = crp->crp_desc; 1165 crp->crp_desc = crd; 1166 } 1167 } 1168 return crp; 1169 } 1170 1171 /* 1172 * Invoke the callback on behalf of the driver. 1173 */ 1174 void 1175 crypto_done(struct cryptop *crp) 1176 { 1177 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 1178 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 1179 crp->crp_flags |= CRYPTO_F_DONE; 1180 if (crp->crp_etype != 0) 1181 cryptostats.cs_errs++; 1182 #ifdef CRYPTO_TIMING 1183 if (crypto_timing) 1184 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 1185 #endif 1186 /* 1187 * CBIMM means unconditionally do the callback immediately; 1188 * CBIFSYNC means do the callback immediately only if the 1189 * operation was done synchronously. Both are used to avoid 1190 * doing extraneous context switches; the latter is mostly 1191 * used with the software crypto driver. 1192 */ 1193 if ((crp->crp_flags & CRYPTO_F_CBIMM) || 1194 ((crp->crp_flags & CRYPTO_F_CBIFSYNC) && 1195 (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SYNC))) { 1196 /* 1197 * Do the callback directly. This is ok when the 1198 * callback routine does very little (e.g. the 1199 * /dev/crypto callback method just does a wakeup). 1200 */ 1201 #ifdef CRYPTO_TIMING 1202 if (crypto_timing) { 1203 /* 1204 * NB: We must copy the timestamp before 1205 * doing the callback as the cryptop is 1206 * likely to be reclaimed. 1207 */ 1208 struct timespec t = crp->crp_tstamp; 1209 crypto_tstat(&cryptostats.cs_cb, &t); 1210 crp->crp_callback(crp); 1211 crypto_tstat(&cryptostats.cs_finis, &t); 1212 } else 1213 #endif 1214 crp->crp_callback(crp); 1215 } else { 1216 /* 1217 * Normal case; queue the callback for the thread. 1218 */ 1219 CRYPTO_RETQ_LOCK(); 1220 if (CRYPTO_RETQ_EMPTY()) 1221 wakeup_one(&crp_ret_q); /* shared wait channel */ 1222 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 1223 CRYPTO_RETQ_UNLOCK(); 1224 } 1225 } 1226 1227 /* 1228 * Invoke the callback on behalf of the driver. 1229 */ 1230 void 1231 crypto_kdone(struct cryptkop *krp) 1232 { 1233 struct cryptocap *cap; 1234 1235 if (krp->krp_status != 0) 1236 cryptostats.cs_kerrs++; 1237 CRYPTO_DRIVER_LOCK(); 1238 /* XXX: What if driver is loaded in the meantime? */ 1239 if (krp->krp_hid < crypto_drivers_num) { 1240 cap = &crypto_drivers[krp->krp_hid]; 1241 cap->cc_koperations--; 1242 KASSERT(cap->cc_koperations >= 0, ("cc_koperations < 0")); 1243 if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) 1244 crypto_remove(cap); 1245 } 1246 CRYPTO_DRIVER_UNLOCK(); 1247 CRYPTO_RETQ_LOCK(); 1248 if (CRYPTO_RETQ_EMPTY()) 1249 wakeup_one(&crp_ret_q); /* shared wait channel */ 1250 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 1251 CRYPTO_RETQ_UNLOCK(); 1252 } 1253 1254 int 1255 crypto_getfeat(int *featp) 1256 { 1257 int hid, kalg, feat = 0; 1258 1259 CRYPTO_DRIVER_LOCK(); 1260 for (hid = 0; hid < crypto_drivers_num; hid++) { 1261 const struct cryptocap *cap = &crypto_drivers[hid]; 1262 1263 if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) && 1264 !crypto_devallowsoft) { 1265 continue; 1266 } 1267 for (kalg = 0; kalg <= CRK_ALGORITHM_MAX; kalg++) 1268 if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED) 1269 feat |= 1 << kalg; 1270 } 1271 CRYPTO_DRIVER_UNLOCK(); 1272 *featp = feat; 1273 return (0); 1274 } 1275 1276 /* 1277 * Terminate a thread at module unload. The process that 1278 * initiated this is waiting for us to signal that we're gone; 1279 * wake it up and exit. We use the driver table lock to insure 1280 * we don't do the wakeup before they're waiting. There is no 1281 * race here because the waiter sleeps on the proc lock for the 1282 * thread so it gets notified at the right time because of an 1283 * extra wakeup that's done in exit1(). 1284 */ 1285 static void 1286 crypto_finis(void *chan) 1287 { 1288 CRYPTO_DRIVER_LOCK(); 1289 wakeup_one(chan); 1290 CRYPTO_DRIVER_UNLOCK(); 1291 kthread_exit(); 1292 } 1293 1294 /* 1295 * Crypto thread, dispatches crypto requests. 1296 * 1297 * MPSAFE 1298 */ 1299 static void 1300 crypto_proc(void *arg) 1301 { 1302 crypto_tdinfo_t tdinfo = arg; 1303 struct cryptop *crp, *submit; 1304 struct cryptkop *krp; 1305 struct cryptocap *cap; 1306 u_int32_t hid; 1307 int result, hint; 1308 1309 CRYPTO_Q_LOCK(tdinfo); 1310 1311 curthread->td_flags |= TDF_CRYPTO; 1312 1313 for (;;) { 1314 /* 1315 * Find the first element in the queue that can be 1316 * processed and look-ahead to see if multiple ops 1317 * are ready for the same driver. 1318 */ 1319 submit = NULL; 1320 hint = 0; 1321 TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) { 1322 hid = CRYPTO_SESID2HID(crp->crp_sid); 1323 cap = crypto_checkdriver(hid); 1324 /* 1325 * Driver cannot disappeared when there is an active 1326 * session. 1327 */ 1328 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1329 __func__, __LINE__)); 1330 if (cap == NULL || cap->cc_dev == NULL) { 1331 /* Op needs to be migrated, process it. */ 1332 if (submit == NULL) 1333 submit = crp; 1334 break; 1335 } 1336 if (!cap->cc_qblocked) { 1337 if (submit != NULL) { 1338 /* 1339 * We stop on finding another op, 1340 * regardless whether its for the same 1341 * driver or not. We could keep 1342 * searching the queue but it might be 1343 * better to just use a per-driver 1344 * queue instead. 1345 */ 1346 if (CRYPTO_SESID2HID(submit->crp_sid) == hid) 1347 hint = CRYPTO_HINT_MORE; 1348 break; 1349 } else { 1350 submit = crp; 1351 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1352 break; 1353 /* keep scanning for more are q'd */ 1354 } 1355 } 1356 } 1357 if (submit != NULL) { 1358 TAILQ_REMOVE(&tdinfo->crp_q, submit, crp_next); 1359 hid = CRYPTO_SESID2HID(submit->crp_sid); 1360 cap = crypto_checkdriver(hid); 1361 KASSERT(cap != NULL, ("%s:%u Driver disappeared.", 1362 __func__, __LINE__)); 1363 1364 CRYPTO_Q_UNLOCK(tdinfo); 1365 result = crypto_invoke(cap, submit, hint); 1366 CRYPTO_Q_LOCK(tdinfo); 1367 1368 if (result == ERESTART) { 1369 /* 1370 * The driver ran out of resources, mark the 1371 * driver ``blocked'' for cryptop's and put 1372 * the request back in the queue. It would 1373 * best to put the request back where we got 1374 * it but that's hard so for now we put it 1375 * at the front. This should be ok; putting 1376 * it at the end does not work. 1377 */ 1378 /* XXX validate sid again? */ 1379 crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1380 TAILQ_INSERT_HEAD(&tdinfo->crp_q, 1381 submit, crp_next); 1382 cryptostats.cs_blocks++; 1383 } 1384 } 1385 1386 /* As above, but for key ops */ 1387 TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) { 1388 cap = crypto_checkdriver(krp->krp_hid); 1389 if (cap == NULL || cap->cc_dev == NULL) { 1390 /* 1391 * Operation needs to be migrated, invalidate 1392 * the assigned device so it will reselect a 1393 * new one below. Propagate the original 1394 * crid selection flags if supplied. 1395 */ 1396 krp->krp_hid = krp->krp_crid & 1397 (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE); 1398 if (krp->krp_hid == 0) 1399 krp->krp_hid = 1400 CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE; 1401 break; 1402 } 1403 if (!cap->cc_kqblocked) 1404 break; 1405 } 1406 if (krp != NULL) { 1407 TAILQ_REMOVE(&tdinfo->crp_kq, krp, krp_next); 1408 1409 CRYPTO_Q_UNLOCK(tdinfo); 1410 result = crypto_kinvoke(krp, krp->krp_hid); 1411 CRYPTO_Q_LOCK(tdinfo); 1412 1413 if (result == ERESTART) { 1414 /* 1415 * The driver ran out of resources, mark the 1416 * driver ``blocked'' for cryptkop's and put 1417 * the request back in the queue. It would 1418 * best to put the request back where we got 1419 * it but that's hard so for now we put it 1420 * at the front. This should be ok; putting 1421 * it at the end does not work. 1422 */ 1423 /* XXX validate sid again? */ 1424 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1425 TAILQ_INSERT_HEAD(&tdinfo->crp_kq, 1426 krp, krp_next); 1427 cryptostats.cs_kblocks++; 1428 } 1429 } 1430 1431 if (submit == NULL && krp == NULL) { 1432 /* 1433 * Nothing more to be processed. Sleep until we're 1434 * woken because there are more ops to process. 1435 * This happens either by submission or by a driver 1436 * becoming unblocked and notifying us through 1437 * crypto_unblock. Note that when we wakeup we 1438 * start processing each queue again from the 1439 * front. It's not clear that it's important to 1440 * preserve this ordering since ops may finish 1441 * out of order if dispatched to different devices 1442 * and some become blocked while others do not. 1443 */ 1444 tdinfo->crp_sleep = 1; 1445 lksleep (&tdinfo->crp_q, &tdinfo->crp_lock, 1446 0, "crypto_wait", 0); 1447 tdinfo->crp_sleep = 0; 1448 if (tdinfo->crp_td == NULL) 1449 break; 1450 cryptostats.cs_intrs++; 1451 } 1452 } 1453 CRYPTO_Q_UNLOCK(tdinfo); 1454 1455 crypto_finis(&tdinfo->crp_q); 1456 } 1457 1458 /* 1459 * Crypto returns thread, does callbacks for processed crypto requests. 1460 * Callbacks are done here, rather than in the crypto drivers, because 1461 * callbacks typically are expensive and would slow interrupt handling. 1462 * 1463 * MPSAFE 1464 */ 1465 static void 1466 crypto_ret_proc(void *dummy __unused) 1467 { 1468 struct cryptop *crpt; 1469 struct cryptkop *krpt; 1470 1471 get_mplock(); 1472 CRYPTO_RETQ_LOCK(); 1473 for (;;) { 1474 /* Harvest return q's for completed ops */ 1475 crpt = TAILQ_FIRST(&crp_ret_q); 1476 if (crpt != NULL) 1477 TAILQ_REMOVE(&crp_ret_q, crpt, crp_next); 1478 1479 krpt = TAILQ_FIRST(&crp_ret_kq); 1480 if (krpt != NULL) 1481 TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next); 1482 1483 if (crpt != NULL || krpt != NULL) { 1484 CRYPTO_RETQ_UNLOCK(); 1485 /* 1486 * Run callbacks unlocked. 1487 */ 1488 if (crpt != NULL) { 1489 #ifdef CRYPTO_TIMING 1490 if (crypto_timing) { 1491 /* 1492 * NB: We must copy the timestamp before 1493 * doing the callback as the cryptop is 1494 * likely to be reclaimed. 1495 */ 1496 struct timespec t = crpt->crp_tstamp; 1497 crypto_tstat(&cryptostats.cs_cb, &t); 1498 crpt->crp_callback(crpt); 1499 crypto_tstat(&cryptostats.cs_finis, &t); 1500 } else 1501 #endif 1502 crpt->crp_callback(crpt); 1503 } 1504 if (krpt != NULL) 1505 krpt->krp_callback(krpt); 1506 CRYPTO_RETQ_LOCK(); 1507 } else { 1508 /* 1509 * Nothing more to be processed. Sleep until we're 1510 * woken because there are more returns to process. 1511 */ 1512 lksleep (&crp_ret_q, &crypto_ret_q_lock, 1513 0, "crypto_ret_wait", 0); 1514 if (cryptoretthread == NULL) 1515 break; 1516 cryptostats.cs_rets++; 1517 } 1518 } 1519 CRYPTO_RETQ_UNLOCK(); 1520 1521 crypto_finis(&crp_ret_q); 1522 } 1523 1524 #ifdef DDB 1525 static void 1526 db_show_drivers(void) 1527 { 1528 int hid; 1529 1530 db_printf("%12s %4s %4s %8s %2s %2s\n" 1531 , "Device" 1532 , "Ses" 1533 , "Kops" 1534 , "Flags" 1535 , "QB" 1536 , "KB" 1537 ); 1538 for (hid = 0; hid < crypto_drivers_num; hid++) { 1539 const struct cryptocap *cap = &crypto_drivers[hid]; 1540 if (cap->cc_dev == NULL) 1541 continue; 1542 db_printf("%-12s %4u %4u %08x %2u %2u\n" 1543 , device_get_nameunit(cap->cc_dev) 1544 , cap->cc_sessions 1545 , cap->cc_koperations 1546 , cap->cc_flags 1547 , cap->cc_qblocked 1548 , cap->cc_kqblocked 1549 ); 1550 } 1551 } 1552 1553 DB_SHOW_COMMAND(crypto, db_show_crypto) 1554 { 1555 crypto_tdinfo_t tdinfo; 1556 struct cryptop *crp; 1557 int n; 1558 1559 db_show_drivers(); 1560 db_printf("\n"); 1561 1562 db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n", 1563 "HID", "Caps", "Ilen", "Olen", "Etype", "Flags", 1564 "Desc", "Callback"); 1565 1566 for (n = 0; n < ncpus; ++n) { 1567 tdinfo = &tdinfo_array[n]; 1568 1569 TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) { 1570 db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n" 1571 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1572 , (int) CRYPTO_SESID2CAPS(crp->crp_sid) 1573 , crp->crp_ilen, crp->crp_olen 1574 , crp->crp_etype 1575 , crp->crp_flags 1576 , crp->crp_desc 1577 , crp->crp_callback 1578 ); 1579 } 1580 } 1581 if (!TAILQ_EMPTY(&crp_ret_q)) { 1582 db_printf("\n%4s %4s %4s %8s\n", 1583 "HID", "Etype", "Flags", "Callback"); 1584 TAILQ_FOREACH(crp, &crp_ret_q, crp_next) { 1585 db_printf("%4u %4u %04x %8p\n" 1586 , (int) CRYPTO_SESID2HID(crp->crp_sid) 1587 , crp->crp_etype 1588 , crp->crp_flags 1589 , crp->crp_callback 1590 ); 1591 } 1592 } 1593 } 1594 1595 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto) 1596 { 1597 crypto_tdinfo_t tdinfo; 1598 struct cryptkop *krp; 1599 int n; 1600 1601 db_show_drivers(); 1602 db_printf("\n"); 1603 1604 db_printf("%4s %5s %4s %4s %8s %4s %8s\n", 1605 "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback"); 1606 1607 for (n = 0; n < ncpus; ++n) { 1608 tdinfo = &tdinfo_array[n]; 1609 1610 TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) { 1611 db_printf("%4u %5u %4u %4u %08x %4u %8p\n" 1612 , krp->krp_op 1613 , krp->krp_status 1614 , krp->krp_iparams, krp->krp_oparams 1615 , krp->krp_crid, krp->krp_hid 1616 , krp->krp_callback 1617 ); 1618 } 1619 } 1620 if (!TAILQ_EMPTY(&crp_ret_q)) { 1621 db_printf("%4s %5s %8s %4s %8s\n", 1622 "Op", "Status", "CRID", "HID", "Callback"); 1623 TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) { 1624 db_printf("%4u %5u %08x %4u %8p\n" 1625 , krp->krp_op 1626 , krp->krp_status 1627 , krp->krp_crid, krp->krp_hid 1628 , krp->krp_callback 1629 ); 1630 } 1631 } 1632 } 1633 #endif 1634 1635 int crypto_modevent(module_t mod, int type, void *unused); 1636 1637 /* 1638 * Initialization code, both for static and dynamic loading. 1639 * Note this is not invoked with the usual MODULE_DECLARE 1640 * mechanism but instead is listed as a dependency by the 1641 * cryptosoft driver. This guarantees proper ordering of 1642 * calls on module load/unload. 1643 */ 1644 int 1645 crypto_modevent(module_t mod, int type, void *unused) 1646 { 1647 int error = EINVAL; 1648 1649 switch (type) { 1650 case MOD_LOAD: 1651 error = crypto_init(); 1652 if (error == 0 && bootverbose) 1653 kprintf("crypto: <crypto core>\n"); 1654 break; 1655 case MOD_UNLOAD: 1656 /*XXX disallow if active sessions */ 1657 error = 0; 1658 crypto_destroy(); 1659 return 0; 1660 } 1661 return error; 1662 } 1663 MODULE_VERSION(crypto, 1); 1664 MODULE_DEPEND(crypto, zlib, 1, 1, 1); 1665