1 /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.7 2003/06/03 00:09:02 sam Exp $ */ 2 /* $DragonFly: src/sys/opencrypto/crypto.c,v 1.14 2006/12/23 00:27:03 swildner Exp $ */ 3 /* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 4 /* 5 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * 13 * Permission to use, copy, and modify this software with or without fee 14 * is hereby granted, provided that this entire notice is included in 15 * all source code copies of any software which is or includes a copy or 16 * modification of this software. 17 * 18 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 19 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 20 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 21 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 22 * PURPOSE. 23 */ 24 25 #define CRYPTO_TIMING /* enable cryptop timing stuff */ 26 27 #include <sys/param.h> 28 #include <sys/systm.h> 29 #include <sys/eventhandler.h> 30 #include <sys/kernel.h> 31 #include <sys/kthread.h> 32 #include <sys/malloc.h> 33 #include <sys/proc.h> 34 #include <sys/sysctl.h> 35 #include <sys/interrupt.h> 36 #include <sys/thread2.h> 37 38 #include <vm/vm_zone.h> 39 #include <opencrypto/cryptodev.h> 40 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 41 42 #define SESID2HID(sid) (((sid) >> 32) & 0xffffffff) 43 44 /* 45 * Crypto drivers register themselves by allocating a slot in the 46 * crypto_drivers table with crypto_get_driverid() and then registering 47 * each algorithm they support with crypto_register() and crypto_kregister(). 48 */ 49 static struct cryptocap *crypto_drivers = NULL; 50 static int crypto_drivers_num = 0; 51 52 /* 53 * There are two queues for crypto requests; one for symmetric (e.g. 54 * cipher) operations and one for asymmetric (e.g. MOD) operations. 55 * See below for how synchronization is handled. 56 */ 57 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 58 static TAILQ_HEAD(,cryptkop) crp_kq; 59 60 /* 61 * There are two queues for processing completed crypto requests; one 62 * for the symmetric and one for the asymmetric ops. We only need one 63 * but have two to avoid type futzing (cryptop vs. cryptkop). See below 64 * for how synchronization is handled. 65 */ 66 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 67 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 68 69 /* 70 * Crypto op and desciptor data structures are allocated 71 * from separate private zones. 72 */ 73 static vm_zone_t cryptop_zone; 74 static vm_zone_t cryptodesc_zone; 75 76 int crypto_usercrypto = 1; /* userland may open /dev/crypto */ 77 SYSCTL_INT(_kern, OID_AUTO, usercrypto, CTLFLAG_RW, 78 &crypto_usercrypto, 0, 79 "Enable/disable user-mode access to crypto support"); 80 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 81 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 82 &crypto_userasymcrypto, 0, 83 "Enable/disable user-mode access to asymmetric crypto support"); 84 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 85 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 86 &crypto_devallowsoft, 0, 87 "Enable/disable use of software asym crypto support"); 88 89 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 90 91 /* 92 * Synchronization: read carefully, this is non-trivial. 93 * 94 * Crypto requests are submitted via crypto_dispatch. No critical 95 * section or lock/interlock guarentees are made on entry. 96 * 97 * Requests are typically passed on the driver directly, but they 98 * may also be queued for processing by a software interrupt thread, 99 * cryptointr, that runs in a critical section. This thread dispatches 100 * the requests to crypto drivers (h/w or s/w) who call crypto_done 101 * when a request is complete. Hardware crypto drivers are assumed 102 * to register their IRQ's as network devices so their interrupt handlers 103 * and subsequent "done callbacks" happen at appropriate protection levels. 104 * 105 * Completed crypto ops are queued for a separate kernel thread that 106 * handles the callbacks with no critical section or lock/interlock 107 * guarentees. This decoupling insures the crypto driver interrupt service 108 * routine is not delayed while the callback takes place and that callbacks 109 * are delivered after a context switch (as opposed to a software interrupt 110 * that clients must block). 111 * 112 * This scheme is not intended for SMP machines. 113 */ 114 static inthand2_t cryptointr; 115 static void cryptoret(void); /* kernel thread for callbacks*/ 116 static struct thread *cryptothread; 117 static void crypto_destroy(void); 118 static int crypto_invoke(struct cryptop *crp, int hint); 119 static int crypto_kinvoke(struct cryptkop *krp, int hint); 120 121 static struct cryptostats cryptostats; 122 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 123 cryptostats, "Crypto system statistics"); 124 125 #ifdef CRYPTO_TIMING 126 static int crypto_timing = 0; 127 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 128 &crypto_timing, 0, "Enable/disable crypto timing support"); 129 #endif 130 131 static void *crypto_int_id; 132 133 static int 134 crypto_init(void) 135 { 136 int error; 137 138 cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1); 139 cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc), 140 0, 0, 1); 141 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 142 kprintf("crypto_init: cannot setup crypto zones\n"); 143 return ENOMEM; 144 } 145 146 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 147 crypto_drivers = kmalloc(crypto_drivers_num * 148 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 149 if (crypto_drivers == NULL) { 150 kprintf("crypto_init: cannot malloc driver table\n"); 151 return ENOMEM; 152 } 153 154 TAILQ_INIT(&crp_q); 155 TAILQ_INIT(&crp_kq); 156 157 TAILQ_INIT(&crp_ret_q); 158 TAILQ_INIT(&crp_ret_kq); 159 160 crypto_int_id = register_swi(SWI_CRYPTO, cryptointr, NULL, 161 "swi_crypto", NULL); 162 error = kthread_create((void (*)(void *)) cryptoret, NULL, 163 &cryptothread, "cryptoret"); 164 if (error) { 165 kprintf("crypto_init: cannot start cryptoret thread; error %d", 166 error); 167 crypto_destroy(); 168 } 169 return error; 170 } 171 172 static void 173 crypto_destroy(void) 174 { 175 /* XXX no wait to reclaim zones */ 176 if (crypto_drivers != NULL) 177 kfree(crypto_drivers, M_CRYPTO_DATA); 178 unregister_swi(crypto_int_id); 179 } 180 181 /* 182 * Initialization code, both for static and dynamic loading. 183 */ 184 static int 185 crypto_modevent(module_t mod, int type, void *unused) 186 { 187 int error = EINVAL; 188 189 switch (type) { 190 case MOD_LOAD: 191 error = crypto_init(); 192 if (error == 0 && bootverbose) 193 kprintf("crypto: <crypto core>\n"); 194 break; 195 case MOD_UNLOAD: 196 /*XXX disallow if active sessions */ 197 error = 0; 198 crypto_destroy(); 199 break; 200 } 201 return error; 202 } 203 204 static moduledata_t crypto_mod = { 205 "crypto", 206 crypto_modevent, 207 0 208 }; 209 MODULE_VERSION(crypto, 1); 210 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 211 212 /* 213 * Create a new session. 214 */ 215 int 216 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 217 { 218 struct cryptoini *cr; 219 u_int32_t hid, lid; 220 int err = EINVAL; 221 222 crit_enter(); 223 224 if (crypto_drivers == NULL) 225 goto done; 226 227 /* 228 * The algorithm we use here is pretty stupid; just use the 229 * first driver that supports all the algorithms we need. 230 * 231 * XXX We need more smarts here (in real life too, but that's 232 * XXX another story altogether). 233 */ 234 235 for (hid = 0; hid < crypto_drivers_num; hid++) { 236 /* 237 * If it's not initialized or has remaining sessions 238 * referencing it, skip. 239 */ 240 if (crypto_drivers[hid].cc_newsession == NULL || 241 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)) 242 continue; 243 244 /* Hardware required -- ignore software drivers. */ 245 if (hard > 0 && 246 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)) 247 continue; 248 /* Software required -- ignore hardware drivers. */ 249 if (hard < 0 && 250 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 251 continue; 252 253 /* See if all the algorithms are supported. */ 254 for (cr = cri; cr; cr = cr->cri_next) 255 if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) 256 break; 257 258 if (cr == NULL) { 259 /* Ok, all algorithms are supported. */ 260 261 /* 262 * Can't do everything in one session. 263 * 264 * XXX Fix this. We need to inject a "virtual" session layer right 265 * XXX about here. 266 */ 267 268 /* Call the driver initialization routine. */ 269 lid = hid; /* Pass the driver ID. */ 270 err = crypto_drivers[hid].cc_newsession( 271 crypto_drivers[hid].cc_arg, &lid, cri); 272 if (err == 0) { 273 (*sid) = hid; 274 (*sid) <<= 32; 275 (*sid) |= (lid & 0xffffffff); 276 crypto_drivers[hid].cc_sessions++; 277 } 278 break; 279 } 280 } 281 done: 282 crit_exit(); 283 return err; 284 } 285 286 /* 287 * Delete an existing session (or a reserved session on an unregistered 288 * driver). 289 */ 290 int 291 crypto_freesession(u_int64_t sid) 292 { 293 u_int32_t hid; 294 int err; 295 296 crit_enter(); 297 298 if (crypto_drivers == NULL) { 299 err = EINVAL; 300 goto done; 301 } 302 303 /* Determine two IDs. */ 304 hid = SESID2HID(sid); 305 306 if (hid >= crypto_drivers_num) { 307 err = ENOENT; 308 goto done; 309 } 310 311 if (crypto_drivers[hid].cc_sessions) 312 crypto_drivers[hid].cc_sessions--; 313 314 /* Call the driver cleanup routine, if available. */ 315 if (crypto_drivers[hid].cc_freesession) 316 err = crypto_drivers[hid].cc_freesession( 317 crypto_drivers[hid].cc_arg, sid); 318 else 319 err = 0; 320 321 /* 322 * If this was the last session of a driver marked as invalid, 323 * make the entry available for reuse. 324 */ 325 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 326 crypto_drivers[hid].cc_sessions == 0) 327 bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 328 329 done: 330 crit_exit(); 331 return err; 332 } 333 334 /* 335 * Return an unused driver id. Used by drivers prior to registering 336 * support for the algorithms they handle. 337 */ 338 int32_t 339 crypto_get_driverid(u_int32_t flags) 340 { 341 struct cryptocap *newdrv; 342 int i; 343 344 crit_enter(); 345 for (i = 0; i < crypto_drivers_num; i++) 346 if (crypto_drivers[i].cc_process == NULL && 347 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 348 crypto_drivers[i].cc_sessions == 0) 349 break; 350 351 /* Out of entries, allocate some more. */ 352 if (i == crypto_drivers_num) { 353 /* Be careful about wrap-around. */ 354 if (2 * crypto_drivers_num <= crypto_drivers_num) { 355 crit_exit(); 356 kprintf("crypto: driver count wraparound!\n"); 357 return -1; 358 } 359 360 newdrv = kmalloc(2 * crypto_drivers_num * 361 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 362 if (newdrv == NULL) { 363 crit_exit(); 364 kprintf("crypto: no space to expand driver table!\n"); 365 return -1; 366 } 367 368 bcopy(crypto_drivers, newdrv, 369 crypto_drivers_num * sizeof(struct cryptocap)); 370 371 crypto_drivers_num *= 2; 372 373 kfree(crypto_drivers, M_CRYPTO_DATA); 374 crypto_drivers = newdrv; 375 } 376 377 /* NB: state is zero'd on free */ 378 crypto_drivers[i].cc_sessions = 1; /* Mark */ 379 crypto_drivers[i].cc_flags = flags; 380 if (bootverbose) 381 kprintf("crypto: assign driver %u, flags %u\n", i, flags); 382 383 crit_exit(); 384 385 return i; 386 } 387 388 static struct cryptocap * 389 crypto_checkdriver(u_int32_t hid) 390 { 391 if (crypto_drivers == NULL) 392 return NULL; 393 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 394 } 395 396 /* 397 * Register support for a key-related algorithm. This routine 398 * is called once for each algorithm supported a driver. 399 */ 400 int 401 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 402 int (*kprocess)(void*, struct cryptkop *, int), 403 void *karg) 404 { 405 struct cryptocap *cap; 406 int err; 407 408 crit_enter(); 409 410 cap = crypto_checkdriver(driverid); 411 if (cap != NULL && 412 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 413 /* 414 * XXX Do some performance testing to determine placing. 415 * XXX We probably need an auxiliary data structure that 416 * XXX describes relative performances. 417 */ 418 419 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 420 if (bootverbose) 421 kprintf("crypto: driver %u registers key alg %u flags %u\n" 422 , driverid 423 , kalg 424 , flags 425 ); 426 427 if (cap->cc_kprocess == NULL) { 428 cap->cc_karg = karg; 429 cap->cc_kprocess = kprocess; 430 } 431 err = 0; 432 } else 433 err = EINVAL; 434 435 crit_exit(); 436 return err; 437 } 438 439 /* 440 * Register support for a non-key-related algorithm. This routine 441 * is called once for each such algorithm supported by a driver. 442 */ 443 int 444 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 445 u_int32_t flags, 446 int (*newses)(void*, u_int32_t*, struct cryptoini*), 447 int (*freeses)(void*, u_int64_t), 448 int (*process)(void*, struct cryptop *, int), 449 void *arg) 450 { 451 struct cryptocap *cap; 452 int err; 453 454 crit_enter(); 455 456 cap = crypto_checkdriver(driverid); 457 /* NB: algorithms are in the range [1..max] */ 458 if (cap != NULL && 459 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 460 /* 461 * XXX Do some performance testing to determine placing. 462 * XXX We probably need an auxiliary data structure that 463 * XXX describes relative performances. 464 */ 465 466 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 467 cap->cc_max_op_len[alg] = maxoplen; 468 if (bootverbose) 469 kprintf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 470 , driverid 471 , alg 472 , flags 473 , maxoplen 474 ); 475 476 if (cap->cc_process == NULL) { 477 cap->cc_arg = arg; 478 cap->cc_newsession = newses; 479 cap->cc_process = process; 480 cap->cc_freesession = freeses; 481 cap->cc_sessions = 0; /* Unmark */ 482 } 483 err = 0; 484 } else 485 err = EINVAL; 486 487 crit_exit(); 488 return err; 489 } 490 491 /* 492 * Unregister a crypto driver. If there are pending sessions using it, 493 * leave enough information around so that subsequent calls using those 494 * sessions will correctly detect the driver has been unregistered and 495 * reroute requests. 496 */ 497 int 498 crypto_unregister(u_int32_t driverid, int alg) 499 { 500 int i, err; 501 u_int32_t ses; 502 struct cryptocap *cap; 503 504 crit_enter(); 505 506 cap = crypto_checkdriver(driverid); 507 if (cap != NULL && 508 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 509 cap->cc_alg[alg] != 0) { 510 cap->cc_alg[alg] = 0; 511 cap->cc_max_op_len[alg] = 0; 512 513 /* Was this the last algorithm ? */ 514 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 515 if (cap->cc_alg[i] != 0) 516 break; 517 518 if (i == CRYPTO_ALGORITHM_MAX + 1) { 519 ses = cap->cc_sessions; 520 bzero(cap, sizeof(struct cryptocap)); 521 if (ses != 0) { 522 /* 523 * If there are pending sessions, just mark as invalid. 524 */ 525 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 526 cap->cc_sessions = ses; 527 } 528 } 529 err = 0; 530 } else 531 err = EINVAL; 532 533 crit_exit(); 534 return err; 535 } 536 537 /* 538 * Unregister all algorithms associated with a crypto driver. 539 * If there are pending sessions using it, leave enough information 540 * around so that subsequent calls using those sessions will 541 * correctly detect the driver has been unregistered and reroute 542 * requests. 543 */ 544 int 545 crypto_unregister_all(u_int32_t driverid) 546 { 547 int i, err; 548 u_int32_t ses; 549 struct cryptocap *cap; 550 551 crit_enter(); 552 cap = crypto_checkdriver(driverid); 553 if (cap != NULL) { 554 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 555 cap->cc_alg[i] = 0; 556 cap->cc_max_op_len[i] = 0; 557 } 558 ses = cap->cc_sessions; 559 bzero(cap, sizeof(struct cryptocap)); 560 if (ses != 0) { 561 /* 562 * If there are pending sessions, just mark as invalid. 563 */ 564 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 565 cap->cc_sessions = ses; 566 } 567 err = 0; 568 } else 569 err = EINVAL; 570 571 crit_exit(); 572 return err; 573 } 574 575 /* 576 * Clear blockage on a driver. The what parameter indicates whether 577 * the driver is now ready for cryptop's and/or cryptokop's. 578 */ 579 int 580 crypto_unblock(u_int32_t driverid, int what) 581 { 582 struct cryptocap *cap; 583 int needwakeup, err; 584 585 crit_enter(); 586 cap = crypto_checkdriver(driverid); 587 if (cap != NULL) { 588 needwakeup = 0; 589 if (what & CRYPTO_SYMQ) { 590 needwakeup |= cap->cc_qblocked; 591 cap->cc_qblocked = 0; 592 } 593 if (what & CRYPTO_ASYMQ) { 594 needwakeup |= cap->cc_kqblocked; 595 cap->cc_kqblocked = 0; 596 } 597 if (needwakeup) 598 setsoftcrypto(); 599 err = 0; 600 } else 601 err = EINVAL; 602 crit_exit(); 603 604 return err; 605 } 606 607 /* 608 * Dispatch a crypto request to a driver or queue 609 * it, to be processed by the kernel thread. 610 */ 611 int 612 crypto_dispatch(struct cryptop *crp) 613 { 614 u_int32_t hid = SESID2HID(crp->crp_sid); 615 int result; 616 617 cryptostats.cs_ops++; 618 619 #ifdef CRYPTO_TIMING 620 if (crypto_timing) 621 nanouptime(&crp->crp_tstamp); 622 #endif 623 crit_enter(); 624 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 625 struct cryptocap *cap; 626 /* 627 * Caller marked the request to be processed 628 * immediately; dispatch it directly to the 629 * driver unless the driver is currently blocked. 630 */ 631 cap = crypto_checkdriver(hid); 632 if (cap && !cap->cc_qblocked) { 633 result = crypto_invoke(crp, 0); 634 if (result == ERESTART) { 635 /* 636 * The driver ran out of resources, mark the 637 * driver ``blocked'' for cryptop's and put 638 * the op on the queue. 639 */ 640 crypto_drivers[hid].cc_qblocked = 1; 641 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); 642 cryptostats.cs_blocks++; 643 result = 0; 644 } 645 } else { 646 /* 647 * The driver is blocked, just queue the op until 648 * it unblocks and the swi thread gets kicked. 649 */ 650 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 651 result = 0; 652 } 653 } else { 654 int wasempty = TAILQ_EMPTY(&crp_q); 655 /* 656 * Caller marked the request as ``ok to delay''; 657 * queue it for the swi thread. This is desirable 658 * when the operation is low priority and/or suitable 659 * for batching. 660 */ 661 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 662 if (wasempty) 663 setsoftcrypto(); 664 result = 0; 665 } 666 crit_exit(); 667 668 return result; 669 } 670 671 /* 672 * Add an asymetric crypto request to a queue, 673 * to be processed by the kernel thread. 674 */ 675 int 676 crypto_kdispatch(struct cryptkop *krp) 677 { 678 struct cryptocap *cap; 679 int result; 680 681 cryptostats.cs_kops++; 682 683 crit_enter(); 684 cap = crypto_checkdriver(krp->krp_hid); 685 if (cap && !cap->cc_kqblocked) { 686 result = crypto_kinvoke(krp, 0); 687 if (result == ERESTART) { 688 /* 689 * The driver ran out of resources, mark the 690 * driver ``blocked'' for cryptop's and put 691 * the op on the queue. 692 */ 693 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 694 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 695 cryptostats.cs_kblocks++; 696 } 697 } else { 698 /* 699 * The driver is blocked, just queue the op until 700 * it unblocks and the swi thread gets kicked. 701 */ 702 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 703 result = 0; 704 } 705 crit_exit(); 706 707 return result; 708 } 709 710 /* 711 * Dispatch an assymetric crypto request to the appropriate crypto devices. 712 */ 713 static int 714 crypto_kinvoke(struct cryptkop *krp, int hint) 715 { 716 u_int32_t hid; 717 int error; 718 719 /* Sanity checks. */ 720 if (krp == NULL) 721 return EINVAL; 722 if (krp->krp_callback == NULL) { 723 kfree(krp, M_XDATA); /* XXX allocated in cryptodev */ 724 return EINVAL; 725 } 726 727 for (hid = 0; hid < crypto_drivers_num; hid++) { 728 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 729 !crypto_devallowsoft) 730 continue; 731 if (crypto_drivers[hid].cc_kprocess == NULL) 732 continue; 733 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 734 CRYPTO_ALG_FLAG_SUPPORTED) == 0) 735 continue; 736 break; 737 } 738 if (hid < crypto_drivers_num) { 739 krp->krp_hid = hid; 740 error = crypto_drivers[hid].cc_kprocess( 741 crypto_drivers[hid].cc_karg, krp, hint); 742 } else 743 error = ENODEV; 744 745 if (error) { 746 krp->krp_status = error; 747 crypto_kdone(krp); 748 } 749 return 0; 750 } 751 752 #ifdef CRYPTO_TIMING 753 static void 754 crypto_tstat(struct cryptotstat *ts, struct timespec *tv) 755 { 756 struct timespec now, t; 757 758 nanouptime(&now); 759 t.tv_sec = now.tv_sec - tv->tv_sec; 760 t.tv_nsec = now.tv_nsec - tv->tv_nsec; 761 if (t.tv_nsec < 0) { 762 t.tv_sec--; 763 t.tv_nsec += 1000000000; 764 } 765 timespecadd(&ts->acc, &t); 766 if (timespeccmp(&t, &ts->min, <)) 767 ts->min = t; 768 if (timespeccmp(&t, &ts->max, >)) 769 ts->max = t; 770 ts->count++; 771 772 *tv = now; 773 } 774 #endif 775 776 /* 777 * Dispatch a crypto request to the appropriate crypto devices. 778 */ 779 static int 780 crypto_invoke(struct cryptop *crp, int hint) 781 { 782 u_int32_t hid; 783 int (*process)(void*, struct cryptop *, int); 784 785 #ifdef CRYPTO_TIMING 786 if (crypto_timing) 787 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 788 #endif 789 /* Sanity checks. */ 790 if (crp == NULL) 791 return EINVAL; 792 if (crp->crp_callback == NULL) { 793 crypto_freereq(crp); 794 return EINVAL; 795 } 796 if (crp->crp_desc == NULL) { 797 crp->crp_etype = EINVAL; 798 crypto_done(crp); 799 return 0; 800 } 801 802 hid = SESID2HID(crp->crp_sid); 803 if (hid < crypto_drivers_num) { 804 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 805 crypto_freesession(crp->crp_sid); 806 process = crypto_drivers[hid].cc_process; 807 } else { 808 process = NULL; 809 } 810 811 if (process == NULL) { 812 struct cryptodesc *crd; 813 u_int64_t nid; 814 815 /* 816 * Driver has unregistered; migrate the session and return 817 * an error to the caller so they'll resubmit the op. 818 */ 819 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 820 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 821 822 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 823 crp->crp_sid = nid; 824 825 crp->crp_etype = EAGAIN; 826 crypto_done(crp); 827 return 0; 828 } else { 829 /* 830 * Invoke the driver to process the request. 831 */ 832 return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 833 } 834 } 835 836 /* 837 * Release a set of crypto descriptors. 838 */ 839 void 840 crypto_freereq(struct cryptop *crp) 841 { 842 struct cryptodesc *crd; 843 844 if (crp) { 845 while ((crd = crp->crp_desc) != NULL) { 846 crp->crp_desc = crd->crd_next; 847 zfree(cryptodesc_zone, crd); 848 } 849 zfree(cryptop_zone, crp); 850 } 851 } 852 853 /* 854 * Acquire a set of crypto descriptors. The descriptors are self contained 855 * so no special lock/interlock protection is necessary. 856 */ 857 struct cryptop * 858 crypto_getreq(int num) 859 { 860 struct cryptodesc *crd; 861 struct cryptop *crp; 862 863 crp = zalloc(cryptop_zone); 864 if (crp != NULL) { 865 bzero(crp, sizeof (*crp)); 866 while (num--) { 867 crd = zalloc(cryptodesc_zone); 868 if (crd == NULL) { 869 crypto_freereq(crp); 870 crp = NULL; 871 break; 872 } 873 bzero(crd, sizeof (*crd)); 874 crd->crd_next = crp->crp_desc; 875 crp->crp_desc = crd; 876 } 877 } 878 return crp; 879 } 880 881 /* 882 * Invoke the callback on behalf of the driver. 883 */ 884 void 885 crypto_done(struct cryptop *crp) 886 { 887 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 888 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 889 crp->crp_flags |= CRYPTO_F_DONE; 890 if (crp->crp_etype != 0) 891 cryptostats.cs_errs++; 892 #ifdef CRYPTO_TIMING 893 if (crypto_timing) 894 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 895 #endif 896 if (crp->crp_flags & CRYPTO_F_CBIMM) { 897 /* 898 * Do the callback directly. This is ok when the 899 * callback routine does very little (e.g. the 900 * /dev/crypto callback method just does a wakeup). 901 */ 902 #ifdef CRYPTO_TIMING 903 if (crypto_timing) { 904 /* 905 * NB: We must copy the timestamp before 906 * doing the callback as the cryptop is 907 * likely to be reclaimed. 908 */ 909 struct timespec t = crp->crp_tstamp; 910 crypto_tstat(&cryptostats.cs_cb, &t); 911 crp->crp_callback(crp); 912 crypto_tstat(&cryptostats.cs_finis, &t); 913 } else 914 #endif 915 crp->crp_callback(crp); 916 } else { 917 int wasempty; 918 /* 919 * Normal case; queue the callback for the thread. 920 * 921 * The return queue is manipulated by the swi thread 922 * and, potentially, by crypto device drivers calling 923 * back to mark operations completed. Thus we need 924 * to mask both while manipulating the return queue. 925 */ 926 crit_enter(); 927 wasempty = TAILQ_EMPTY(&crp_ret_q); 928 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 929 if (wasempty) 930 wakeup_one(&crp_ret_q); 931 crit_exit(); 932 } 933 } 934 935 /* 936 * Invoke the callback on behalf of the driver. 937 */ 938 void 939 crypto_kdone(struct cryptkop *krp) 940 { 941 int wasempty; 942 943 if (krp->krp_status != 0) 944 cryptostats.cs_kerrs++; 945 /* 946 * The return queue is manipulated by the swi thread 947 * and, potentially, by crypto device drivers calling 948 * back to mark operations completed. Thus we need 949 * to mask both while manipulating the return queue. 950 */ 951 crit_enter(); 952 wasempty = TAILQ_EMPTY(&crp_ret_kq); 953 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 954 if (wasempty) 955 wakeup_one(&crp_ret_q); 956 crit_exit(); 957 } 958 959 int 960 crypto_getfeat(int *featp) 961 { 962 int hid, kalg, feat = 0; 963 964 crit_enter(); 965 if (!crypto_userasymcrypto) 966 goto out; 967 968 for (hid = 0; hid < crypto_drivers_num; hid++) { 969 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 970 !crypto_devallowsoft) { 971 continue; 972 } 973 if (crypto_drivers[hid].cc_kprocess == NULL) 974 continue; 975 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 976 if ((crypto_drivers[hid].cc_kalg[kalg] & 977 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 978 feat |= 1 << kalg; 979 } 980 out: 981 crit_exit(); 982 *featp = feat; 983 return (0); 984 } 985 986 /* 987 * Software interrupt thread to dispatch crypto requests. 988 */ 989 static void 990 cryptointr(void *dummy, void *frame) 991 { 992 struct cryptop *crp, *submit; 993 struct cryptkop *krp; 994 struct cryptocap *cap; 995 int result, hint; 996 997 cryptostats.cs_intrs++; 998 crit_enter(); 999 do { 1000 /* 1001 * Find the first element in the queue that can be 1002 * processed and look-ahead to see if multiple ops 1003 * are ready for the same driver. 1004 */ 1005 submit = NULL; 1006 hint = 0; 1007 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1008 u_int32_t hid = SESID2HID(crp->crp_sid); 1009 cap = crypto_checkdriver(hid); 1010 if (cap == NULL || cap->cc_process == NULL) { 1011 /* Op needs to be migrated, process it. */ 1012 if (submit == NULL) 1013 submit = crp; 1014 break; 1015 } 1016 if (!cap->cc_qblocked) { 1017 if (submit != NULL) { 1018 /* 1019 * We stop on finding another op, 1020 * regardless whether its for the same 1021 * driver or not. We could keep 1022 * searching the queue but it might be 1023 * better to just use a per-driver 1024 * queue instead. 1025 */ 1026 if (SESID2HID(submit->crp_sid) == hid) 1027 hint = CRYPTO_HINT_MORE; 1028 break; 1029 } else { 1030 submit = crp; 1031 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1032 break; 1033 /* keep scanning for more are q'd */ 1034 } 1035 } 1036 } 1037 if (submit != NULL) { 1038 TAILQ_REMOVE(&crp_q, submit, crp_next); 1039 result = crypto_invoke(submit, hint); 1040 if (result == ERESTART) { 1041 /* 1042 * The driver ran out of resources, mark the 1043 * driver ``blocked'' for cryptop's and put 1044 * the request back in the queue. It would 1045 * best to put the request back where we got 1046 * it but that's hard so for now we put it 1047 * at the front. This should be ok; putting 1048 * it at the end does not work. 1049 */ 1050 /* XXX validate sid again? */ 1051 crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1052 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1053 cryptostats.cs_blocks++; 1054 } 1055 } 1056 1057 /* As above, but for key ops */ 1058 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1059 cap = crypto_checkdriver(krp->krp_hid); 1060 if (cap == NULL || cap->cc_kprocess == NULL) { 1061 /* Op needs to be migrated, process it. */ 1062 break; 1063 } 1064 if (!cap->cc_kqblocked) 1065 break; 1066 } 1067 if (krp != NULL) { 1068 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1069 result = crypto_kinvoke(krp, 0); 1070 if (result == ERESTART) { 1071 /* 1072 * The driver ran out of resources, mark the 1073 * driver ``blocked'' for cryptkop's and put 1074 * the request back in the queue. It would 1075 * best to put the request back where we got 1076 * it but that's hard so for now we put it 1077 * at the front. This should be ok; putting 1078 * it at the end does not work. 1079 */ 1080 /* XXX validate sid again? */ 1081 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1082 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1083 cryptostats.cs_kblocks++; 1084 } 1085 } 1086 } while (submit != NULL || krp != NULL); 1087 crit_exit(); 1088 } 1089 1090 /* 1091 * Kernel thread to do callbacks. 1092 */ 1093 static void 1094 cryptoret(void) 1095 { 1096 struct cryptop *crp; 1097 struct cryptkop *krp; 1098 1099 crit_enter(); 1100 for (;;) { 1101 crp = TAILQ_FIRST(&crp_ret_q); 1102 if (crp != NULL) 1103 TAILQ_REMOVE(&crp_ret_q, crp, crp_next); 1104 krp = TAILQ_FIRST(&crp_ret_kq); 1105 if (krp != NULL) 1106 TAILQ_REMOVE(&crp_ret_kq, krp, krp_next); 1107 1108 if (crp != NULL || krp != NULL) { 1109 crit_exit(); /* lower ipl for callbacks */ 1110 if (crp != NULL) { 1111 #ifdef CRYPTO_TIMING 1112 if (crypto_timing) { 1113 /* 1114 * NB: We must copy the timestamp before 1115 * doing the callback as the cryptop is 1116 * likely to be reclaimed. 1117 */ 1118 struct timespec t = crp->crp_tstamp; 1119 crypto_tstat(&cryptostats.cs_cb, &t); 1120 crp->crp_callback(crp); 1121 crypto_tstat(&cryptostats.cs_finis, &t); 1122 } else 1123 #endif 1124 crp->crp_callback(crp); 1125 } 1126 if (krp != NULL) 1127 krp->krp_callback(krp); 1128 crit_enter(); 1129 } else { 1130 (void) tsleep(&crp_ret_q, 0, "crypto_wait", 0); 1131 cryptostats.cs_rets++; 1132 } 1133 } 1134 /* CODE NOT REACHED (crit_exit() would go here otherwise) */ 1135 } 1136