1 /* $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.7 2003/06/03 00:09:02 sam Exp $ */ 2 /* $DragonFly: src/sys/opencrypto/crypto.c,v 1.10 2005/10/13 00:02:48 dillon Exp $ */ 3 /* $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $ */ 4 /* 5 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu) 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * 13 * Permission to use, copy, and modify this software with or without fee 14 * is hereby granted, provided that this entire notice is included in 15 * all source code copies of any software which is or includes a copy or 16 * modification of this software. 17 * 18 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 19 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 20 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 21 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 22 * PURPOSE. 23 */ 24 25 #define CRYPTO_TIMING /* enable cryptop timing stuff */ 26 27 #include <sys/param.h> 28 #include <sys/systm.h> 29 #include <sys/eventhandler.h> 30 #include <sys/kernel.h> 31 #include <sys/kthread.h> 32 #include <sys/malloc.h> 33 #include <sys/proc.h> 34 #include <sys/sysctl.h> 35 #include <sys/interrupt.h> 36 #include <sys/thread2.h> 37 #include <machine/ipl.h> 38 39 #include <vm/vm_zone.h> 40 #include <opencrypto/cryptodev.h> 41 #include <opencrypto/xform.h> /* XXX for M_XDATA */ 42 43 #define SESID2HID(sid) (((sid) >> 32) & 0xffffffff) 44 45 /* 46 * Crypto drivers register themselves by allocating a slot in the 47 * crypto_drivers table with crypto_get_driverid() and then registering 48 * each algorithm they support with crypto_register() and crypto_kregister(). 49 */ 50 static struct cryptocap *crypto_drivers = NULL; 51 static int crypto_drivers_num = 0; 52 53 /* 54 * There are two queues for crypto requests; one for symmetric (e.g. 55 * cipher) operations and one for asymmetric (e.g. MOD) operations. 56 * See below for how synchronization is handled. 57 */ 58 static TAILQ_HEAD(,cryptop) crp_q; /* request queues */ 59 static TAILQ_HEAD(,cryptkop) crp_kq; 60 61 /* 62 * There are two queues for processing completed crypto requests; one 63 * for the symmetric and one for the asymmetric ops. We only need one 64 * but have two to avoid type futzing (cryptop vs. cryptkop). See below 65 * for how synchronization is handled. 66 */ 67 static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */ 68 static TAILQ_HEAD(,cryptkop) crp_ret_kq; 69 70 /* 71 * Crypto op and desciptor data structures are allocated 72 * from separate private zones. 73 */ 74 static vm_zone_t cryptop_zone; 75 static vm_zone_t cryptodesc_zone; 76 77 int crypto_usercrypto = 1; /* userland may open /dev/crypto */ 78 SYSCTL_INT(_kern, OID_AUTO, usercrypto, CTLFLAG_RW, 79 &crypto_usercrypto, 0, 80 "Enable/disable user-mode access to crypto support"); 81 int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */ 82 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW, 83 &crypto_userasymcrypto, 0, 84 "Enable/disable user-mode access to asymmetric crypto support"); 85 int crypto_devallowsoft = 0; /* only use hardware crypto for asym */ 86 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW, 87 &crypto_devallowsoft, 0, 88 "Enable/disable use of software asym crypto support"); 89 90 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records"); 91 92 /* 93 * Synchronization: read carefully, this is non-trivial. 94 * 95 * Crypto requests are submitted via crypto_dispatch. No critical 96 * section or lock/interlock guarentees are made on entry. 97 * 98 * Requests are typically passed on the driver directly, but they 99 * may also be queued for processing by a software interrupt thread, 100 * cryptointr, that runs in a critical section. This thread dispatches 101 * the requests to crypto drivers (h/w or s/w) who call crypto_done 102 * when a request is complete. Hardware crypto drivers are assumed 103 * to register their IRQ's as network devices so their interrupt handlers 104 * and subsequent "done callbacks" happen at appropriate protection levels. 105 * 106 * Completed crypto ops are queued for a separate kernel thread that 107 * handles the callbacks with no critical section or lock/interlock 108 * guarentees. This decoupling insures the crypto driver interrupt service 109 * routine is not delayed while the callback takes place and that callbacks 110 * are delivered after a context switch (as opposed to a software interrupt 111 * that clients must block). 112 * 113 * This scheme is not intended for SMP machines. 114 */ 115 static inthand2_t cryptointr; 116 static void cryptoret(void); /* kernel thread for callbacks*/ 117 static struct thread *cryptothread; 118 static void crypto_destroy(void); 119 static int crypto_invoke(struct cryptop *crp, int hint); 120 static int crypto_kinvoke(struct cryptkop *krp, int hint); 121 122 static struct cryptostats cryptostats; 123 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats, 124 cryptostats, "Crypto system statistics"); 125 126 #ifdef CRYPTO_TIMING 127 static int crypto_timing = 0; 128 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW, 129 &crypto_timing, 0, "Enable/disable crypto timing support"); 130 #endif 131 132 static void *crypto_int_id; 133 134 static int 135 crypto_init(void) 136 { 137 int error; 138 139 cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1); 140 cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc), 141 0, 0, 1); 142 if (cryptodesc_zone == NULL || cryptop_zone == NULL) { 143 printf("crypto_init: cannot setup crypto zones\n"); 144 return ENOMEM; 145 } 146 147 crypto_drivers_num = CRYPTO_DRIVERS_INITIAL; 148 crypto_drivers = malloc(crypto_drivers_num * 149 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO); 150 if (crypto_drivers == NULL) { 151 printf("crypto_init: cannot malloc driver table\n"); 152 return ENOMEM; 153 } 154 155 TAILQ_INIT(&crp_q); 156 TAILQ_INIT(&crp_kq); 157 158 TAILQ_INIT(&crp_ret_q); 159 TAILQ_INIT(&crp_ret_kq); 160 161 crypto_int_id = register_swi(SWI_CRYPTO, cryptointr, NULL, 162 "swi_crypto", NULL); 163 error = kthread_create((void (*)(void *)) cryptoret, NULL, 164 &cryptothread, "cryptoret"); 165 if (error) { 166 printf("crypto_init: cannot start cryptoret thread; error %d", 167 error); 168 crypto_destroy(); 169 } 170 return error; 171 } 172 173 static void 174 crypto_destroy(void) 175 { 176 /* XXX no wait to reclaim zones */ 177 if (crypto_drivers != NULL) 178 free(crypto_drivers, M_CRYPTO_DATA); 179 unregister_swi(crypto_int_id); 180 } 181 182 /* 183 * Initialization code, both for static and dynamic loading. 184 */ 185 static int 186 crypto_modevent(module_t mod, int type, void *unused) 187 { 188 int error = EINVAL; 189 190 switch (type) { 191 case MOD_LOAD: 192 error = crypto_init(); 193 if (error == 0 && bootverbose) 194 printf("crypto: <crypto core>\n"); 195 break; 196 case MOD_UNLOAD: 197 /*XXX disallow if active sessions */ 198 error = 0; 199 crypto_destroy(); 200 break; 201 } 202 return error; 203 } 204 205 static moduledata_t crypto_mod = { 206 "crypto", 207 crypto_modevent, 208 0 209 }; 210 MODULE_VERSION(crypto, 1); 211 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST); 212 213 /* 214 * Create a new session. 215 */ 216 int 217 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard) 218 { 219 struct cryptoini *cr; 220 u_int32_t hid, lid; 221 int err = EINVAL; 222 223 crit_enter(); 224 225 if (crypto_drivers == NULL) 226 goto done; 227 228 /* 229 * The algorithm we use here is pretty stupid; just use the 230 * first driver that supports all the algorithms we need. 231 * 232 * XXX We need more smarts here (in real life too, but that's 233 * XXX another story altogether). 234 */ 235 236 for (hid = 0; hid < crypto_drivers_num; hid++) { 237 /* 238 * If it's not initialized or has remaining sessions 239 * referencing it, skip. 240 */ 241 if (crypto_drivers[hid].cc_newsession == NULL || 242 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)) 243 continue; 244 245 /* Hardware required -- ignore software drivers. */ 246 if (hard > 0 && 247 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)) 248 continue; 249 /* Software required -- ignore hardware drivers. */ 250 if (hard < 0 && 251 (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0) 252 continue; 253 254 /* See if all the algorithms are supported. */ 255 for (cr = cri; cr; cr = cr->cri_next) 256 if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0) 257 break; 258 259 if (cr == NULL) { 260 /* Ok, all algorithms are supported. */ 261 262 /* 263 * Can't do everything in one session. 264 * 265 * XXX Fix this. We need to inject a "virtual" session layer right 266 * XXX about here. 267 */ 268 269 /* Call the driver initialization routine. */ 270 lid = hid; /* Pass the driver ID. */ 271 err = crypto_drivers[hid].cc_newsession( 272 crypto_drivers[hid].cc_arg, &lid, cri); 273 if (err == 0) { 274 (*sid) = hid; 275 (*sid) <<= 32; 276 (*sid) |= (lid & 0xffffffff); 277 crypto_drivers[hid].cc_sessions++; 278 } 279 break; 280 } 281 } 282 done: 283 crit_exit(); 284 return err; 285 } 286 287 /* 288 * Delete an existing session (or a reserved session on an unregistered 289 * driver). 290 */ 291 int 292 crypto_freesession(u_int64_t sid) 293 { 294 u_int32_t hid; 295 int err; 296 297 crit_enter(); 298 299 if (crypto_drivers == NULL) { 300 err = EINVAL; 301 goto done; 302 } 303 304 /* Determine two IDs. */ 305 hid = SESID2HID(sid); 306 307 if (hid >= crypto_drivers_num) { 308 err = ENOENT; 309 goto done; 310 } 311 312 if (crypto_drivers[hid].cc_sessions) 313 crypto_drivers[hid].cc_sessions--; 314 315 /* Call the driver cleanup routine, if available. */ 316 if (crypto_drivers[hid].cc_freesession) 317 err = crypto_drivers[hid].cc_freesession( 318 crypto_drivers[hid].cc_arg, sid); 319 else 320 err = 0; 321 322 /* 323 * If this was the last session of a driver marked as invalid, 324 * make the entry available for reuse. 325 */ 326 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) && 327 crypto_drivers[hid].cc_sessions == 0) 328 bzero(&crypto_drivers[hid], sizeof(struct cryptocap)); 329 330 done: 331 crit_exit(); 332 return err; 333 } 334 335 /* 336 * Return an unused driver id. Used by drivers prior to registering 337 * support for the algorithms they handle. 338 */ 339 int32_t 340 crypto_get_driverid(u_int32_t flags) 341 { 342 struct cryptocap *newdrv; 343 int i; 344 345 crit_enter(); 346 for (i = 0; i < crypto_drivers_num; i++) 347 if (crypto_drivers[i].cc_process == NULL && 348 (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 && 349 crypto_drivers[i].cc_sessions == 0) 350 break; 351 352 /* Out of entries, allocate some more. */ 353 if (i == crypto_drivers_num) { 354 /* Be careful about wrap-around. */ 355 if (2 * crypto_drivers_num <= crypto_drivers_num) { 356 crit_exit(); 357 printf("crypto: driver count wraparound!\n"); 358 return -1; 359 } 360 361 newdrv = malloc(2 * crypto_drivers_num * 362 sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO); 363 if (newdrv == NULL) { 364 crit_exit(); 365 printf("crypto: no space to expand driver table!\n"); 366 return -1; 367 } 368 369 bcopy(crypto_drivers, newdrv, 370 crypto_drivers_num * sizeof(struct cryptocap)); 371 372 crypto_drivers_num *= 2; 373 374 free(crypto_drivers, M_CRYPTO_DATA); 375 crypto_drivers = newdrv; 376 } 377 378 /* NB: state is zero'd on free */ 379 crypto_drivers[i].cc_sessions = 1; /* Mark */ 380 crypto_drivers[i].cc_flags = flags; 381 if (bootverbose) 382 printf("crypto: assign driver %u, flags %u\n", i, flags); 383 384 crit_exit(); 385 386 return i; 387 } 388 389 static struct cryptocap * 390 crypto_checkdriver(u_int32_t hid) 391 { 392 if (crypto_drivers == NULL) 393 return NULL; 394 return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]); 395 } 396 397 /* 398 * Register support for a key-related algorithm. This routine 399 * is called once for each algorithm supported a driver. 400 */ 401 int 402 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags, 403 int (*kprocess)(void*, struct cryptkop *, int), 404 void *karg) 405 { 406 struct cryptocap *cap; 407 int err; 408 409 crit_enter(); 410 411 cap = crypto_checkdriver(driverid); 412 if (cap != NULL && 413 (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) { 414 /* 415 * XXX Do some performance testing to determine placing. 416 * XXX We probably need an auxiliary data structure that 417 * XXX describes relative performances. 418 */ 419 420 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 421 if (bootverbose) 422 printf("crypto: driver %u registers key alg %u flags %u\n" 423 , driverid 424 , kalg 425 , flags 426 ); 427 428 if (cap->cc_kprocess == NULL) { 429 cap->cc_karg = karg; 430 cap->cc_kprocess = kprocess; 431 } 432 err = 0; 433 } else 434 err = EINVAL; 435 436 crit_exit(); 437 return err; 438 } 439 440 /* 441 * Register support for a non-key-related algorithm. This routine 442 * is called once for each such algorithm supported by a driver. 443 */ 444 int 445 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen, 446 u_int32_t flags, 447 int (*newses)(void*, u_int32_t*, struct cryptoini*), 448 int (*freeses)(void*, u_int64_t), 449 int (*process)(void*, struct cryptop *, int), 450 void *arg) 451 { 452 struct cryptocap *cap; 453 int err; 454 455 crit_enter(); 456 457 cap = crypto_checkdriver(driverid); 458 /* NB: algorithms are in the range [1..max] */ 459 if (cap != NULL && 460 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) { 461 /* 462 * XXX Do some performance testing to determine placing. 463 * XXX We probably need an auxiliary data structure that 464 * XXX describes relative performances. 465 */ 466 467 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED; 468 cap->cc_max_op_len[alg] = maxoplen; 469 if (bootverbose) 470 printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n" 471 , driverid 472 , alg 473 , flags 474 , maxoplen 475 ); 476 477 if (cap->cc_process == NULL) { 478 cap->cc_arg = arg; 479 cap->cc_newsession = newses; 480 cap->cc_process = process; 481 cap->cc_freesession = freeses; 482 cap->cc_sessions = 0; /* Unmark */ 483 } 484 err = 0; 485 } else 486 err = EINVAL; 487 488 crit_exit(); 489 return err; 490 } 491 492 /* 493 * Unregister a crypto driver. If there are pending sessions using it, 494 * leave enough information around so that subsequent calls using those 495 * sessions will correctly detect the driver has been unregistered and 496 * reroute requests. 497 */ 498 int 499 crypto_unregister(u_int32_t driverid, int alg) 500 { 501 int i, err; 502 u_int32_t ses; 503 struct cryptocap *cap; 504 505 crit_enter(); 506 507 cap = crypto_checkdriver(driverid); 508 if (cap != NULL && 509 (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) && 510 cap->cc_alg[alg] != 0) { 511 cap->cc_alg[alg] = 0; 512 cap->cc_max_op_len[alg] = 0; 513 514 /* Was this the last algorithm ? */ 515 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) 516 if (cap->cc_alg[i] != 0) 517 break; 518 519 if (i == CRYPTO_ALGORITHM_MAX + 1) { 520 ses = cap->cc_sessions; 521 bzero(cap, sizeof(struct cryptocap)); 522 if (ses != 0) { 523 /* 524 * If there are pending sessions, just mark as invalid. 525 */ 526 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 527 cap->cc_sessions = ses; 528 } 529 } 530 err = 0; 531 } else 532 err = EINVAL; 533 534 crit_exit(); 535 return err; 536 } 537 538 /* 539 * Unregister all algorithms associated with a crypto driver. 540 * If there are pending sessions using it, leave enough information 541 * around so that subsequent calls using those sessions will 542 * correctly detect the driver has been unregistered and reroute 543 * requests. 544 */ 545 int 546 crypto_unregister_all(u_int32_t driverid) 547 { 548 int i, err; 549 u_int32_t ses; 550 struct cryptocap *cap; 551 552 crit_enter(); 553 cap = crypto_checkdriver(driverid); 554 if (cap != NULL) { 555 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) { 556 cap->cc_alg[i] = 0; 557 cap->cc_max_op_len[i] = 0; 558 } 559 ses = cap->cc_sessions; 560 bzero(cap, sizeof(struct cryptocap)); 561 if (ses != 0) { 562 /* 563 * If there are pending sessions, just mark as invalid. 564 */ 565 cap->cc_flags |= CRYPTOCAP_F_CLEANUP; 566 cap->cc_sessions = ses; 567 } 568 err = 0; 569 } else 570 err = EINVAL; 571 572 crit_exit(); 573 return err; 574 } 575 576 /* 577 * Clear blockage on a driver. The what parameter indicates whether 578 * the driver is now ready for cryptop's and/or cryptokop's. 579 */ 580 int 581 crypto_unblock(u_int32_t driverid, int what) 582 { 583 struct cryptocap *cap; 584 int needwakeup, err; 585 586 crit_enter(); 587 cap = crypto_checkdriver(driverid); 588 if (cap != NULL) { 589 needwakeup = 0; 590 if (what & CRYPTO_SYMQ) { 591 needwakeup |= cap->cc_qblocked; 592 cap->cc_qblocked = 0; 593 } 594 if (what & CRYPTO_ASYMQ) { 595 needwakeup |= cap->cc_kqblocked; 596 cap->cc_kqblocked = 0; 597 } 598 if (needwakeup) 599 setsoftcrypto(); 600 err = 0; 601 } else 602 err = EINVAL; 603 crit_exit(); 604 605 return err; 606 } 607 608 /* 609 * Dispatch a crypto request to a driver or queue 610 * it, to be processed by the kernel thread. 611 */ 612 int 613 crypto_dispatch(struct cryptop *crp) 614 { 615 u_int32_t hid = SESID2HID(crp->crp_sid); 616 int result; 617 618 cryptostats.cs_ops++; 619 620 #ifdef CRYPTO_TIMING 621 if (crypto_timing) 622 nanouptime(&crp->crp_tstamp); 623 #endif 624 crit_enter(); 625 if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { 626 struct cryptocap *cap; 627 /* 628 * Caller marked the request to be processed 629 * immediately; dispatch it directly to the 630 * driver unless the driver is currently blocked. 631 */ 632 cap = crypto_checkdriver(hid); 633 if (cap && !cap->cc_qblocked) { 634 result = crypto_invoke(crp, 0); 635 if (result == ERESTART) { 636 /* 637 * The driver ran out of resources, mark the 638 * driver ``blocked'' for cryptop's and put 639 * the op on the queue. 640 */ 641 crypto_drivers[hid].cc_qblocked = 1; 642 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); 643 cryptostats.cs_blocks++; 644 result = 0; 645 } 646 } else { 647 /* 648 * The driver is blocked, just queue the op until 649 * it unblocks and the swi thread gets kicked. 650 */ 651 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 652 result = 0; 653 } 654 } else { 655 int wasempty = TAILQ_EMPTY(&crp_q); 656 /* 657 * Caller marked the request as ``ok to delay''; 658 * queue it for the swi thread. This is desirable 659 * when the operation is low priority and/or suitable 660 * for batching. 661 */ 662 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); 663 if (wasempty) 664 setsoftcrypto(); 665 result = 0; 666 } 667 crit_exit(); 668 669 return result; 670 } 671 672 /* 673 * Add an asymetric crypto request to a queue, 674 * to be processed by the kernel thread. 675 */ 676 int 677 crypto_kdispatch(struct cryptkop *krp) 678 { 679 struct cryptocap *cap; 680 int result; 681 682 cryptostats.cs_kops++; 683 684 crit_enter(); 685 cap = crypto_checkdriver(krp->krp_hid); 686 if (cap && !cap->cc_kqblocked) { 687 result = crypto_kinvoke(krp, 0); 688 if (result == ERESTART) { 689 /* 690 * The driver ran out of resources, mark the 691 * driver ``blocked'' for cryptop's and put 692 * the op on the queue. 693 */ 694 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 695 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 696 cryptostats.cs_kblocks++; 697 } 698 } else { 699 /* 700 * The driver is blocked, just queue the op until 701 * it unblocks and the swi thread gets kicked. 702 */ 703 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next); 704 result = 0; 705 } 706 crit_exit(); 707 708 return result; 709 } 710 711 /* 712 * Dispatch an assymetric crypto request to the appropriate crypto devices. 713 */ 714 static int 715 crypto_kinvoke(struct cryptkop *krp, int hint) 716 { 717 u_int32_t hid; 718 int error; 719 720 /* Sanity checks. */ 721 if (krp == NULL) 722 return EINVAL; 723 if (krp->krp_callback == NULL) { 724 free(krp, M_XDATA); /* XXX allocated in cryptodev */ 725 return EINVAL; 726 } 727 728 for (hid = 0; hid < crypto_drivers_num; hid++) { 729 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 730 !crypto_devallowsoft) 731 continue; 732 if (crypto_drivers[hid].cc_kprocess == NULL) 733 continue; 734 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] & 735 CRYPTO_ALG_FLAG_SUPPORTED) == 0) 736 continue; 737 break; 738 } 739 if (hid < crypto_drivers_num) { 740 krp->krp_hid = hid; 741 error = crypto_drivers[hid].cc_kprocess( 742 crypto_drivers[hid].cc_karg, krp, hint); 743 } else 744 error = ENODEV; 745 746 if (error) { 747 krp->krp_status = error; 748 crypto_kdone(krp); 749 } 750 return 0; 751 } 752 753 #ifdef CRYPTO_TIMING 754 static void 755 crypto_tstat(struct cryptotstat *ts, struct timespec *tv) 756 { 757 struct timespec now, t; 758 759 nanouptime(&now); 760 t.tv_sec = now.tv_sec - tv->tv_sec; 761 t.tv_nsec = now.tv_nsec - tv->tv_nsec; 762 if (t.tv_nsec < 0) { 763 t.tv_sec--; 764 t.tv_nsec += 1000000000; 765 } 766 timespecadd(&ts->acc, &t); 767 if (timespeccmp(&t, &ts->min, <)) 768 ts->min = t; 769 if (timespeccmp(&t, &ts->max, >)) 770 ts->max = t; 771 ts->count++; 772 773 *tv = now; 774 } 775 #endif 776 777 /* 778 * Dispatch a crypto request to the appropriate crypto devices. 779 */ 780 static int 781 crypto_invoke(struct cryptop *crp, int hint) 782 { 783 u_int32_t hid; 784 int (*process)(void*, struct cryptop *, int); 785 786 #ifdef CRYPTO_TIMING 787 if (crypto_timing) 788 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); 789 #endif 790 /* Sanity checks. */ 791 if (crp == NULL) 792 return EINVAL; 793 if (crp->crp_callback == NULL) { 794 crypto_freereq(crp); 795 return EINVAL; 796 } 797 if (crp->crp_desc == NULL) { 798 crp->crp_etype = EINVAL; 799 crypto_done(crp); 800 return 0; 801 } 802 803 hid = SESID2HID(crp->crp_sid); 804 if (hid < crypto_drivers_num) { 805 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) 806 crypto_freesession(crp->crp_sid); 807 process = crypto_drivers[hid].cc_process; 808 } else { 809 process = NULL; 810 } 811 812 if (process == NULL) { 813 struct cryptodesc *crd; 814 u_int64_t nid; 815 816 /* 817 * Driver has unregistered; migrate the session and return 818 * an error to the caller so they'll resubmit the op. 819 */ 820 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next) 821 crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI); 822 823 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0) 824 crp->crp_sid = nid; 825 826 crp->crp_etype = EAGAIN; 827 crypto_done(crp); 828 return 0; 829 } else { 830 /* 831 * Invoke the driver to process the request. 832 */ 833 return (*process)(crypto_drivers[hid].cc_arg, crp, hint); 834 } 835 } 836 837 /* 838 * Release a set of crypto descriptors. 839 */ 840 void 841 crypto_freereq(struct cryptop *crp) 842 { 843 struct cryptodesc *crd; 844 845 if (crp) { 846 while ((crd = crp->crp_desc) != NULL) { 847 crp->crp_desc = crd->crd_next; 848 zfree(cryptodesc_zone, crd); 849 } 850 zfree(cryptop_zone, crp); 851 } 852 } 853 854 /* 855 * Acquire a set of crypto descriptors. The descriptors are self contained 856 * so no special lock/interlock protection is necessary. 857 */ 858 struct cryptop * 859 crypto_getreq(int num) 860 { 861 struct cryptodesc *crd; 862 struct cryptop *crp; 863 864 crp = zalloc(cryptop_zone); 865 if (crp != NULL) { 866 bzero(crp, sizeof (*crp)); 867 while (num--) { 868 crd = zalloc(cryptodesc_zone); 869 if (crd == NULL) { 870 crypto_freereq(crp); 871 crp = NULL; 872 break; 873 } 874 bzero(crd, sizeof (*crd)); 875 crd->crd_next = crp->crp_desc; 876 crp->crp_desc = crd; 877 } 878 } 879 return crp; 880 } 881 882 /* 883 * Invoke the callback on behalf of the driver. 884 */ 885 void 886 crypto_done(struct cryptop *crp) 887 { 888 KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0, 889 ("crypto_done: op already done, flags 0x%x", crp->crp_flags)); 890 crp->crp_flags |= CRYPTO_F_DONE; 891 if (crp->crp_etype != 0) 892 cryptostats.cs_errs++; 893 #ifdef CRYPTO_TIMING 894 if (crypto_timing) 895 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); 896 #endif 897 if (crp->crp_flags & CRYPTO_F_CBIMM) { 898 /* 899 * Do the callback directly. This is ok when the 900 * callback routine does very little (e.g. the 901 * /dev/crypto callback method just does a wakeup). 902 */ 903 #ifdef CRYPTO_TIMING 904 if (crypto_timing) { 905 /* 906 * NB: We must copy the timestamp before 907 * doing the callback as the cryptop is 908 * likely to be reclaimed. 909 */ 910 struct timespec t = crp->crp_tstamp; 911 crypto_tstat(&cryptostats.cs_cb, &t); 912 crp->crp_callback(crp); 913 crypto_tstat(&cryptostats.cs_finis, &t); 914 } else 915 #endif 916 crp->crp_callback(crp); 917 } else { 918 int wasempty; 919 /* 920 * Normal case; queue the callback for the thread. 921 * 922 * The return queue is manipulated by the swi thread 923 * and, potentially, by crypto device drivers calling 924 * back to mark operations completed. Thus we need 925 * to mask both while manipulating the return queue. 926 */ 927 crit_enter(); 928 wasempty = TAILQ_EMPTY(&crp_ret_q); 929 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); 930 if (wasempty) 931 wakeup_one(&crp_ret_q); 932 crit_exit(); 933 } 934 } 935 936 /* 937 * Invoke the callback on behalf of the driver. 938 */ 939 void 940 crypto_kdone(struct cryptkop *krp) 941 { 942 int wasempty; 943 944 if (krp->krp_status != 0) 945 cryptostats.cs_kerrs++; 946 /* 947 * The return queue is manipulated by the swi thread 948 * and, potentially, by crypto device drivers calling 949 * back to mark operations completed. Thus we need 950 * to mask both while manipulating the return queue. 951 */ 952 crit_enter(); 953 wasempty = TAILQ_EMPTY(&crp_ret_kq); 954 TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next); 955 if (wasempty) 956 wakeup_one(&crp_ret_q); 957 crit_exit(); 958 } 959 960 int 961 crypto_getfeat(int *featp) 962 { 963 int hid, kalg, feat = 0; 964 965 crit_enter(); 966 if (!crypto_userasymcrypto) 967 goto out; 968 969 for (hid = 0; hid < crypto_drivers_num; hid++) { 970 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) && 971 !crypto_devallowsoft) { 972 continue; 973 } 974 if (crypto_drivers[hid].cc_kprocess == NULL) 975 continue; 976 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++) 977 if ((crypto_drivers[hid].cc_kalg[kalg] & 978 CRYPTO_ALG_FLAG_SUPPORTED) != 0) 979 feat |= 1 << kalg; 980 } 981 out: 982 crit_exit(); 983 *featp = feat; 984 return (0); 985 } 986 987 /* 988 * Software interrupt thread to dispatch crypto requests. 989 */ 990 static void 991 cryptointr(void *dummy, void *frame) 992 { 993 struct cryptop *crp, *submit; 994 struct cryptkop *krp; 995 struct cryptocap *cap; 996 int result, hint; 997 998 cryptostats.cs_intrs++; 999 crit_enter(); 1000 do { 1001 /* 1002 * Find the first element in the queue that can be 1003 * processed and look-ahead to see if multiple ops 1004 * are ready for the same driver. 1005 */ 1006 submit = NULL; 1007 hint = 0; 1008 TAILQ_FOREACH(crp, &crp_q, crp_next) { 1009 u_int32_t hid = SESID2HID(crp->crp_sid); 1010 cap = crypto_checkdriver(hid); 1011 if (cap == NULL || cap->cc_process == NULL) { 1012 /* Op needs to be migrated, process it. */ 1013 if (submit == NULL) 1014 submit = crp; 1015 break; 1016 } 1017 if (!cap->cc_qblocked) { 1018 if (submit != NULL) { 1019 /* 1020 * We stop on finding another op, 1021 * regardless whether its for the same 1022 * driver or not. We could keep 1023 * searching the queue but it might be 1024 * better to just use a per-driver 1025 * queue instead. 1026 */ 1027 if (SESID2HID(submit->crp_sid) == hid) 1028 hint = CRYPTO_HINT_MORE; 1029 break; 1030 } else { 1031 submit = crp; 1032 if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) 1033 break; 1034 /* keep scanning for more are q'd */ 1035 } 1036 } 1037 } 1038 if (submit != NULL) { 1039 TAILQ_REMOVE(&crp_q, submit, crp_next); 1040 result = crypto_invoke(submit, hint); 1041 if (result == ERESTART) { 1042 /* 1043 * The driver ran out of resources, mark the 1044 * driver ``blocked'' for cryptop's and put 1045 * the request back in the queue. It would 1046 * best to put the request back where we got 1047 * it but that's hard so for now we put it 1048 * at the front. This should be ok; putting 1049 * it at the end does not work. 1050 */ 1051 /* XXX validate sid again? */ 1052 crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1; 1053 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next); 1054 cryptostats.cs_blocks++; 1055 } 1056 } 1057 1058 /* As above, but for key ops */ 1059 TAILQ_FOREACH(krp, &crp_kq, krp_next) { 1060 cap = crypto_checkdriver(krp->krp_hid); 1061 if (cap == NULL || cap->cc_kprocess == NULL) { 1062 /* Op needs to be migrated, process it. */ 1063 break; 1064 } 1065 if (!cap->cc_kqblocked) 1066 break; 1067 } 1068 if (krp != NULL) { 1069 TAILQ_REMOVE(&crp_kq, krp, krp_next); 1070 result = crypto_kinvoke(krp, 0); 1071 if (result == ERESTART) { 1072 /* 1073 * The driver ran out of resources, mark the 1074 * driver ``blocked'' for cryptkop's and put 1075 * the request back in the queue. It would 1076 * best to put the request back where we got 1077 * it but that's hard so for now we put it 1078 * at the front. This should be ok; putting 1079 * it at the end does not work. 1080 */ 1081 /* XXX validate sid again? */ 1082 crypto_drivers[krp->krp_hid].cc_kqblocked = 1; 1083 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next); 1084 cryptostats.cs_kblocks++; 1085 } 1086 } 1087 } while (submit != NULL || krp != NULL); 1088 crit_exit(); 1089 } 1090 1091 /* 1092 * Kernel thread to do callbacks. 1093 */ 1094 static void 1095 cryptoret(void) 1096 { 1097 struct cryptop *crp; 1098 struct cryptkop *krp; 1099 1100 crit_enter(); 1101 for (;;) { 1102 crp = TAILQ_FIRST(&crp_ret_q); 1103 if (crp != NULL) 1104 TAILQ_REMOVE(&crp_ret_q, crp, crp_next); 1105 krp = TAILQ_FIRST(&crp_ret_kq); 1106 if (krp != NULL) 1107 TAILQ_REMOVE(&crp_ret_kq, krp, krp_next); 1108 1109 if (crp != NULL || krp != NULL) { 1110 crit_exit(); /* lower ipl for callbacks */ 1111 if (crp != NULL) { 1112 #ifdef CRYPTO_TIMING 1113 if (crypto_timing) { 1114 /* 1115 * NB: We must copy the timestamp before 1116 * doing the callback as the cryptop is 1117 * likely to be reclaimed. 1118 */ 1119 struct timespec t = crp->crp_tstamp; 1120 crypto_tstat(&cryptostats.cs_cb, &t); 1121 crp->crp_callback(crp); 1122 crypto_tstat(&cryptostats.cs_finis, &t); 1123 } else 1124 #endif 1125 crp->crp_callback(crp); 1126 } 1127 if (krp != NULL) 1128 krp->krp_callback(krp); 1129 crit_enter(); 1130 } else { 1131 (void) tsleep(&crp_ret_q, 0, "crypto_wait", 0); 1132 cryptostats.cs_rets++; 1133 } 1134 } 1135 /* CODE NOT REACHED (crit_exit() would go here otherwise) */ 1136 } 1137