1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2017 Chelsio Communications, Inc. 5 * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org> 6 * All rights reserved. 7 * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org> 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_ddb.h" 35 36 #include <sys/types.h> 37 #include <sys/bus.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/module.h> 43 #include <sys/random.h> 44 #include <sys/sglist.h> 45 #include <sys/sysctl.h> 46 47 #ifdef DDB 48 #include <ddb/ddb.h> 49 #endif 50 51 #include <dev/pci/pcivar.h> 52 53 #include <dev/random/randomdev.h> 54 55 #include <opencrypto/cryptodev.h> 56 #include <opencrypto/xform.h> 57 58 #include "cryptodev_if.h" 59 60 #include "ccp.h" 61 #include "ccp_hardware.h" 62 63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto"); 64 65 /* 66 * Need a global softc available for garbage random_source API, which lacks any 67 * context pointer. It's also handy for debugging. 68 */ 69 struct ccp_softc *g_ccp_softc; 70 71 bool g_debug_print = false; 72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0, 73 "Set to enable debugging log messages"); 74 75 static struct pciid { 76 uint32_t devid; 77 const char *desc; 78 } ccp_ids[] = { 79 { 0x14561022, "AMD CCP-5a" }, 80 { 0x14681022, "AMD CCP-5b" }, 81 }; 82 83 static struct random_source random_ccp = { 84 .rs_ident = "AMD CCP TRNG", 85 .rs_source = RANDOM_PURE_CCP, 86 .rs_read = random_ccp_read, 87 }; 88 89 /* 90 * ccp_populate_sglist() generates a scatter/gather list that covers the entire 91 * crypto operation buffer. 92 */ 93 static int 94 ccp_populate_sglist(struct sglist *sg, struct cryptop *crp) 95 { 96 int error; 97 98 sglist_reset(sg); 99 if (crp->crp_flags & CRYPTO_F_IMBUF) 100 error = sglist_append_mbuf(sg, crp->crp_mbuf); 101 else if (crp->crp_flags & CRYPTO_F_IOV) 102 error = sglist_append_uio(sg, crp->crp_uio); 103 else 104 error = sglist_append(sg, crp->crp_buf, crp->crp_ilen); 105 return (error); 106 } 107 108 /* 109 * Handle a GCM request with an empty payload by performing the 110 * operation in software. Derived from swcr_authenc(). 111 */ 112 static void 113 ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp, 114 struct cryptodesc *crda, struct cryptodesc *crde) 115 { 116 struct aes_gmac_ctx gmac_ctx; 117 char block[GMAC_BLOCK_LEN]; 118 char digest[GMAC_DIGEST_LEN]; 119 char iv[AES_BLOCK_LEN]; 120 int i, len; 121 122 /* 123 * This assumes a 12-byte IV from the crp. See longer comment 124 * above in ccp_gcm() for more details. 125 */ 126 if (crde->crd_flags & CRD_F_ENCRYPT) { 127 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 128 memcpy(iv, crde->crd_iv, 12); 129 else 130 arc4rand(iv, 12, 0); 131 if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0) 132 crypto_copyback(crp->crp_flags, crp->crp_buf, 133 crde->crd_inject, 12, iv); 134 } else { 135 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 136 memcpy(iv, crde->crd_iv, 12); 137 else 138 crypto_copydata(crp->crp_flags, crp->crp_buf, 139 crde->crd_inject, 12, iv); 140 } 141 *(uint32_t *)&iv[12] = htobe32(1); 142 143 /* Initialize the MAC. */ 144 AES_GMAC_Init(&gmac_ctx); 145 AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len); 146 AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv)); 147 148 /* MAC the AAD. */ 149 for (i = 0; i < crda->crd_len; i += sizeof(block)) { 150 len = imin(crda->crd_len - i, sizeof(block)); 151 crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip + 152 i, len, block); 153 bzero(block + len, sizeof(block) - len); 154 AES_GMAC_Update(&gmac_ctx, block, sizeof(block)); 155 } 156 157 /* Length block. */ 158 bzero(block, sizeof(block)); 159 ((uint32_t *)block)[1] = htobe32(crda->crd_len * 8); 160 AES_GMAC_Update(&gmac_ctx, block, sizeof(block)); 161 AES_GMAC_Final(digest, &gmac_ctx); 162 163 if (crde->crd_flags & CRD_F_ENCRYPT) { 164 crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject, 165 sizeof(digest), digest); 166 crp->crp_etype = 0; 167 } else { 168 char digest2[GMAC_DIGEST_LEN]; 169 170 crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject, 171 sizeof(digest2), digest2); 172 if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) 173 crp->crp_etype = 0; 174 else 175 crp->crp_etype = EBADMSG; 176 } 177 crypto_done(crp); 178 } 179 180 static int 181 ccp_probe(device_t dev) 182 { 183 struct pciid *ip; 184 uint32_t id; 185 186 id = pci_get_devid(dev); 187 for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) { 188 if (id == ip->devid) { 189 device_set_desc(dev, ip->desc); 190 return (0); 191 } 192 } 193 return (ENXIO); 194 } 195 196 static void 197 ccp_initialize_queues(struct ccp_softc *sc) 198 { 199 struct ccp_queue *qp; 200 size_t i; 201 202 for (i = 0; i < nitems(sc->queues); i++) { 203 qp = &sc->queues[i]; 204 205 qp->cq_softc = sc; 206 qp->cq_qindex = i; 207 mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF); 208 /* XXX - arbitrarily chosen sizes */ 209 qp->cq_sg_crp = sglist_alloc(32, M_WAITOK); 210 /* Two more SGEs than sg_crp to accommodate ipad. */ 211 qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK); 212 qp->cq_sg_dst = sglist_alloc(2, M_WAITOK); 213 } 214 } 215 216 static void 217 ccp_free_queues(struct ccp_softc *sc) 218 { 219 struct ccp_queue *qp; 220 size_t i; 221 222 for (i = 0; i < nitems(sc->queues); i++) { 223 qp = &sc->queues[i]; 224 225 mtx_destroy(&qp->cq_lock); 226 sglist_free(qp->cq_sg_crp); 227 sglist_free(qp->cq_sg_ulptx); 228 sglist_free(qp->cq_sg_dst); 229 } 230 } 231 232 static int 233 ccp_attach(device_t dev) 234 { 235 struct ccp_softc *sc; 236 int error; 237 238 sc = device_get_softc(dev); 239 sc->dev = dev; 240 241 sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session), 242 CRYPTOCAP_F_HARDWARE); 243 if (sc->cid < 0) { 244 device_printf(dev, "could not get crypto driver id\n"); 245 return (ENXIO); 246 } 247 248 error = ccp_hw_attach(dev); 249 if (error != 0) 250 return (error); 251 252 mtx_init(&sc->lock, "ccp", NULL, MTX_DEF); 253 254 ccp_initialize_queues(sc); 255 256 if (g_ccp_softc == NULL) { 257 g_ccp_softc = sc; 258 if ((sc->hw_features & VERSION_CAP_TRNG) != 0) 259 random_source_register(&random_ccp); 260 } 261 262 if ((sc->hw_features & VERSION_CAP_AES) != 0) { 263 crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0); 264 crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0); 265 crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0); 266 crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0); 267 crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0); 268 crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0); 269 crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0); 270 } 271 if ((sc->hw_features & VERSION_CAP_SHA) != 0) { 272 crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0); 273 crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0); 274 crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0); 275 crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0); 276 } 277 278 return (0); 279 } 280 281 static int 282 ccp_detach(device_t dev) 283 { 284 struct ccp_softc *sc; 285 286 sc = device_get_softc(dev); 287 288 mtx_lock(&sc->lock); 289 sc->detaching = true; 290 mtx_unlock(&sc->lock); 291 292 crypto_unregister_all(sc->cid); 293 if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0) 294 random_source_deregister(&random_ccp); 295 296 ccp_hw_detach(dev); 297 ccp_free_queues(sc); 298 299 if (g_ccp_softc == sc) 300 g_ccp_softc = NULL; 301 302 mtx_destroy(&sc->lock); 303 return (0); 304 } 305 306 static void 307 ccp_init_hmac_digest(struct ccp_session *s, int cri_alg, char *key, 308 int klen) 309 { 310 union authctx auth_ctx; 311 struct auth_hash *axf; 312 u_int i; 313 314 /* 315 * If the key is larger than the block size, use the digest of 316 * the key as the key instead. 317 */ 318 axf = s->hmac.auth_hash; 319 klen /= 8; 320 if (klen > axf->blocksize) { 321 axf->Init(&auth_ctx); 322 axf->Update(&auth_ctx, key, klen); 323 axf->Final(s->hmac.ipad, &auth_ctx); 324 explicit_bzero(&auth_ctx, sizeof(auth_ctx)); 325 klen = axf->hashsize; 326 } else 327 memcpy(s->hmac.ipad, key, klen); 328 329 memset(s->hmac.ipad + klen, 0, axf->blocksize - klen); 330 memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize); 331 332 for (i = 0; i < axf->blocksize; i++) { 333 s->hmac.ipad[i] ^= HMAC_IPAD_VAL; 334 s->hmac.opad[i] ^= HMAC_OPAD_VAL; 335 } 336 } 337 338 static int 339 ccp_aes_check_keylen(int alg, int klen) 340 { 341 342 switch (klen) { 343 case 128: 344 case 192: 345 if (alg == CRYPTO_AES_XTS) 346 return (EINVAL); 347 break; 348 case 256: 349 break; 350 case 512: 351 if (alg != CRYPTO_AES_XTS) 352 return (EINVAL); 353 break; 354 default: 355 return (EINVAL); 356 } 357 return (0); 358 } 359 360 static void 361 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen) 362 { 363 unsigned kbits; 364 365 if (alg == CRYPTO_AES_XTS) 366 kbits = klen / 2; 367 else 368 kbits = klen; 369 370 switch (kbits) { 371 case 128: 372 s->blkcipher.cipher_type = CCP_AES_TYPE_128; 373 break; 374 case 192: 375 s->blkcipher.cipher_type = CCP_AES_TYPE_192; 376 break; 377 case 256: 378 s->blkcipher.cipher_type = CCP_AES_TYPE_256; 379 break; 380 default: 381 panic("should not get here"); 382 } 383 384 s->blkcipher.key_len = klen / 8; 385 memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len); 386 } 387 388 static int 389 ccp_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) 390 { 391 struct ccp_softc *sc; 392 struct ccp_session *s; 393 struct auth_hash *auth_hash; 394 struct cryptoini *c, *hash, *cipher; 395 enum ccp_aes_mode cipher_mode; 396 unsigned auth_mode, iv_len; 397 unsigned partial_digest_len; 398 unsigned q; 399 int error; 400 bool gcm_hash; 401 402 if (cri == NULL) 403 return (EINVAL); 404 405 s = crypto_get_driver_session(cses); 406 407 gcm_hash = false; 408 cipher = NULL; 409 hash = NULL; 410 auth_hash = NULL; 411 /* XXX reconcile auth_mode with use by ccp_sha */ 412 auth_mode = 0; 413 cipher_mode = CCP_AES_MODE_ECB; 414 iv_len = 0; 415 partial_digest_len = 0; 416 for (c = cri; c != NULL; c = c->cri_next) { 417 switch (c->cri_alg) { 418 case CRYPTO_SHA1_HMAC: 419 case CRYPTO_SHA2_256_HMAC: 420 case CRYPTO_SHA2_384_HMAC: 421 case CRYPTO_SHA2_512_HMAC: 422 case CRYPTO_AES_128_NIST_GMAC: 423 case CRYPTO_AES_192_NIST_GMAC: 424 case CRYPTO_AES_256_NIST_GMAC: 425 if (hash) 426 return (EINVAL); 427 hash = c; 428 switch (c->cri_alg) { 429 case CRYPTO_SHA1_HMAC: 430 auth_hash = &auth_hash_hmac_sha1; 431 auth_mode = SHA1; 432 partial_digest_len = SHA1_HASH_LEN; 433 break; 434 case CRYPTO_SHA2_256_HMAC: 435 auth_hash = &auth_hash_hmac_sha2_256; 436 auth_mode = SHA2_256; 437 partial_digest_len = SHA2_256_HASH_LEN; 438 break; 439 case CRYPTO_SHA2_384_HMAC: 440 auth_hash = &auth_hash_hmac_sha2_384; 441 auth_mode = SHA2_384; 442 partial_digest_len = SHA2_512_HASH_LEN; 443 break; 444 case CRYPTO_SHA2_512_HMAC: 445 auth_hash = &auth_hash_hmac_sha2_512; 446 auth_mode = SHA2_512; 447 partial_digest_len = SHA2_512_HASH_LEN; 448 break; 449 case CRYPTO_AES_128_NIST_GMAC: 450 case CRYPTO_AES_192_NIST_GMAC: 451 case CRYPTO_AES_256_NIST_GMAC: 452 gcm_hash = true; 453 #if 0 454 auth_mode = CHCR_SCMD_AUTH_MODE_GHASH; 455 #endif 456 break; 457 } 458 break; 459 case CRYPTO_AES_CBC: 460 case CRYPTO_AES_ICM: 461 case CRYPTO_AES_NIST_GCM_16: 462 case CRYPTO_AES_XTS: 463 if (cipher) 464 return (EINVAL); 465 cipher = c; 466 switch (c->cri_alg) { 467 case CRYPTO_AES_CBC: 468 cipher_mode = CCP_AES_MODE_CBC; 469 iv_len = AES_BLOCK_LEN; 470 break; 471 case CRYPTO_AES_ICM: 472 cipher_mode = CCP_AES_MODE_CTR; 473 iv_len = AES_BLOCK_LEN; 474 break; 475 case CRYPTO_AES_NIST_GCM_16: 476 cipher_mode = CCP_AES_MODE_GCTR; 477 iv_len = AES_GCM_IV_LEN; 478 break; 479 case CRYPTO_AES_XTS: 480 cipher_mode = CCP_AES_MODE_XTS; 481 iv_len = AES_BLOCK_LEN; 482 break; 483 } 484 if (c->cri_key != NULL) { 485 error = ccp_aes_check_keylen(c->cri_alg, 486 c->cri_klen); 487 if (error != 0) 488 return (error); 489 } 490 break; 491 default: 492 return (EINVAL); 493 } 494 } 495 if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR)) 496 return (EINVAL); 497 if (hash == NULL && cipher == NULL) 498 return (EINVAL); 499 if (hash != NULL && hash->cri_key == NULL) 500 return (EINVAL); 501 502 sc = device_get_softc(dev); 503 mtx_lock(&sc->lock); 504 if (sc->detaching) { 505 mtx_unlock(&sc->lock); 506 return (ENXIO); 507 } 508 509 /* Just grab the first usable queue for now. */ 510 for (q = 0; q < nitems(sc->queues); q++) 511 if ((sc->valid_queues & (1 << q)) != 0) 512 break; 513 if (q == nitems(sc->queues)) { 514 mtx_unlock(&sc->lock); 515 return (ENXIO); 516 } 517 s->queue = q; 518 519 if (gcm_hash) 520 s->mode = GCM; 521 else if (hash != NULL && cipher != NULL) 522 s->mode = AUTHENC; 523 else if (hash != NULL) 524 s->mode = HMAC; 525 else { 526 MPASS(cipher != NULL); 527 s->mode = BLKCIPHER; 528 } 529 if (gcm_hash) { 530 if (hash->cri_mlen == 0) 531 s->gmac.hash_len = AES_GMAC_HASH_LEN; 532 else 533 s->gmac.hash_len = hash->cri_mlen; 534 } else if (hash != NULL) { 535 s->hmac.auth_hash = auth_hash; 536 s->hmac.auth_mode = auth_mode; 537 s->hmac.partial_digest_len = partial_digest_len; 538 if (hash->cri_mlen == 0) 539 s->hmac.hash_len = auth_hash->hashsize; 540 else 541 s->hmac.hash_len = hash->cri_mlen; 542 ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key, 543 hash->cri_klen); 544 } 545 if (cipher != NULL) { 546 s->blkcipher.cipher_mode = cipher_mode; 547 s->blkcipher.iv_len = iv_len; 548 if (cipher->cri_key != NULL) 549 ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key, 550 cipher->cri_klen); 551 } 552 553 s->active = true; 554 mtx_unlock(&sc->lock); 555 556 return (0); 557 } 558 559 static void 560 ccp_freesession(device_t dev, crypto_session_t cses) 561 { 562 struct ccp_session *s; 563 564 s = crypto_get_driver_session(cses); 565 566 if (s->pending != 0) 567 device_printf(dev, 568 "session %p freed with %d pending requests\n", s, 569 s->pending); 570 s->active = false; 571 } 572 573 static int 574 ccp_process(device_t dev, struct cryptop *crp, int hint) 575 { 576 struct ccp_softc *sc; 577 struct ccp_queue *qp; 578 struct ccp_session *s; 579 struct cryptodesc *crd, *crda, *crde; 580 int error; 581 bool qpheld; 582 583 qpheld = false; 584 qp = NULL; 585 if (crp == NULL) 586 return (EINVAL); 587 588 crd = crp->crp_desc; 589 s = crypto_get_driver_session(crp->crp_session); 590 sc = device_get_softc(dev); 591 mtx_lock(&sc->lock); 592 qp = &sc->queues[s->queue]; 593 mtx_unlock(&sc->lock); 594 error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT); 595 if (error != 0) 596 goto out; 597 qpheld = true; 598 599 error = ccp_populate_sglist(qp->cq_sg_crp, crp); 600 if (error != 0) 601 goto out; 602 603 switch (s->mode) { 604 case HMAC: 605 if (crd->crd_flags & CRD_F_KEY_EXPLICIT) 606 ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key, 607 crd->crd_klen); 608 error = ccp_hmac(qp, s, crp); 609 break; 610 case BLKCIPHER: 611 if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { 612 error = ccp_aes_check_keylen(crd->crd_alg, 613 crd->crd_klen); 614 if (error != 0) 615 break; 616 ccp_aes_setkey(s, crd->crd_alg, crd->crd_key, 617 crd->crd_klen); 618 } 619 error = ccp_blkcipher(qp, s, crp); 620 break; 621 case AUTHENC: 622 error = 0; 623 switch (crd->crd_alg) { 624 case CRYPTO_AES_CBC: 625 case CRYPTO_AES_ICM: 626 case CRYPTO_AES_XTS: 627 /* Only encrypt-then-authenticate supported. */ 628 crde = crd; 629 crda = crd->crd_next; 630 if (!(crde->crd_flags & CRD_F_ENCRYPT)) { 631 error = EINVAL; 632 break; 633 } 634 s->cipher_first = true; 635 break; 636 default: 637 crda = crd; 638 crde = crd->crd_next; 639 if (crde->crd_flags & CRD_F_ENCRYPT) { 640 error = EINVAL; 641 break; 642 } 643 s->cipher_first = false; 644 break; 645 } 646 if (error != 0) 647 break; 648 if (crda->crd_flags & CRD_F_KEY_EXPLICIT) 649 ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key, 650 crda->crd_klen); 651 if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { 652 error = ccp_aes_check_keylen(crde->crd_alg, 653 crde->crd_klen); 654 if (error != 0) 655 break; 656 ccp_aes_setkey(s, crde->crd_alg, crde->crd_key, 657 crde->crd_klen); 658 } 659 error = ccp_authenc(qp, s, crp, crda, crde); 660 break; 661 case GCM: 662 error = 0; 663 if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) { 664 crde = crd; 665 crda = crd->crd_next; 666 s->cipher_first = true; 667 } else { 668 crda = crd; 669 crde = crd->crd_next; 670 s->cipher_first = false; 671 } 672 if (crde->crd_flags & CRD_F_KEY_EXPLICIT) { 673 error = ccp_aes_check_keylen(crde->crd_alg, 674 crde->crd_klen); 675 if (error != 0) 676 break; 677 ccp_aes_setkey(s, crde->crd_alg, crde->crd_key, 678 crde->crd_klen); 679 } 680 if (crde->crd_len == 0) { 681 mtx_unlock(&qp->cq_lock); 682 ccp_gcm_soft(s, crp, crda, crde); 683 return (0); 684 } 685 error = ccp_gcm(qp, s, crp, crda, crde); 686 break; 687 } 688 689 if (error == 0) 690 s->pending++; 691 692 out: 693 if (qpheld) { 694 if (error != 0) { 695 /* 696 * Squash EAGAIN so callers don't uselessly and 697 * expensively retry if the ring was full. 698 */ 699 if (error == EAGAIN) 700 error = ENOMEM; 701 ccp_queue_abort(qp); 702 } else 703 ccp_queue_release(qp); 704 } 705 706 if (error != 0) { 707 DPRINTF(dev, "%s: early error:%d\n", __func__, error); 708 crp->crp_etype = error; 709 crypto_done(crp); 710 } 711 return (0); 712 } 713 714 static device_method_t ccp_methods[] = { 715 DEVMETHOD(device_probe, ccp_probe), 716 DEVMETHOD(device_attach, ccp_attach), 717 DEVMETHOD(device_detach, ccp_detach), 718 719 DEVMETHOD(cryptodev_newsession, ccp_newsession), 720 DEVMETHOD(cryptodev_freesession, ccp_freesession), 721 DEVMETHOD(cryptodev_process, ccp_process), 722 723 DEVMETHOD_END 724 }; 725 726 static driver_t ccp_driver = { 727 "ccp", 728 ccp_methods, 729 sizeof(struct ccp_softc) 730 }; 731 732 static devclass_t ccp_devclass; 733 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL); 734 MODULE_VERSION(ccp, 1); 735 MODULE_DEPEND(ccp, crypto, 1, 1, 1); 736 MODULE_DEPEND(ccp, random_device, 1, 1, 1); 737 #if 0 /* There are enough known issues that we shouldn't load automatically */ 738 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids, sizeof(ccp_ids[0]), 739 nitems(ccp_ids)); 740 #endif 741 742 static int 743 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags) 744 { 745 struct ccp_softc *sc; 746 747 mtx_assert(&qp->cq_lock, MA_OWNED); 748 sc = qp->cq_softc; 749 750 if (n < 1 || n >= (1 << sc->ring_size_order)) 751 return (EINVAL); 752 753 while (true) { 754 if (ccp_queue_get_ring_space(qp) >= n) 755 return (0); 756 if ((mflags & M_WAITOK) == 0) 757 return (EAGAIN); 758 qp->cq_waiting = true; 759 msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0); 760 } 761 } 762 763 int 764 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags) 765 { 766 int error; 767 768 mtx_lock(&qp->cq_lock); 769 qp->cq_acq_tail = qp->cq_tail; 770 error = ccp_queue_reserve_space(qp, n, mflags); 771 if (error != 0) 772 mtx_unlock(&qp->cq_lock); 773 return (error); 774 } 775 776 void 777 ccp_queue_release(struct ccp_queue *qp) 778 { 779 780 mtx_assert(&qp->cq_lock, MA_OWNED); 781 if (qp->cq_tail != qp->cq_acq_tail) { 782 wmb(); 783 ccp_queue_write_tail(qp); 784 } 785 mtx_unlock(&qp->cq_lock); 786 } 787 788 void 789 ccp_queue_abort(struct ccp_queue *qp) 790 { 791 unsigned i; 792 793 mtx_assert(&qp->cq_lock, MA_OWNED); 794 795 /* Wipe out any descriptors associated with this aborted txn. */ 796 for (i = qp->cq_acq_tail; i != qp->cq_tail; 797 i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) { 798 memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i])); 799 } 800 qp->cq_tail = qp->cq_acq_tail; 801 802 mtx_unlock(&qp->cq_lock); 803 } 804 805 #ifdef DDB 806 #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo) 807 #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object) 808 static void 809 db_show_ccp_sc(struct ccp_softc *sc) 810 { 811 812 db_printf("ccp softc at %p\n", sc); 813 db_printf(" cid: %d\n", (int)sc->cid); 814 815 db_printf(" lock: "); 816 db_show_lock(&sc->lock); 817 818 db_printf(" detaching: %d\n", (int)sc->detaching); 819 db_printf(" ring_size_order: %u\n", sc->ring_size_order); 820 821 db_printf(" hw_version: %d\n", (int)sc->hw_version); 822 db_printf(" hw_features: %b\n", (int)sc->hw_features, 823 "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA" 824 "\11SHA\0103DES\07AES"); 825 826 db_printf(" hw status:\n"); 827 db_ccp_show_hw(sc); 828 } 829 830 static void 831 db_show_ccp_qp(struct ccp_queue *qp) 832 { 833 834 db_printf(" lock: "); 835 db_show_lock(&qp->cq_lock); 836 837 db_printf(" cq_qindex: %u\n", qp->cq_qindex); 838 db_printf(" cq_softc: %p\n", qp->cq_softc); 839 840 db_printf(" head: %u\n", qp->cq_head); 841 db_printf(" tail: %u\n", qp->cq_tail); 842 db_printf(" acq_tail: %u\n", qp->cq_acq_tail); 843 db_printf(" desc_ring: %p\n", qp->desc_ring); 844 db_printf(" completions_ring: %p\n", qp->completions_ring); 845 db_printf(" descriptors (phys): 0x%jx\n", 846 (uintmax_t)qp->desc_ring_bus_addr); 847 848 db_printf(" hw status:\n"); 849 db_ccp_show_queue_hw(qp); 850 } 851 852 DB_SHOW_COMMAND(ccp, db_show_ccp) 853 { 854 struct ccp_softc *sc; 855 unsigned unit, qindex; 856 857 if (!have_addr) 858 goto usage; 859 860 unit = (unsigned)addr; 861 862 sc = devclass_get_softc(ccp_devclass, unit); 863 if (sc == NULL) { 864 db_printf("No such device ccp%u\n", unit); 865 goto usage; 866 } 867 868 if (count == -1) { 869 db_show_ccp_sc(sc); 870 return; 871 } 872 873 qindex = (unsigned)count; 874 if (qindex >= nitems(sc->queues)) { 875 db_printf("No such queue %u\n", qindex); 876 goto usage; 877 } 878 db_show_ccp_qp(&sc->queues[qindex]); 879 return; 880 881 usage: 882 db_printf("usage: show ccp <unit>[,<qindex>]\n"); 883 return; 884 } 885 #endif /* DDB */ 886