1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 19 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and 30 * 3.0 are supported. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/endian.h> 40 #include <sys/kernel.h> 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/module.h> 45 #include <sys/mutex.h> 46 #include <sys/random.h> 47 #include <sys/rman.h> 48 49 #include <machine/_inttypes.h> 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform_auth.h> 55 #include "cryptodev_if.h" 56 57 #include <dev/ofw/ofw_bus_subr.h> 58 #include <dev/sec/sec.h> 59 60 static int sec_probe(device_t dev); 61 static int sec_attach(device_t dev); 62 static int sec_detach(device_t dev); 63 static int sec_suspend(device_t dev); 64 static int sec_resume(device_t dev); 65 static int sec_shutdown(device_t dev); 66 static void sec_primary_intr(void *arg); 67 static void sec_secondary_intr(void *arg); 68 static int sec_setup_intr(struct sec_softc *sc, struct resource **ires, 69 void **ihand, int *irid, driver_intr_t handler, const char *iname); 70 static void sec_release_intr(struct sec_softc *sc, struct resource *ires, 71 void *ihand, int irid, const char *iname); 72 static int sec_controller_reset(struct sec_softc *sc); 73 static int sec_channel_reset(struct sec_softc *sc, int channel, int full); 74 static int sec_init(struct sec_softc *sc); 75 static int sec_alloc_dma_mem(struct sec_softc *sc, 76 struct sec_dma_mem *dma_mem, bus_size_t size); 77 static int sec_desc_map_dma(struct sec_softc *sc, 78 struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size, 79 struct sec_desc_map_info *sdmi); 80 static void sec_free_dma_mem(struct sec_dma_mem *dma_mem); 81 static void sec_enqueue(struct sec_softc *sc); 82 static int sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, 83 int channel); 84 static int sec_eu_channel(struct sec_softc *sc, int eu); 85 static int sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 86 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize); 87 static int sec_make_pointer_direct(struct sec_softc *sc, 88 struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize); 89 static int sec_probesession(device_t dev, 90 const struct crypto_session_params *csp); 91 static int sec_newsession(device_t dev, crypto_session_t cses, 92 const struct crypto_session_params *csp); 93 static int sec_process(device_t dev, struct cryptop *crp, int hint); 94 static int sec_build_common_ns_desc(struct sec_softc *sc, 95 struct sec_desc *desc, const struct crypto_session_params *csp, 96 struct cryptop *crp); 97 static int sec_build_common_s_desc(struct sec_softc *sc, 98 struct sec_desc *desc, const struct crypto_session_params *csp, 99 struct cryptop *crp); 100 101 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr); 102 103 /* AESU */ 104 static bool sec_aesu_newsession(const struct crypto_session_params *csp); 105 static int sec_aesu_make_desc(struct sec_softc *sc, 106 const struct crypto_session_params *csp, struct sec_desc *desc, 107 struct cryptop *crp); 108 109 /* MDEU */ 110 static bool sec_mdeu_can_handle(u_int alg); 111 static int sec_mdeu_config(const struct crypto_session_params *csp, 112 u_int *eu, u_int *mode, u_int *hashlen); 113 static bool sec_mdeu_newsession(const struct crypto_session_params *csp); 114 static int sec_mdeu_make_desc(struct sec_softc *sc, 115 const struct crypto_session_params *csp, struct sec_desc *desc, 116 struct cryptop *crp); 117 118 static device_method_t sec_methods[] = { 119 /* Device interface */ 120 DEVMETHOD(device_probe, sec_probe), 121 DEVMETHOD(device_attach, sec_attach), 122 DEVMETHOD(device_detach, sec_detach), 123 124 DEVMETHOD(device_suspend, sec_suspend), 125 DEVMETHOD(device_resume, sec_resume), 126 DEVMETHOD(device_shutdown, sec_shutdown), 127 128 /* Crypto methods */ 129 DEVMETHOD(cryptodev_probesession, sec_probesession), 130 DEVMETHOD(cryptodev_newsession, sec_newsession), 131 DEVMETHOD(cryptodev_process, sec_process), 132 133 DEVMETHOD_END 134 }; 135 static driver_t sec_driver = { 136 "sec", 137 sec_methods, 138 sizeof(struct sec_softc), 139 }; 140 141 static devclass_t sec_devclass; 142 DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0); 143 MODULE_DEPEND(sec, crypto, 1, 1, 1); 144 145 static struct sec_eu_methods sec_eus[] = { 146 { 147 sec_aesu_newsession, 148 sec_aesu_make_desc, 149 }, 150 { 151 sec_mdeu_newsession, 152 sec_mdeu_make_desc, 153 }, 154 { NULL, NULL } 155 }; 156 157 static inline void 158 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op) 159 { 160 161 /* Sync only if dma memory is valid */ 162 if (dma_mem->dma_vaddr != NULL) 163 bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op); 164 } 165 166 static inline void * 167 sec_get_pointer_data(struct sec_desc *desc, u_int n) 168 { 169 170 return (desc->sd_ptr_dmem[n].dma_vaddr); 171 } 172 173 static int 174 sec_probe(device_t dev) 175 { 176 struct sec_softc *sc; 177 uint64_t id; 178 179 if (!ofw_bus_status_okay(dev)) 180 return (ENXIO); 181 182 if (!ofw_bus_is_compatible(dev, "fsl,sec2.0")) 183 return (ENXIO); 184 185 sc = device_get_softc(dev); 186 187 sc->sc_rrid = 0; 188 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 189 RF_ACTIVE); 190 191 if (sc->sc_rres == NULL) 192 return (ENXIO); 193 194 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 195 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 196 197 id = SEC_READ(sc, SEC_ID); 198 199 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 200 201 switch (id) { 202 case SEC_20_ID: 203 device_set_desc(dev, "Freescale Security Engine 2.0"); 204 sc->sc_version = 2; 205 break; 206 case SEC_30_ID: 207 device_set_desc(dev, "Freescale Security Engine 3.0"); 208 sc->sc_version = 3; 209 break; 210 case SEC_31_ID: 211 device_set_desc(dev, "Freescale Security Engine 3.1"); 212 sc->sc_version = 3; 213 break; 214 default: 215 device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id); 216 return (ENXIO); 217 } 218 219 return (0); 220 } 221 222 static int 223 sec_attach(device_t dev) 224 { 225 struct sec_softc *sc; 226 struct sec_hw_lt *lt; 227 int error = 0; 228 int i; 229 230 sc = device_get_softc(dev); 231 sc->sc_dev = dev; 232 sc->sc_blocked = 0; 233 sc->sc_shutdown = 0; 234 235 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session), 236 CRYPTOCAP_F_HARDWARE); 237 if (sc->sc_cid < 0) { 238 device_printf(dev, "could not get crypto driver ID!\n"); 239 return (ENXIO); 240 } 241 242 /* Init locks */ 243 mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev), 244 "SEC Controller lock", MTX_DEF); 245 mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev), 246 "SEC Descriptors lock", MTX_DEF); 247 248 /* Allocate I/O memory for SEC registers */ 249 sc->sc_rrid = 0; 250 sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid, 251 RF_ACTIVE); 252 253 if (sc->sc_rres == NULL) { 254 device_printf(dev, "could not allocate I/O memory!\n"); 255 goto fail1; 256 } 257 258 sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres); 259 sc->sc_bas.bst = rman_get_bustag(sc->sc_rres); 260 261 /* Setup interrupts */ 262 sc->sc_pri_irid = 0; 263 error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand, 264 &sc->sc_pri_irid, sec_primary_intr, "primary"); 265 266 if (error) 267 goto fail2; 268 269 270 if (sc->sc_version == 3) { 271 sc->sc_sec_irid = 1; 272 error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand, 273 &sc->sc_sec_irid, sec_secondary_intr, "secondary"); 274 275 if (error) 276 goto fail3; 277 } 278 279 /* Alloc DMA memory for descriptors and link tables */ 280 error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem), 281 SEC_DESCRIPTORS * sizeof(struct sec_hw_desc)); 282 283 if (error) 284 goto fail4; 285 286 error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem), 287 (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt)); 288 289 if (error) 290 goto fail5; 291 292 /* Fill in descriptors and link tables */ 293 for (i = 0; i < SEC_DESCRIPTORS; i++) { 294 sc->sc_desc[i].sd_desc = 295 (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i; 296 sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr + 297 (i * sizeof(struct sec_hw_desc)); 298 } 299 300 for (i = 0; i < SEC_LT_ENTRIES + 1; i++) { 301 sc->sc_lt[i].sl_lt = 302 (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i; 303 sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr + 304 (i * sizeof(struct sec_hw_lt)); 305 } 306 307 /* Last entry in link table is used to create a circle */ 308 lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt; 309 lt->shl_length = 0; 310 lt->shl_r = 0; 311 lt->shl_n = 1; 312 lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr; 313 314 /* Init descriptor and link table queues pointers */ 315 SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS); 316 SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS); 317 SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS); 318 SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS); 319 SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS); 320 SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS); 321 SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES); 322 SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES); 323 324 /* Create masks for fast checks */ 325 sc->sc_int_error_mask = 0; 326 for (i = 0; i < SEC_CHANNELS; i++) 327 sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i)); 328 329 switch (sc->sc_version) { 330 case 2: 331 sc->sc_channel_idle_mask = 332 (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) | 333 (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) | 334 (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) | 335 (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S); 336 break; 337 case 3: 338 sc->sc_channel_idle_mask = 339 (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) | 340 (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) | 341 (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) | 342 (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S); 343 break; 344 } 345 346 /* Init hardware */ 347 error = sec_init(sc); 348 349 if (error) 350 goto fail6; 351 352 return (0); 353 354 fail6: 355 sec_free_dma_mem(&(sc->sc_lt_dmem)); 356 fail5: 357 sec_free_dma_mem(&(sc->sc_desc_dmem)); 358 fail4: 359 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 360 sc->sc_sec_irid, "secondary"); 361 fail3: 362 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 363 sc->sc_pri_irid, "primary"); 364 fail2: 365 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres); 366 fail1: 367 mtx_destroy(&sc->sc_controller_lock); 368 mtx_destroy(&sc->sc_descriptors_lock); 369 370 return (ENXIO); 371 } 372 373 static int 374 sec_detach(device_t dev) 375 { 376 struct sec_softc *sc = device_get_softc(dev); 377 int i, error, timeout = SEC_TIMEOUT; 378 379 /* Prepare driver to shutdown */ 380 SEC_LOCK(sc, descriptors); 381 sc->sc_shutdown = 1; 382 SEC_UNLOCK(sc, descriptors); 383 384 /* Wait until all queued processing finishes */ 385 while (1) { 386 SEC_LOCK(sc, descriptors); 387 i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc); 388 SEC_UNLOCK(sc, descriptors); 389 390 if (i == 0) 391 break; 392 393 if (timeout < 0) { 394 device_printf(dev, "queue flush timeout!\n"); 395 396 /* DMA can be still active - stop it */ 397 for (i = 0; i < SEC_CHANNELS; i++) 398 sec_channel_reset(sc, i, 1); 399 400 break; 401 } 402 403 timeout -= 1000; 404 DELAY(1000); 405 } 406 407 /* Disable interrupts */ 408 SEC_WRITE(sc, SEC_IER, 0); 409 410 /* Unregister from OCF */ 411 crypto_unregister_all(sc->sc_cid); 412 413 /* Free DMA memory */ 414 for (i = 0; i < SEC_DESCRIPTORS; i++) 415 SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i])); 416 417 sec_free_dma_mem(&(sc->sc_lt_dmem)); 418 sec_free_dma_mem(&(sc->sc_desc_dmem)); 419 420 /* Release interrupts */ 421 sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand, 422 sc->sc_pri_irid, "primary"); 423 sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand, 424 sc->sc_sec_irid, "secondary"); 425 426 /* Release memory */ 427 if (sc->sc_rres) { 428 error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, 429 sc->sc_rres); 430 if (error) 431 device_printf(dev, "bus_release_resource() failed for" 432 " I/O memory, error %d\n", error); 433 434 sc->sc_rres = NULL; 435 } 436 437 mtx_destroy(&sc->sc_controller_lock); 438 mtx_destroy(&sc->sc_descriptors_lock); 439 440 return (0); 441 } 442 443 static int 444 sec_suspend(device_t dev) 445 { 446 447 return (0); 448 } 449 450 static int 451 sec_resume(device_t dev) 452 { 453 454 return (0); 455 } 456 457 static int 458 sec_shutdown(device_t dev) 459 { 460 461 return (0); 462 } 463 464 static int 465 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand, 466 int *irid, driver_intr_t handler, const char *iname) 467 { 468 int error; 469 470 (*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid, 471 RF_ACTIVE); 472 473 if ((*ires) == NULL) { 474 device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname); 475 return (ENXIO); 476 } 477 478 error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET, 479 NULL, handler, sc, ihand); 480 481 if (error) { 482 device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname); 483 if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires)) 484 device_printf(sc->sc_dev, "could not release %s IRQ\n", 485 iname); 486 487 (*ires) = NULL; 488 return (error); 489 } 490 491 return (0); 492 } 493 494 static void 495 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand, 496 int irid, const char *iname) 497 { 498 int error; 499 500 if (ires == NULL) 501 return; 502 503 error = bus_teardown_intr(sc->sc_dev, ires, ihand); 504 if (error) 505 device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s" 506 " IRQ, error %d\n", iname, error); 507 508 error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires); 509 if (error) 510 device_printf(sc->sc_dev, "bus_release_resource() failed for %s" 511 " IRQ, error %d\n", iname, error); 512 } 513 514 static void 515 sec_primary_intr(void *arg) 516 { 517 struct sec_session *ses; 518 struct sec_softc *sc = arg; 519 struct sec_desc *desc; 520 struct cryptop *crp; 521 uint64_t isr; 522 uint8_t hash[HASH_MAX_LEN]; 523 int i, wakeup = 0; 524 525 SEC_LOCK(sc, controller); 526 527 /* Check for errors */ 528 isr = SEC_READ(sc, SEC_ISR); 529 if (isr & sc->sc_int_error_mask) { 530 /* Check each channel for error */ 531 for (i = 0; i < SEC_CHANNELS; i++) { 532 if ((isr & SEC_INT_CH_ERR(i)) == 0) 533 continue; 534 535 device_printf(sc->sc_dev, 536 "I/O error on channel %i!\n", i); 537 538 /* Find and mark problematic descriptor */ 539 desc = sec_find_desc(sc, SEC_READ(sc, 540 SEC_CHAN_CDPR(i))); 541 542 if (desc != NULL) 543 desc->sd_error = EIO; 544 545 /* Do partial channel reset */ 546 sec_channel_reset(sc, i, 0); 547 } 548 } 549 550 /* ACK interrupt */ 551 SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL); 552 553 SEC_UNLOCK(sc, controller); 554 SEC_LOCK(sc, descriptors); 555 556 /* Handle processed descriptors */ 557 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 558 559 while (SEC_QUEUED_DESC_CNT(sc) > 0) { 560 desc = SEC_GET_QUEUED_DESC(sc); 561 562 if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) { 563 SEC_PUT_BACK_QUEUED_DESC(sc); 564 break; 565 } 566 567 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD | 568 BUS_DMASYNC_PREWRITE); 569 570 crp = desc->sd_crp; 571 crp->crp_etype = desc->sd_error; 572 if (crp->crp_etype == 0) { 573 ses = crypto_get_driver_session(crp->crp_session); 574 if (ses->ss_mlen != 0) { 575 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 576 crypto_copydata(crp, 577 crp->crp_digest_start, 578 ses->ss_mlen, hash); 579 if (timingsafe_bcmp( 580 desc->sd_desc->shd_digest, 581 hash, ses->ss_mlen) != 0) 582 crp->crp_etype = EBADMSG; 583 } else 584 crypto_copyback(crp, 585 crp->crp_digest_start, 586 ses->ss_mlen, 587 desc->sd_desc->shd_digest); 588 } 589 } 590 crypto_done(desc->sd_crp); 591 592 SEC_DESC_FREE_POINTERS(desc); 593 SEC_DESC_FREE_LT(sc, desc); 594 SEC_DESC_QUEUED2FREE(sc); 595 } 596 597 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 598 599 if (!sc->sc_shutdown) { 600 wakeup = sc->sc_blocked; 601 sc->sc_blocked = 0; 602 } 603 604 SEC_UNLOCK(sc, descriptors); 605 606 /* Enqueue ready descriptors in hardware */ 607 sec_enqueue(sc); 608 609 if (wakeup) 610 crypto_unblock(sc->sc_cid, wakeup); 611 } 612 613 static void 614 sec_secondary_intr(void *arg) 615 { 616 struct sec_softc *sc = arg; 617 618 device_printf(sc->sc_dev, "spurious secondary interrupt!\n"); 619 sec_primary_intr(arg); 620 } 621 622 static int 623 sec_controller_reset(struct sec_softc *sc) 624 { 625 int timeout = SEC_TIMEOUT; 626 627 /* Reset Controller */ 628 SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR); 629 630 while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) { 631 DELAY(1000); 632 timeout -= 1000; 633 634 if (timeout < 0) { 635 device_printf(sc->sc_dev, "timeout while waiting for " 636 "device reset!\n"); 637 return (ETIMEDOUT); 638 } 639 } 640 641 return (0); 642 } 643 644 static int 645 sec_channel_reset(struct sec_softc *sc, int channel, int full) 646 { 647 int timeout = SEC_TIMEOUT; 648 uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON; 649 uint64_t reg; 650 651 /* Reset Channel */ 652 reg = SEC_READ(sc, SEC_CHAN_CCR(channel)); 653 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit); 654 655 while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) { 656 DELAY(1000); 657 timeout -= 1000; 658 659 if (timeout < 0) { 660 device_printf(sc->sc_dev, "timeout while waiting for " 661 "channel reset!\n"); 662 return (ETIMEDOUT); 663 } 664 } 665 666 if (full) { 667 reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS; 668 669 switch(sc->sc_version) { 670 case 2: 671 reg |= SEC_CHAN_CCR_CDWE; 672 break; 673 case 3: 674 reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN; 675 break; 676 } 677 678 SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg); 679 } 680 681 return (0); 682 } 683 684 static int 685 sec_init(struct sec_softc *sc) 686 { 687 uint64_t reg; 688 int error, i; 689 690 /* Reset controller twice to clear all pending interrupts */ 691 error = sec_controller_reset(sc); 692 if (error) 693 return (error); 694 695 error = sec_controller_reset(sc); 696 if (error) 697 return (error); 698 699 /* Reset channels */ 700 for (i = 0; i < SEC_CHANNELS; i++) { 701 error = sec_channel_reset(sc, i, 1); 702 if (error) 703 return (error); 704 } 705 706 /* Enable Interrupts */ 707 reg = SEC_INT_ITO; 708 for (i = 0; i < SEC_CHANNELS; i++) 709 reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i); 710 711 SEC_WRITE(sc, SEC_IER, reg); 712 713 return (error); 714 } 715 716 static void 717 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 718 { 719 struct sec_dma_mem *dma_mem = arg; 720 721 if (error) 722 return; 723 724 KASSERT(nseg == 1, ("Wrong number of segments, should be 1")); 725 dma_mem->dma_paddr = segs->ds_addr; 726 } 727 728 static void 729 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg, 730 int error) 731 { 732 struct sec_desc_map_info *sdmi = arg; 733 struct sec_softc *sc = sdmi->sdmi_sc; 734 struct sec_lt *lt = NULL; 735 bus_addr_t addr; 736 bus_size_t size; 737 int i; 738 739 SEC_LOCK_ASSERT(sc, descriptors); 740 741 if (error) 742 return; 743 744 for (i = 0; i < nseg; i++) { 745 addr = segs[i].ds_addr; 746 size = segs[i].ds_len; 747 748 /* Skip requested offset */ 749 if (sdmi->sdmi_offset >= size) { 750 sdmi->sdmi_offset -= size; 751 continue; 752 } 753 754 addr += sdmi->sdmi_offset; 755 size -= sdmi->sdmi_offset; 756 sdmi->sdmi_offset = 0; 757 758 /* Do not link more than requested */ 759 if (sdmi->sdmi_size < size) 760 size = sdmi->sdmi_size; 761 762 lt = SEC_ALLOC_LT_ENTRY(sc); 763 lt->sl_lt->shl_length = size; 764 lt->sl_lt->shl_r = 0; 765 lt->sl_lt->shl_n = 0; 766 lt->sl_lt->shl_ptr = addr; 767 768 if (sdmi->sdmi_lt_first == NULL) 769 sdmi->sdmi_lt_first = lt; 770 771 sdmi->sdmi_lt_used += 1; 772 773 if ((sdmi->sdmi_size -= size) == 0) 774 break; 775 } 776 777 sdmi->sdmi_lt_last = lt; 778 } 779 780 static int 781 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 782 bus_size_t size) 783 { 784 int error; 785 786 if (dma_mem->dma_vaddr != NULL) 787 return (EBUSY); 788 789 error = bus_dma_tag_create(NULL, /* parent */ 790 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 791 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 792 BUS_SPACE_MAXADDR, /* highaddr */ 793 NULL, NULL, /* filtfunc, filtfuncarg */ 794 size, 1, /* maxsize, nsegments */ 795 size, 0, /* maxsegsz, flags */ 796 NULL, NULL, /* lockfunc, lockfuncarg */ 797 &(dma_mem->dma_tag)); /* dmat */ 798 799 if (error) { 800 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 801 " %i!\n", error); 802 goto err1; 803 } 804 805 error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr), 806 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map)); 807 808 if (error) { 809 device_printf(sc->sc_dev, "failed to allocate DMA safe" 810 " memory, error %i!\n", error); 811 goto err2; 812 } 813 814 error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map, 815 dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem, 816 BUS_DMA_NOWAIT); 817 818 if (error) { 819 device_printf(sc->sc_dev, "cannot get address of the DMA" 820 " memory, error %i\n", error); 821 goto err3; 822 } 823 824 dma_mem->dma_is_map = 0; 825 return (0); 826 827 err3: 828 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map); 829 err2: 830 bus_dma_tag_destroy(dma_mem->dma_tag); 831 err1: 832 dma_mem->dma_vaddr = NULL; 833 return(error); 834 } 835 836 static int 837 sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, 838 struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi) 839 { 840 int error; 841 842 if (dma_mem->dma_vaddr != NULL) 843 return (EBUSY); 844 845 switch (crp->crp_buf.cb_type) { 846 case CRYPTO_BUF_CONTIG: 847 break; 848 case CRYPTO_BUF_UIO: 849 size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE; 850 break; 851 case CRYPTO_BUF_MBUF: 852 size = m_length(crp->crp_buf.cb_mbuf, NULL); 853 break; 854 default: 855 return (EINVAL); 856 } 857 858 error = bus_dma_tag_create(NULL, /* parent */ 859 SEC_DMA_ALIGNMENT, 0, /* alignment, boundary */ 860 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 861 BUS_SPACE_MAXADDR, /* highaddr */ 862 NULL, NULL, /* filtfunc, filtfuncarg */ 863 size, /* maxsize */ 864 SEC_FREE_LT_CNT(sc), /* nsegments */ 865 SEC_MAX_DMA_BLOCK_SIZE, 0, /* maxsegsz, flags */ 866 NULL, NULL, /* lockfunc, lockfuncarg */ 867 &(dma_mem->dma_tag)); /* dmat */ 868 869 if (error) { 870 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 871 " %i!\n", error); 872 dma_mem->dma_vaddr = NULL; 873 return (error); 874 } 875 876 error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map)); 877 878 if (error) { 879 device_printf(sc->sc_dev, "failed to create DMA map, error %i!" 880 "\n", error); 881 bus_dma_tag_destroy(dma_mem->dma_tag); 882 return (error); 883 } 884 885 error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp, 886 sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT); 887 888 if (error) { 889 device_printf(sc->sc_dev, "cannot get address of the DMA" 890 " memory, error %i!\n", error); 891 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 892 bus_dma_tag_destroy(dma_mem->dma_tag); 893 return (error); 894 } 895 896 dma_mem->dma_is_map = 1; 897 dma_mem->dma_vaddr = crp; 898 899 return (0); 900 } 901 902 static void 903 sec_free_dma_mem(struct sec_dma_mem *dma_mem) 904 { 905 906 /* Check for double free */ 907 if (dma_mem->dma_vaddr == NULL) 908 return; 909 910 bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map); 911 912 if (dma_mem->dma_is_map) 913 bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map); 914 else 915 bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, 916 dma_mem->dma_map); 917 918 bus_dma_tag_destroy(dma_mem->dma_tag); 919 dma_mem->dma_vaddr = NULL; 920 } 921 922 static int 923 sec_eu_channel(struct sec_softc *sc, int eu) 924 { 925 uint64_t reg; 926 int channel = 0; 927 928 SEC_LOCK_ASSERT(sc, controller); 929 930 reg = SEC_READ(sc, SEC_EUASR); 931 932 switch (eu) { 933 case SEC_EU_AFEU: 934 channel = SEC_EUASR_AFEU(reg); 935 break; 936 case SEC_EU_DEU: 937 channel = SEC_EUASR_DEU(reg); 938 break; 939 case SEC_EU_MDEU_A: 940 case SEC_EU_MDEU_B: 941 channel = SEC_EUASR_MDEU(reg); 942 break; 943 case SEC_EU_RNGU: 944 channel = SEC_EUASR_RNGU(reg); 945 break; 946 case SEC_EU_PKEU: 947 channel = SEC_EUASR_PKEU(reg); 948 break; 949 case SEC_EU_AESU: 950 channel = SEC_EUASR_AESU(reg); 951 break; 952 case SEC_EU_KEU: 953 channel = SEC_EUASR_KEU(reg); 954 break; 955 case SEC_EU_CRCU: 956 channel = SEC_EUASR_CRCU(reg); 957 break; 958 } 959 960 return (channel - 1); 961 } 962 963 static int 964 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel) 965 { 966 u_int fflvl = SEC_MAX_FIFO_LEVEL; 967 uint64_t reg; 968 int i; 969 970 SEC_LOCK_ASSERT(sc, controller); 971 972 /* Find free channel if have not got one */ 973 if (channel < 0) { 974 for (i = 0; i < SEC_CHANNELS; i++) { 975 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 976 977 if ((reg & sc->sc_channel_idle_mask) == 0) { 978 channel = i; 979 break; 980 } 981 } 982 } 983 984 /* There is no free channel */ 985 if (channel < 0) 986 return (-1); 987 988 /* Check FIFO level on selected channel */ 989 reg = SEC_READ(sc, SEC_CHAN_CSR(channel)); 990 991 switch(sc->sc_version) { 992 case 2: 993 fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M; 994 break; 995 case 3: 996 fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M; 997 break; 998 } 999 1000 if (fflvl >= SEC_MAX_FIFO_LEVEL) 1001 return (-1); 1002 1003 /* Enqueue descriptor in channel */ 1004 SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr); 1005 1006 return (channel); 1007 } 1008 1009 static void 1010 sec_enqueue(struct sec_softc *sc) 1011 { 1012 struct sec_desc *desc; 1013 int ch0, ch1; 1014 1015 SEC_LOCK(sc, descriptors); 1016 SEC_LOCK(sc, controller); 1017 1018 while (SEC_READY_DESC_CNT(sc) > 0) { 1019 desc = SEC_GET_READY_DESC(sc); 1020 1021 ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0); 1022 ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1); 1023 1024 /* 1025 * Both EU are used by the same channel. 1026 * Enqueue descriptor in channel used by busy EUs. 1027 */ 1028 if (ch0 >= 0 && ch0 == ch1) { 1029 if (sec_enqueue_desc(sc, desc, ch0) >= 0) { 1030 SEC_DESC_READY2QUEUED(sc); 1031 continue; 1032 } 1033 } 1034 1035 /* 1036 * Only one EU is free. 1037 * Enqueue descriptor in channel used by busy EU. 1038 */ 1039 if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) { 1040 if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1) 1041 >= 0) { 1042 SEC_DESC_READY2QUEUED(sc); 1043 continue; 1044 } 1045 } 1046 1047 /* 1048 * Both EU are free. 1049 * Enqueue descriptor in first free channel. 1050 */ 1051 if (ch0 < 0 && ch1 < 0) { 1052 if (sec_enqueue_desc(sc, desc, -1) >= 0) { 1053 SEC_DESC_READY2QUEUED(sc); 1054 continue; 1055 } 1056 } 1057 1058 /* Current descriptor can not be queued at the moment */ 1059 SEC_PUT_BACK_READY_DESC(sc); 1060 break; 1061 } 1062 1063 SEC_UNLOCK(sc, controller); 1064 SEC_UNLOCK(sc, descriptors); 1065 } 1066 1067 static struct sec_desc * 1068 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr) 1069 { 1070 struct sec_desc *desc = NULL; 1071 int i; 1072 1073 SEC_LOCK_ASSERT(sc, descriptors); 1074 1075 for (i = 0; i < SEC_CHANNELS; i++) { 1076 if (sc->sc_desc[i].sd_desc_paddr == paddr) { 1077 desc = &(sc->sc_desc[i]); 1078 break; 1079 } 1080 } 1081 1082 return (desc); 1083 } 1084 1085 static int 1086 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n, 1087 bus_addr_t data, bus_size_t dsize) 1088 { 1089 struct sec_hw_desc_ptr *ptr; 1090 1091 SEC_LOCK_ASSERT(sc, descriptors); 1092 1093 ptr = &(desc->sd_desc->shd_pointer[n]); 1094 ptr->shdp_length = dsize; 1095 ptr->shdp_extent = 0; 1096 ptr->shdp_j = 0; 1097 ptr->shdp_ptr = data; 1098 1099 return (0); 1100 } 1101 1102 static int 1103 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc, 1104 u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize) 1105 { 1106 struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 }; 1107 struct sec_hw_desc_ptr *ptr; 1108 int error; 1109 1110 SEC_LOCK_ASSERT(sc, descriptors); 1111 1112 error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize, 1113 &sdmi); 1114 1115 if (error) 1116 return (error); 1117 1118 sdmi.sdmi_lt_last->sl_lt->shl_r = 1; 1119 desc->sd_lt_used += sdmi.sdmi_lt_used; 1120 1121 ptr = &(desc->sd_desc->shd_pointer[n]); 1122 ptr->shdp_length = dsize; 1123 ptr->shdp_extent = 0; 1124 ptr->shdp_j = 1; 1125 ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr; 1126 1127 return (0); 1128 } 1129 1130 static bool 1131 sec_cipher_supported(const struct crypto_session_params *csp) 1132 { 1133 1134 switch (csp->csp_cipher_alg) { 1135 case CRYPTO_AES_CBC: 1136 /* AESU */ 1137 if (csp->csp_ivlen != AES_BLOCK_LEN) 1138 return (false); 1139 break; 1140 default: 1141 return (false); 1142 } 1143 1144 if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN) 1145 return (false); 1146 1147 return (true); 1148 } 1149 1150 static bool 1151 sec_auth_supported(struct sec_softc *sc, 1152 const struct crypto_session_params *csp) 1153 { 1154 1155 switch (csp->csp_auth_alg) { 1156 case CRYPTO_SHA2_384_HMAC: 1157 case CRYPTO_SHA2_512_HMAC: 1158 if (sc->sc_version < 3) 1159 return (false); 1160 /* FALLTHROUGH */ 1161 case CRYPTO_SHA1_HMAC: 1162 case CRYPTO_SHA2_256_HMAC: 1163 if (csp->csp_auth_klen > SEC_MAX_KEY_LEN) 1164 return (false); 1165 break; 1166 case CRYPTO_SHA1: 1167 break; 1168 default: 1169 return (false); 1170 } 1171 return (true); 1172 } 1173 1174 static int 1175 sec_probesession(device_t dev, const struct crypto_session_params *csp) 1176 { 1177 struct sec_softc *sc = device_get_softc(dev); 1178 1179 if (csp->csp_flags != 0) 1180 return (EINVAL); 1181 switch (csp->csp_mode) { 1182 case CSP_MODE_DIGEST: 1183 if (!sec_auth_supported(sc, csp)) 1184 return (EINVAL); 1185 break; 1186 case CSP_MODE_CIPHER: 1187 if (!sec_cipher_supported(csp)) 1188 return (EINVAL); 1189 break; 1190 case CSP_MODE_ETA: 1191 if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp)) 1192 return (EINVAL); 1193 break; 1194 default: 1195 return (EINVAL); 1196 } 1197 return (CRYPTODEV_PROBE_HARDWARE); 1198 } 1199 1200 static int 1201 sec_newsession(device_t dev, crypto_session_t cses, 1202 const struct crypto_session_params *csp) 1203 { 1204 struct sec_eu_methods *eu = sec_eus; 1205 struct sec_session *ses; 1206 1207 ses = crypto_get_driver_session(cses); 1208 1209 /* Find EU for this session */ 1210 while (eu->sem_make_desc != NULL) { 1211 if (eu->sem_newsession(csp)) 1212 break; 1213 eu++; 1214 } 1215 KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session")); 1216 1217 /* Save cipher key */ 1218 if (csp->csp_cipher_key != NULL) 1219 memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen); 1220 1221 /* Save digest key */ 1222 if (csp->csp_auth_key != NULL) 1223 memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen); 1224 1225 if (csp->csp_auth_alg != 0) { 1226 if (csp->csp_auth_mlen == 0) 1227 ses->ss_mlen = crypto_auth_hash(csp)->hashsize; 1228 else 1229 ses->ss_mlen = csp->csp_auth_mlen; 1230 } 1231 1232 return (0); 1233 } 1234 1235 static int 1236 sec_process(device_t dev, struct cryptop *crp, int hint) 1237 { 1238 struct sec_softc *sc = device_get_softc(dev); 1239 struct sec_desc *desc = NULL; 1240 const struct crypto_session_params *csp; 1241 struct sec_session *ses; 1242 int error = 0; 1243 1244 ses = crypto_get_driver_session(crp->crp_session); 1245 csp = crypto_get_params(crp->crp_session); 1246 1247 /* Check for input length */ 1248 if (crypto_buffer_len(&crp->crp_buf) > SEC_MAX_DMA_BLOCK_SIZE) { 1249 crp->crp_etype = E2BIG; 1250 crypto_done(crp); 1251 return (0); 1252 } 1253 1254 SEC_LOCK(sc, descriptors); 1255 SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1256 1257 /* Block driver if there is no free descriptors or we are going down */ 1258 if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) { 1259 sc->sc_blocked |= CRYPTO_SYMQ; 1260 SEC_UNLOCK(sc, descriptors); 1261 return (ERESTART); 1262 } 1263 1264 /* Prepare descriptor */ 1265 desc = SEC_GET_FREE_DESC(sc); 1266 desc->sd_lt_used = 0; 1267 desc->sd_error = 0; 1268 desc->sd_crp = crp; 1269 1270 if (csp->csp_cipher_alg != 0) 1271 crypto_read_iv(crp, desc->sd_desc->shd_iv); 1272 1273 if (crp->crp_cipher_key != NULL) 1274 memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen); 1275 1276 if (crp->crp_auth_key != NULL) 1277 memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen); 1278 1279 memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen); 1280 memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen); 1281 1282 error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp); 1283 1284 if (error) { 1285 SEC_DESC_FREE_POINTERS(desc); 1286 SEC_DESC_PUT_BACK_LT(sc, desc); 1287 SEC_PUT_BACK_FREE_DESC(sc); 1288 SEC_UNLOCK(sc, descriptors); 1289 crp->crp_etype = error; 1290 crypto_done(crp); 1291 return (0); 1292 } 1293 1294 /* 1295 * Skip DONE interrupt if this is not last request in burst, but only 1296 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE 1297 * signaling on each descriptor. 1298 */ 1299 if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3) 1300 desc->sd_desc->shd_dn = 0; 1301 else 1302 desc->sd_desc->shd_dn = 1; 1303 1304 SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1305 SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD | 1306 BUS_DMASYNC_POSTWRITE); 1307 SEC_DESC_FREE2READY(sc); 1308 SEC_UNLOCK(sc, descriptors); 1309 1310 /* Enqueue ready descriptors in hardware */ 1311 sec_enqueue(sc); 1312 1313 return (0); 1314 } 1315 1316 static int 1317 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc, 1318 const struct crypto_session_params *csp, struct cryptop *crp) 1319 { 1320 struct sec_hw_desc *hd = desc->sd_desc; 1321 int error; 1322 1323 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1324 hd->shd_eu_sel1 = SEC_EU_NONE; 1325 hd->shd_mode1 = 0; 1326 1327 /* Pointer 0: NULL */ 1328 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1329 if (error) 1330 return (error); 1331 1332 /* Pointer 1: IV IN */ 1333 error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr + 1334 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1335 if (error) 1336 return (error); 1337 1338 /* Pointer 2: Cipher Key */ 1339 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1340 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1341 if (error) 1342 return (error); 1343 1344 /* Pointer 3: Data IN */ 1345 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1346 crp->crp_payload_length); 1347 if (error) 1348 return (error); 1349 1350 /* Pointer 4: Data OUT */ 1351 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1352 crp->crp_payload_length); 1353 if (error) 1354 return (error); 1355 1356 /* Pointer 5: IV OUT (Not used: NULL) */ 1357 error = sec_make_pointer_direct(sc, desc, 5, 0, 0); 1358 if (error) 1359 return (error); 1360 1361 /* Pointer 6: NULL */ 1362 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1363 1364 return (error); 1365 } 1366 1367 static int 1368 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc, 1369 const struct crypto_session_params *csp, struct cryptop *crp) 1370 { 1371 struct sec_hw_desc *hd = desc->sd_desc; 1372 u_int eu, mode, hashlen; 1373 int error; 1374 1375 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1376 if (error) 1377 return (error); 1378 1379 hd->shd_desc_type = SEC_DT_HMAC_SNOOP; 1380 hd->shd_eu_sel1 = eu; 1381 hd->shd_mode1 = mode; 1382 1383 /* Pointer 0: HMAC Key */ 1384 error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr + 1385 offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen); 1386 if (error) 1387 return (error); 1388 1389 /* Pointer 1: HMAC-Only Data IN */ 1390 error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start, 1391 crp->crp_aad_length); 1392 if (error) 1393 return (error); 1394 1395 /* Pointer 2: Cipher Key */ 1396 error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr + 1397 offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen); 1398 if (error) 1399 return (error); 1400 1401 /* Pointer 3: IV IN */ 1402 error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr + 1403 offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen); 1404 if (error) 1405 return (error); 1406 1407 /* Pointer 4: Data IN */ 1408 error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start, 1409 crp->crp_payload_length); 1410 if (error) 1411 return (error); 1412 1413 /* Pointer 5: Data OUT */ 1414 error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start, 1415 crp->crp_payload_length); 1416 if (error) 1417 return (error); 1418 1419 /* Pointer 6: HMAC OUT */ 1420 error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr + 1421 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1422 1423 return (error); 1424 } 1425 1426 /* AESU */ 1427 1428 static bool 1429 sec_aesu_newsession(const struct crypto_session_params *csp) 1430 { 1431 1432 return (csp->csp_cipher_alg == CRYPTO_AES_CBC); 1433 } 1434 1435 static int 1436 sec_aesu_make_desc(struct sec_softc *sc, 1437 const struct crypto_session_params *csp, struct sec_desc *desc, 1438 struct cryptop *crp) 1439 { 1440 struct sec_hw_desc *hd = desc->sd_desc; 1441 int error; 1442 1443 hd->shd_eu_sel0 = SEC_EU_AESU; 1444 hd->shd_mode0 = SEC_AESU_MODE_CBC; 1445 1446 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1447 hd->shd_mode0 |= SEC_AESU_MODE_ED; 1448 hd->shd_dir = 0; 1449 } else 1450 hd->shd_dir = 1; 1451 1452 if (csp->csp_mode == CSP_MODE_ETA) 1453 error = sec_build_common_s_desc(sc, desc, csp, crp); 1454 else 1455 error = sec_build_common_ns_desc(sc, desc, csp, crp); 1456 1457 return (error); 1458 } 1459 1460 /* MDEU */ 1461 1462 static bool 1463 sec_mdeu_can_handle(u_int alg) 1464 { 1465 switch (alg) { 1466 case CRYPTO_SHA1: 1467 case CRYPTO_SHA1_HMAC: 1468 case CRYPTO_SHA2_256_HMAC: 1469 case CRYPTO_SHA2_384_HMAC: 1470 case CRYPTO_SHA2_512_HMAC: 1471 return (true); 1472 default: 1473 return (false); 1474 } 1475 } 1476 1477 static int 1478 sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode, 1479 u_int *hashlen) 1480 { 1481 1482 *mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT; 1483 *eu = SEC_EU_NONE; 1484 1485 switch (csp->csp_auth_alg) { 1486 case CRYPTO_SHA1_HMAC: 1487 *mode |= SEC_MDEU_MODE_HMAC; 1488 /* FALLTHROUGH */ 1489 case CRYPTO_SHA1: 1490 *eu = SEC_EU_MDEU_A; 1491 *mode |= SEC_MDEU_MODE_SHA1; 1492 *hashlen = SHA1_HASH_LEN; 1493 break; 1494 case CRYPTO_SHA2_256_HMAC: 1495 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256; 1496 *eu = SEC_EU_MDEU_A; 1497 break; 1498 case CRYPTO_SHA2_384_HMAC: 1499 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384; 1500 *eu = SEC_EU_MDEU_B; 1501 break; 1502 case CRYPTO_SHA2_512_HMAC: 1503 *mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512; 1504 *eu = SEC_EU_MDEU_B; 1505 break; 1506 default: 1507 return (EINVAL); 1508 } 1509 1510 if (*mode & SEC_MDEU_MODE_HMAC) 1511 *hashlen = SEC_HMAC_HASH_LEN; 1512 1513 return (0); 1514 } 1515 1516 static bool 1517 sec_mdeu_newsession(const struct crypto_session_params *csp) 1518 { 1519 1520 return (sec_mdeu_can_handle(csp->csp_auth_alg)); 1521 } 1522 1523 static int 1524 sec_mdeu_make_desc(struct sec_softc *sc, 1525 const struct crypto_session_params *csp, 1526 struct sec_desc *desc, struct cryptop *crp) 1527 { 1528 struct sec_hw_desc *hd = desc->sd_desc; 1529 u_int eu, mode, hashlen; 1530 int error; 1531 1532 error = sec_mdeu_config(csp, &eu, &mode, &hashlen); 1533 if (error) 1534 return (error); 1535 1536 hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP; 1537 hd->shd_eu_sel0 = eu; 1538 hd->shd_mode0 = mode; 1539 hd->shd_eu_sel1 = SEC_EU_NONE; 1540 hd->shd_mode1 = 0; 1541 1542 /* Pointer 0: NULL */ 1543 error = sec_make_pointer_direct(sc, desc, 0, 0, 0); 1544 if (error) 1545 return (error); 1546 1547 /* Pointer 1: Context In (Not used: NULL) */ 1548 error = sec_make_pointer_direct(sc, desc, 1, 0, 0); 1549 if (error) 1550 return (error); 1551 1552 /* Pointer 2: HMAC Key (or NULL, depending on digest type) */ 1553 if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC) 1554 error = sec_make_pointer_direct(sc, desc, 2, 1555 desc->sd_desc_paddr + offsetof(struct sec_hw_desc, 1556 shd_mkey), csp->csp_auth_klen); 1557 else 1558 error = sec_make_pointer_direct(sc, desc, 2, 0, 0); 1559 1560 if (error) 1561 return (error); 1562 1563 /* Pointer 3: Input Data */ 1564 error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start, 1565 crp->crp_payload_length); 1566 if (error) 1567 return (error); 1568 1569 /* Pointer 4: NULL */ 1570 error = sec_make_pointer_direct(sc, desc, 4, 0, 0); 1571 if (error) 1572 return (error); 1573 1574 /* Pointer 5: Hash out */ 1575 error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr + 1576 offsetof(struct sec_hw_desc, shd_digest), hashlen); 1577 if (error) 1578 return (error); 1579 1580 /* Pointer 6: NULL */ 1581 error = sec_make_pointer_direct(sc, desc, 6, 0, 0); 1582 1583 return (0); 1584 } 1585