1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2009-2011 Semihalf. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * CESA SRAM Memory Map: 31 * 32 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE 33 * | | 34 * | DATA | 35 * | | 36 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0) 37 * | struct cesa_sa_data | 38 * +------------------------+ 39 * | struct cesa_sa_hdesc | 40 * +------------------------+ <= sc->sc_sram_base_va 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/bus.h> 49 #include <sys/endian.h> 50 #include <sys/kernel.h> 51 #include <sys/lock.h> 52 #include <sys/mbuf.h> 53 #include <sys/module.h> 54 #include <sys/mutex.h> 55 #include <sys/rman.h> 56 57 #include <machine/bus.h> 58 #include <machine/intr.h> 59 #include <machine/resource.h> 60 #include <machine/fdt.h> 61 62 #include <dev/fdt/simplebus.h> 63 #include <dev/fdt/fdt_common.h> 64 #include <dev/ofw/ofw_bus.h> 65 #include <dev/ofw/ofw_bus_subr.h> 66 67 #include <sys/md5.h> 68 #include <crypto/sha1.h> 69 #include <crypto/sha2/sha256.h> 70 #include <crypto/rijndael/rijndael.h> 71 #include <opencrypto/cryptodev.h> 72 #include "cryptodev_if.h" 73 74 #include <arm/mv/mvreg.h> 75 #include <arm/mv/mvvar.h> 76 #include "cesa.h" 77 78 static int cesa_probe(device_t); 79 static int cesa_attach(device_t); 80 static int cesa_attach_late(device_t); 81 static int cesa_detach(device_t); 82 static void cesa_intr(void *); 83 static int cesa_newsession(device_t, crypto_session_t, struct cryptoini *); 84 static int cesa_process(device_t, struct cryptop *, int); 85 86 static struct resource_spec cesa_res_spec[] = { 87 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 88 { SYS_RES_MEMORY, 1, RF_ACTIVE }, 89 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE }, 90 { -1, 0 } 91 }; 92 93 static device_method_t cesa_methods[] = { 94 /* Device interface */ 95 DEVMETHOD(device_probe, cesa_probe), 96 DEVMETHOD(device_attach, cesa_attach), 97 DEVMETHOD(device_detach, cesa_detach), 98 99 /* Crypto device methods */ 100 DEVMETHOD(cryptodev_newsession, cesa_newsession), 101 DEVMETHOD(cryptodev_process, cesa_process), 102 103 DEVMETHOD_END 104 }; 105 106 static driver_t cesa_driver = { 107 "cesa", 108 cesa_methods, 109 sizeof (struct cesa_softc) 110 }; 111 static devclass_t cesa_devclass; 112 113 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0); 114 MODULE_DEPEND(cesa, crypto, 1, 1, 1); 115 116 static void 117 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd) 118 { 119 #ifdef DEBUG 120 device_t dev; 121 122 dev = sc->sc_dev; 123 device_printf(dev, "CESA SA Hardware Descriptor:\n"); 124 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config); 125 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src); 126 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst); 127 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen); 128 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key); 129 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv); 130 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf); 131 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src); 132 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst); 133 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen); 134 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen); 135 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in); 136 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out); 137 #endif 138 } 139 140 static void 141 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 142 { 143 struct cesa_dma_mem *cdm; 144 145 if (error) 146 return; 147 148 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1.")); 149 cdm = arg; 150 cdm->cdm_paddr = segs->ds_addr; 151 } 152 153 static int 154 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm, 155 bus_size_t size) 156 { 157 int error; 158 159 KASSERT(cdm->cdm_vaddr == NULL, 160 ("%s(): DMA memory descriptor in use.", __func__)); 161 162 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */ 163 PAGE_SIZE, 0, /* alignment, boundary */ 164 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 165 BUS_SPACE_MAXADDR, /* highaddr */ 166 NULL, NULL, /* filtfunc, filtfuncarg */ 167 size, 1, /* maxsize, nsegments */ 168 size, 0, /* maxsegsz, flags */ 169 NULL, NULL, /* lockfunc, lockfuncarg */ 170 &cdm->cdm_tag); /* dmat */ 171 if (error) { 172 device_printf(sc->sc_dev, "failed to allocate busdma tag, error" 173 " %i!\n", error); 174 175 goto err1; 176 } 177 178 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr, 179 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map); 180 if (error) { 181 device_printf(sc->sc_dev, "failed to allocate DMA safe" 182 " memory, error %i!\n", error); 183 184 goto err2; 185 } 186 187 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr, 188 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT); 189 if (error) { 190 device_printf(sc->sc_dev, "cannot get address of the DMA" 191 " memory, error %i\n", error); 192 193 goto err3; 194 } 195 196 return (0); 197 err3: 198 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 199 err2: 200 bus_dma_tag_destroy(cdm->cdm_tag); 201 err1: 202 cdm->cdm_vaddr = NULL; 203 return (error); 204 } 205 206 static void 207 cesa_free_dma_mem(struct cesa_dma_mem *cdm) 208 { 209 210 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map); 211 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map); 212 bus_dma_tag_destroy(cdm->cdm_tag); 213 cdm->cdm_vaddr = NULL; 214 } 215 216 static void 217 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op) 218 { 219 220 /* Sync only if dma memory is valid */ 221 if (cdm->cdm_vaddr != NULL) 222 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op); 223 } 224 225 static void 226 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op) 227 { 228 229 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op); 230 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op); 231 cesa_sync_dma_mem(&sc->sc_requests_cdm, op); 232 } 233 234 static struct cesa_request * 235 cesa_alloc_request(struct cesa_softc *sc) 236 { 237 struct cesa_request *cr; 238 239 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests); 240 if (!cr) 241 return (NULL); 242 243 STAILQ_INIT(&cr->cr_tdesc); 244 STAILQ_INIT(&cr->cr_sdesc); 245 246 return (cr); 247 } 248 249 static void 250 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr) 251 { 252 253 /* Free TDMA descriptors assigned to this request */ 254 CESA_LOCK(sc, tdesc); 255 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc); 256 CESA_UNLOCK(sc, tdesc); 257 258 /* Free SA descriptors assigned to this request */ 259 CESA_LOCK(sc, sdesc); 260 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc); 261 CESA_UNLOCK(sc, sdesc); 262 263 /* Unload DMA memory associated with request */ 264 if (cr->cr_dmap_loaded) { 265 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap); 266 cr->cr_dmap_loaded = 0; 267 } 268 269 CESA_GENERIC_FREE_LOCKED(sc, cr, requests); 270 } 271 272 static void 273 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr) 274 { 275 276 CESA_LOCK(sc, requests); 277 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq); 278 CESA_UNLOCK(sc, requests); 279 } 280 281 static struct cesa_tdma_desc * 282 cesa_alloc_tdesc(struct cesa_softc *sc) 283 { 284 struct cesa_tdma_desc *ctd; 285 286 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc); 287 288 if (!ctd) 289 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. " 290 "Consider increasing CESA_TDMA_DESCRIPTORS.\n"); 291 292 return (ctd); 293 } 294 295 static struct cesa_sa_desc * 296 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr) 297 { 298 struct cesa_sa_desc *csd; 299 300 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc); 301 if (!csd) { 302 device_printf(sc->sc_dev, "SA descriptors pool exhaused. " 303 "Consider increasing CESA_SA_DESCRIPTORS.\n"); 304 return (NULL); 305 } 306 307 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq); 308 309 /* Fill-in SA descriptor with default values */ 310 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key); 311 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv); 312 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv); 313 csd->csd_cshd->cshd_enc_src = 0; 314 csd->csd_cshd->cshd_enc_dst = 0; 315 csd->csd_cshd->cshd_enc_dlen = 0; 316 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash); 317 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in); 318 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out); 319 csd->csd_cshd->cshd_mac_src = 0; 320 csd->csd_cshd->cshd_mac_dlen = 0; 321 322 return (csd); 323 } 324 325 static struct cesa_tdma_desc * 326 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src, 327 bus_size_t size) 328 { 329 struct cesa_tdma_desc *ctd; 330 331 ctd = cesa_alloc_tdesc(sc); 332 if (!ctd) 333 return (NULL); 334 335 ctd->ctd_cthd->cthd_dst = dst; 336 ctd->ctd_cthd->cthd_src = src; 337 ctd->ctd_cthd->cthd_byte_count = size; 338 339 /* Handle special control packet */ 340 if (size != 0) 341 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED; 342 else 343 ctd->ctd_cthd->cthd_flags = 0; 344 345 return (ctd); 346 } 347 348 static struct cesa_tdma_desc * 349 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 350 { 351 352 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa + 353 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr, 354 sizeof(struct cesa_sa_data))); 355 } 356 357 static struct cesa_tdma_desc * 358 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr) 359 { 360 361 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa + 362 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data))); 363 } 364 365 static struct cesa_tdma_desc * 366 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd) 367 { 368 369 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr, 370 sizeof(struct cesa_sa_hdesc))); 371 } 372 373 static void 374 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd) 375 { 376 struct cesa_tdma_desc *ctd_prev; 377 378 if (!STAILQ_EMPTY(&cr->cr_tdesc)) { 379 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq); 380 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr; 381 } 382 383 ctd->ctd_cthd->cthd_next = 0; 384 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq); 385 } 386 387 static int 388 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr, 389 struct cesa_packet *cp, struct cesa_sa_desc *csd) 390 { 391 struct cesa_tdma_desc *ctd, *tmp; 392 393 /* Copy SA descriptor for this packet */ 394 ctd = cesa_tdma_copy_sdesc(sc, csd); 395 if (!ctd) 396 return (ENOMEM); 397 398 cesa_append_tdesc(cr, ctd); 399 400 /* Copy data to be processed */ 401 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp) 402 cesa_append_tdesc(cr, ctd); 403 STAILQ_INIT(&cp->cp_copyin); 404 405 /* Insert control descriptor */ 406 ctd = cesa_tdma_copy(sc, 0, 0, 0); 407 if (!ctd) 408 return (ENOMEM); 409 410 cesa_append_tdesc(cr, ctd); 411 412 /* Copy back results */ 413 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp) 414 cesa_append_tdesc(cr, ctd); 415 STAILQ_INIT(&cp->cp_copyout); 416 417 return (0); 418 } 419 420 static int 421 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen) 422 { 423 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN]; 424 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN]; 425 SHA1_CTX sha1ctx; 426 SHA256_CTX sha256ctx; 427 MD5_CTX md5ctx; 428 uint32_t *hout; 429 uint32_t *hin; 430 int i; 431 432 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 433 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN); 434 for (i = 0; i < mklen; i++) { 435 ipad[i] ^= mkey[i]; 436 opad[i] ^= mkey[i]; 437 } 438 439 hin = (uint32_t *)cs->cs_hiv_in; 440 hout = (uint32_t *)cs->cs_hiv_out; 441 442 switch (alg) { 443 case CRYPTO_MD5_HMAC: 444 MD5Init(&md5ctx); 445 MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN); 446 memcpy(hin, md5ctx.state, sizeof(md5ctx.state)); 447 MD5Init(&md5ctx); 448 MD5Update(&md5ctx, opad, MD5_BLOCK_LEN); 449 memcpy(hout, md5ctx.state, sizeof(md5ctx.state)); 450 break; 451 case CRYPTO_SHA1_HMAC: 452 SHA1Init(&sha1ctx); 453 SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN); 454 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 455 SHA1Init(&sha1ctx); 456 SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN); 457 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32)); 458 break; 459 case CRYPTO_SHA2_256_HMAC: 460 SHA256_Init(&sha256ctx); 461 SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN); 462 memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state)); 463 SHA256_Init(&sha256ctx); 464 SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN); 465 memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state)); 466 break; 467 default: 468 return (EINVAL); 469 } 470 471 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) { 472 hin[i] = htobe32(hin[i]); 473 hout[i] = htobe32(hout[i]); 474 } 475 476 return (0); 477 } 478 479 static int 480 cesa_prep_aes_key(struct cesa_session *cs) 481 { 482 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)]; 483 uint32_t *dkey; 484 int i; 485 486 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8); 487 488 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK; 489 dkey = (uint32_t *)cs->cs_aes_dkey; 490 491 switch (cs->cs_klen) { 492 case 16: 493 cs->cs_config |= CESA_CSH_AES_KLEN_128; 494 for (i = 0; i < 4; i++) 495 *dkey++ = htobe32(ek[4 * 10 + i]); 496 break; 497 case 24: 498 cs->cs_config |= CESA_CSH_AES_KLEN_192; 499 for (i = 0; i < 4; i++) 500 *dkey++ = htobe32(ek[4 * 12 + i]); 501 for (i = 0; i < 2; i++) 502 *dkey++ = htobe32(ek[4 * 11 + 2 + i]); 503 break; 504 case 32: 505 cs->cs_config |= CESA_CSH_AES_KLEN_256; 506 for (i = 0; i < 4; i++) 507 *dkey++ = htobe32(ek[4 * 14 + i]); 508 for (i = 0; i < 4; i++) 509 *dkey++ = htobe32(ek[4 * 13 + i]); 510 break; 511 default: 512 return (EINVAL); 513 } 514 515 return (0); 516 } 517 518 static int 519 cesa_is_hash(int alg) 520 { 521 522 switch (alg) { 523 case CRYPTO_MD5: 524 case CRYPTO_MD5_HMAC: 525 case CRYPTO_SHA1: 526 case CRYPTO_SHA1_HMAC: 527 case CRYPTO_SHA2_256_HMAC: 528 return (1); 529 default: 530 return (0); 531 } 532 } 533 534 static void 535 cesa_start_packet(struct cesa_packet *cp, unsigned int size) 536 { 537 538 cp->cp_size = size; 539 cp->cp_offset = 0; 540 STAILQ_INIT(&cp->cp_copyin); 541 STAILQ_INIT(&cp->cp_copyout); 542 } 543 544 static int 545 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp, 546 bus_dma_segment_t *seg) 547 { 548 struct cesa_tdma_desc *ctd; 549 unsigned int bsize; 550 551 /* Calculate size of block copy */ 552 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset); 553 554 if (bsize > 0) { 555 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa + 556 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize); 557 if (!ctd) 558 return (-ENOMEM); 559 560 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq); 561 562 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa + 563 CESA_DATA(cp->cp_offset), bsize); 564 if (!ctd) 565 return (-ENOMEM); 566 567 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq); 568 569 seg->ds_len -= bsize; 570 seg->ds_addr += bsize; 571 cp->cp_offset += bsize; 572 } 573 574 return (bsize); 575 } 576 577 static void 578 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 579 { 580 unsigned int mpsize, fragmented; 581 unsigned int mlen, mskip, tmlen; 582 struct cesa_chain_info *cci; 583 unsigned int elen, eskip; 584 unsigned int skip, len; 585 struct cesa_sa_desc *csd; 586 struct cesa_request *cr; 587 struct cesa_softc *sc; 588 struct cesa_packet cp; 589 bus_dma_segment_t seg; 590 uint32_t config; 591 int size; 592 593 cci = arg; 594 sc = cci->cci_sc; 595 cr = cci->cci_cr; 596 597 if (error) { 598 cci->cci_error = error; 599 return; 600 } 601 602 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0; 603 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0; 604 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0; 605 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0; 606 607 if (elen && mlen && 608 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) || 609 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) || 610 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) { 611 /* 612 * Data alignment in the request does not meet CESA requiremnts 613 * for combined encryption/decryption and hashing. We have to 614 * split the request to separate operations and process them 615 * one by one. 616 */ 617 config = cci->cci_config; 618 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) { 619 config &= ~CESA_CSHD_OP_MASK; 620 621 cci->cci_config = config | CESA_CSHD_MAC; 622 cci->cci_enc = NULL; 623 cci->cci_mac = cr->cr_mac; 624 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 625 626 cci->cci_config = config | CESA_CSHD_ENC; 627 cci->cci_enc = cr->cr_enc; 628 cci->cci_mac = NULL; 629 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 630 } else { 631 config &= ~CESA_CSHD_OP_MASK; 632 633 cci->cci_config = config | CESA_CSHD_ENC; 634 cci->cci_enc = cr->cr_enc; 635 cci->cci_mac = NULL; 636 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 637 638 cci->cci_config = config | CESA_CSHD_MAC; 639 cci->cci_enc = NULL; 640 cci->cci_mac = cr->cr_mac; 641 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error); 642 } 643 644 return; 645 } 646 647 tmlen = mlen; 648 fragmented = 0; 649 mpsize = CESA_MAX_PACKET_SIZE; 650 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1)); 651 652 if (elen && mlen) { 653 skip = MIN(eskip, mskip); 654 len = MAX(elen + eskip, mlen + mskip) - skip; 655 } else if (elen) { 656 skip = eskip; 657 len = elen; 658 } else { 659 skip = mskip; 660 len = mlen; 661 } 662 663 /* Start first packet in chain */ 664 cesa_start_packet(&cp, MIN(mpsize, len)); 665 666 while (nseg-- && len > 0) { 667 seg = *(segs++); 668 669 /* 670 * Skip data in buffer on which neither ENC nor MAC operation 671 * is requested. 672 */ 673 if (skip > 0) { 674 size = MIN(skip, seg.ds_len); 675 skip -= size; 676 677 seg.ds_addr += size; 678 seg.ds_len -= size; 679 680 if (eskip > 0) 681 eskip -= size; 682 683 if (mskip > 0) 684 mskip -= size; 685 686 if (seg.ds_len == 0) 687 continue; 688 } 689 690 while (1) { 691 /* 692 * Fill in current packet with data. Break if there is 693 * no more data in current DMA segment or an error 694 * occurred. 695 */ 696 size = cesa_fill_packet(sc, &cp, &seg); 697 if (size <= 0) { 698 error = -size; 699 break; 700 } 701 702 len -= size; 703 704 /* If packet is full, append it to the chain */ 705 if (cp.cp_size == cp.cp_offset) { 706 csd = cesa_alloc_sdesc(sc, cr); 707 if (!csd) { 708 error = ENOMEM; 709 break; 710 } 711 712 /* Create SA descriptor for this packet */ 713 csd->csd_cshd->cshd_config = cci->cci_config; 714 csd->csd_cshd->cshd_mac_total_dlen = tmlen; 715 716 /* 717 * Enable fragmentation if request will not fit 718 * into one packet. 719 */ 720 if (len > 0) { 721 if (!fragmented) { 722 fragmented = 1; 723 csd->csd_cshd->cshd_config |= 724 CESA_CSHD_FRAG_FIRST; 725 } else 726 csd->csd_cshd->cshd_config |= 727 CESA_CSHD_FRAG_MIDDLE; 728 } else if (fragmented) 729 csd->csd_cshd->cshd_config |= 730 CESA_CSHD_FRAG_LAST; 731 732 if (eskip < cp.cp_size && elen > 0) { 733 csd->csd_cshd->cshd_enc_src = 734 CESA_DATA(eskip); 735 csd->csd_cshd->cshd_enc_dst = 736 CESA_DATA(eskip); 737 csd->csd_cshd->cshd_enc_dlen = 738 MIN(elen, cp.cp_size - eskip); 739 } 740 741 if (mskip < cp.cp_size && mlen > 0) { 742 csd->csd_cshd->cshd_mac_src = 743 CESA_DATA(mskip); 744 csd->csd_cshd->cshd_mac_dlen = 745 MIN(mlen, cp.cp_size - mskip); 746 } 747 748 elen -= csd->csd_cshd->cshd_enc_dlen; 749 eskip -= MIN(eskip, cp.cp_size); 750 mlen -= csd->csd_cshd->cshd_mac_dlen; 751 mskip -= MIN(mskip, cp.cp_size); 752 753 cesa_dump_cshd(sc, csd->csd_cshd); 754 755 /* Append packet to the request */ 756 error = cesa_append_packet(sc, cr, &cp, csd); 757 if (error) 758 break; 759 760 /* Start a new packet, as current is full */ 761 cesa_start_packet(&cp, MIN(mpsize, len)); 762 } 763 } 764 765 if (error) 766 break; 767 } 768 769 if (error) { 770 /* 771 * Move all allocated resources to the request. They will be 772 * freed later. 773 */ 774 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin); 775 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout); 776 cci->cci_error = error; 777 } 778 } 779 780 static void 781 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg, 782 bus_size_t size, int error) 783 { 784 785 cesa_create_chain_cb(arg, segs, nseg, error); 786 } 787 788 static int 789 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr) 790 { 791 struct cesa_chain_info cci; 792 struct cesa_tdma_desc *ctd; 793 uint32_t config; 794 int error; 795 796 error = 0; 797 CESA_LOCK_ASSERT(sc, sessions); 798 799 /* Create request metadata */ 800 if (cr->cr_enc) { 801 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC && 802 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 803 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey, 804 cr->cr_cs->cs_klen); 805 else 806 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key, 807 cr->cr_cs->cs_klen); 808 } 809 810 if (cr->cr_mac) { 811 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in, 812 CESA_MAX_HASH_LEN); 813 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out, 814 CESA_MAX_HASH_LEN); 815 } 816 817 ctd = cesa_tdma_copyin_sa_data(sc, cr); 818 if (!ctd) 819 return (ENOMEM); 820 821 cesa_append_tdesc(cr, ctd); 822 823 /* Prepare SA configuration */ 824 config = cr->cr_cs->cs_config; 825 826 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0) 827 config |= CESA_CSHD_DECRYPT; 828 if (cr->cr_enc && !cr->cr_mac) 829 config |= CESA_CSHD_ENC; 830 if (!cr->cr_enc && cr->cr_mac) 831 config |= CESA_CSHD_MAC; 832 if (cr->cr_enc && cr->cr_mac) 833 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC : 834 CESA_CSHD_ENC_AND_MAC; 835 836 /* Create data packets */ 837 cci.cci_sc = sc; 838 cci.cci_cr = cr; 839 cci.cci_enc = cr->cr_enc; 840 cci.cci_mac = cr->cr_mac; 841 cci.cci_config = config; 842 cci.cci_error = 0; 843 844 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV) 845 error = bus_dmamap_load_uio(sc->sc_data_dtag, 846 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf, 847 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 848 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF) 849 error = bus_dmamap_load_mbuf(sc->sc_data_dtag, 850 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf, 851 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT); 852 else 853 error = bus_dmamap_load(sc->sc_data_dtag, 854 cr->cr_dmap, cr->cr_crp->crp_buf, 855 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci, 856 BUS_DMA_NOWAIT); 857 858 if (!error) 859 cr->cr_dmap_loaded = 1; 860 861 if (cci.cci_error) 862 error = cci.cci_error; 863 864 if (error) 865 return (error); 866 867 /* Read back request metadata */ 868 ctd = cesa_tdma_copyout_sa_data(sc, cr); 869 if (!ctd) 870 return (ENOMEM); 871 872 cesa_append_tdesc(cr, ctd); 873 874 return (0); 875 } 876 877 static void 878 cesa_execute(struct cesa_softc *sc) 879 { 880 struct cesa_tdma_desc *prev_ctd, *ctd; 881 struct cesa_request *prev_cr, *cr; 882 883 CESA_LOCK(sc, requests); 884 885 /* 886 * If ready list is empty, there is nothing to execute. If queued list 887 * is not empty, the hardware is busy and we cannot start another 888 * execution. 889 */ 890 if (STAILQ_EMPTY(&sc->sc_ready_requests) || 891 !STAILQ_EMPTY(&sc->sc_queued_requests)) { 892 CESA_UNLOCK(sc, requests); 893 return; 894 } 895 896 /* Move all ready requests to queued list */ 897 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests); 898 STAILQ_INIT(&sc->sc_ready_requests); 899 900 /* Create one execution chain from all requests on the list */ 901 if (STAILQ_FIRST(&sc->sc_queued_requests) != 902 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) { 903 prev_cr = NULL; 904 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD | 905 BUS_DMASYNC_POSTWRITE); 906 907 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) { 908 if (prev_cr) { 909 ctd = STAILQ_FIRST(&cr->cr_tdesc); 910 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc, 911 cesa_tdma_desc, ctd_stq); 912 913 prev_ctd->ctd_cthd->cthd_next = 914 ctd->ctd_cthd_paddr; 915 } 916 917 prev_cr = cr; 918 } 919 920 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD | 921 BUS_DMASYNC_PREWRITE); 922 } 923 924 /* Start chain execution in hardware */ 925 cr = STAILQ_FIRST(&sc->sc_queued_requests); 926 ctd = STAILQ_FIRST(&cr->cr_tdesc); 927 928 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr); 929 930 if (sc->sc_soc_id == MV_DEV_88F6828 || 931 sc->sc_soc_id == MV_DEV_88F6820 || 932 sc->sc_soc_id == MV_DEV_88F6810) 933 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2); 934 else 935 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE); 936 937 CESA_UNLOCK(sc, requests); 938 } 939 940 static int 941 cesa_setup_sram(struct cesa_softc *sc) 942 { 943 phandle_t sram_node; 944 ihandle_t sram_ihandle; 945 pcell_t sram_handle, sram_reg[2]; 946 void *sram_va; 947 int rv; 948 949 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle", 950 (void *)&sram_handle, sizeof(sram_handle)); 951 if (rv <= 0) 952 return (rv); 953 954 sram_ihandle = (ihandle_t)sram_handle; 955 sram_node = OF_instance_to_package(sram_ihandle); 956 957 rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg)); 958 if (rv <= 0) 959 return (rv); 960 961 sc->sc_sram_base_pa = sram_reg[0]; 962 /* Store SRAM size to be able to unmap in detach() */ 963 sc->sc_sram_size = sram_reg[1]; 964 965 if (sc->sc_soc_id != MV_DEV_88F6828 && 966 sc->sc_soc_id != MV_DEV_88F6820 && 967 sc->sc_soc_id != MV_DEV_88F6810) 968 return (0); 969 970 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 971 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 972 if (sram_va == NULL) 973 return (ENOMEM); 974 sc->sc_sram_base_va = (vm_offset_t)sram_va; 975 976 return (0); 977 } 978 979 /* 980 * Function: device_from_node 981 * This function returns appropriate device_t to phandle_t 982 * Parameters: 983 * root - device where you want to start search 984 * if you provide NULL here, function will take 985 * "root0" device as root. 986 * node - we are checking every device_t to be 987 * appropriate with this. 988 */ 989 static device_t 990 device_from_node(device_t root, phandle_t node) 991 { 992 device_t *children, retval; 993 int nkid, i; 994 995 /* Nothing matches no node */ 996 if (node == -1) 997 return (NULL); 998 999 if (root == NULL) 1000 /* Get root of device tree */ 1001 if ((root = device_lookup_by_name("root0")) == NULL) 1002 return (NULL); 1003 1004 if (device_get_children(root, &children, &nkid) != 0) 1005 return (NULL); 1006 1007 retval = NULL; 1008 for (i = 0; i < nkid; i++) { 1009 /* Check if device and node matches */ 1010 if (OFW_BUS_GET_NODE(root, children[i]) == node) { 1011 retval = children[i]; 1012 break; 1013 } 1014 /* or go deeper */ 1015 if ((retval = device_from_node(children[i], node)) != NULL) 1016 break; 1017 } 1018 free(children, M_TEMP); 1019 1020 return (retval); 1021 } 1022 1023 static int 1024 cesa_setup_sram_armada(struct cesa_softc *sc) 1025 { 1026 phandle_t sram_node; 1027 ihandle_t sram_ihandle; 1028 pcell_t sram_handle[2]; 1029 void *sram_va; 1030 int rv, j; 1031 struct resource_list rl; 1032 struct resource_list_entry *rle; 1033 struct simplebus_softc *ssc; 1034 device_t sdev; 1035 1036 /* Get refs to SRAMS from CESA node */ 1037 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams", 1038 (void *)sram_handle, sizeof(sram_handle)); 1039 if (rv <= 0) 1040 return (rv); 1041 1042 if (sc->sc_cesa_engine_id >= 2) 1043 return (ENXIO); 1044 1045 /* Get SRAM node on the basis of sc_cesa_engine_id */ 1046 sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id]; 1047 sram_node = OF_instance_to_package(sram_ihandle); 1048 1049 /* Get device_t of simplebus (sram_node parent) */ 1050 sdev = device_from_node(NULL, OF_parent(sram_node)); 1051 if (!sdev) 1052 return (ENXIO); 1053 1054 ssc = device_get_softc(sdev); 1055 1056 resource_list_init(&rl); 1057 /* Parse reg property to resource list */ 1058 ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells, 1059 ssc->scells, &rl); 1060 1061 /* We expect only one resource */ 1062 rle = resource_list_find(&rl, SYS_RES_MEMORY, 0); 1063 if (rle == NULL) 1064 return (ENXIO); 1065 1066 /* Remap through ranges property */ 1067 for (j = 0; j < ssc->nranges; j++) { 1068 if (rle->start >= ssc->ranges[j].bus && 1069 rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) { 1070 rle->start -= ssc->ranges[j].bus; 1071 rle->start += ssc->ranges[j].host; 1072 rle->end -= ssc->ranges[j].bus; 1073 rle->end += ssc->ranges[j].host; 1074 } 1075 } 1076 1077 sc->sc_sram_base_pa = rle->start; 1078 sc->sc_sram_size = rle->count; 1079 1080 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */ 1081 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size); 1082 if (sram_va == NULL) 1083 return (ENOMEM); 1084 sc->sc_sram_base_va = (vm_offset_t)sram_va; 1085 1086 return (0); 1087 } 1088 1089 struct ofw_compat_data cesa_devices[] = { 1090 { "mrvl,cesa", (uintptr_t)true }, 1091 { "marvell,armada-38x-crypto", (uintptr_t)true }, 1092 { NULL, 0 } 1093 }; 1094 1095 static int 1096 cesa_probe(device_t dev) 1097 { 1098 1099 if (!ofw_bus_status_okay(dev)) 1100 return (ENXIO); 1101 1102 if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data) 1103 return (ENXIO); 1104 1105 device_set_desc(dev, "Marvell Cryptographic Engine and Security " 1106 "Accelerator"); 1107 1108 return (BUS_PROBE_DEFAULT); 1109 } 1110 1111 static int 1112 cesa_attach(device_t dev) 1113 { 1114 static int engine_idx = 0; 1115 struct simplebus_devinfo *ndi; 1116 struct resource_list *rl; 1117 struct cesa_softc *sc; 1118 1119 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) 1120 return (cesa_attach_late(dev)); 1121 1122 /* 1123 * Get simplebus_devinfo which contains 1124 * resource list filled with adresses and 1125 * interrupts read form FDT. 1126 * Let's correct it by splitting resources 1127 * for each engine. 1128 */ 1129 if ((ndi = device_get_ivars(dev)) == NULL) 1130 return (ENXIO); 1131 1132 rl = &ndi->rl; 1133 1134 switch (engine_idx) { 1135 case 0: 1136 /* Update regs values */ 1137 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA0_TDMA_ADDR, 1138 CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); 1139 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA0_CESA_ADDR, 1140 CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); 1141 1142 /* Remove unused interrupt */ 1143 resource_list_delete(rl, SYS_RES_IRQ, 1); 1144 break; 1145 1146 case 1: 1147 /* Update regs values */ 1148 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA1_TDMA_ADDR, 1149 CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE); 1150 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA1_CESA_ADDR, 1151 CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE); 1152 1153 /* Remove unused interrupt */ 1154 resource_list_delete(rl, SYS_RES_IRQ, 0); 1155 resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0; 1156 break; 1157 1158 default: 1159 device_printf(dev, "Bad cesa engine_idx\n"); 1160 return (ENXIO); 1161 } 1162 1163 sc = device_get_softc(dev); 1164 sc->sc_cesa_engine_id = engine_idx; 1165 1166 /* 1167 * Call simplebus_add_device only once. 1168 * It will create second cesa driver instance 1169 * with the same FDT node as first instance. 1170 * When second driver reach this function, 1171 * it will be configured to use second cesa engine 1172 */ 1173 if (engine_idx == 0) 1174 simplebus_add_device(device_get_parent(dev), ofw_bus_get_node(dev), 1175 0, "cesa", 1, NULL); 1176 1177 engine_idx++; 1178 1179 return (cesa_attach_late(dev)); 1180 } 1181 1182 static int 1183 cesa_attach_late(device_t dev) 1184 { 1185 struct cesa_softc *sc; 1186 uint32_t d, r, val; 1187 int error; 1188 int i; 1189 1190 sc = device_get_softc(dev); 1191 sc->sc_blocked = 0; 1192 sc->sc_error = 0; 1193 sc->sc_dev = dev; 1194 1195 soc_id(&d, &r); 1196 1197 switch (d) { 1198 case MV_DEV_88F6281: 1199 case MV_DEV_88F6282: 1200 /* Check if CESA peripheral device has power turned on */ 1201 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) == 1202 CPU_PM_CTRL_CRYPTO) { 1203 device_printf(dev, "not powered on\n"); 1204 return (ENXIO); 1205 } 1206 sc->sc_tperr = 0; 1207 break; 1208 case MV_DEV_88F6828: 1209 case MV_DEV_88F6820: 1210 case MV_DEV_88F6810: 1211 sc->sc_tperr = 0; 1212 break; 1213 case MV_DEV_MV78100: 1214 case MV_DEV_MV78100_Z0: 1215 /* Check if CESA peripheral device has power turned on */ 1216 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) != 1217 CPU_PM_CTRL_CRYPTO) { 1218 device_printf(dev, "not powered on\n"); 1219 return (ENXIO); 1220 } 1221 sc->sc_tperr = CESA_ICR_TPERR; 1222 break; 1223 default: 1224 return (ENXIO); 1225 } 1226 1227 sc->sc_soc_id = d; 1228 1229 /* Initialize mutexes */ 1230 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev), 1231 "CESA Shared Data", MTX_DEF); 1232 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev), 1233 "CESA TDMA Descriptors Pool", MTX_DEF); 1234 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev), 1235 "CESA SA Descriptors Pool", MTX_DEF); 1236 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev), 1237 "CESA Requests Pool", MTX_DEF); 1238 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev), 1239 "CESA Sessions Pool", MTX_DEF); 1240 1241 /* Allocate I/O and IRQ resources */ 1242 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res); 1243 if (error) { 1244 device_printf(dev, "could not allocate resources\n"); 1245 goto err0; 1246 } 1247 1248 /* Acquire SRAM base address */ 1249 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto")) 1250 error = cesa_setup_sram(sc); 1251 else 1252 error = cesa_setup_sram_armada(sc); 1253 1254 if (error) { 1255 device_printf(dev, "could not setup SRAM\n"); 1256 goto err1; 1257 } 1258 1259 /* Setup interrupt handler */ 1260 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET | 1261 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie)); 1262 if (error) { 1263 device_printf(dev, "could not setup engine completion irq\n"); 1264 goto err2; 1265 } 1266 1267 /* Create DMA tag for processed data */ 1268 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 1269 1, 0, /* alignment, boundary */ 1270 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1271 BUS_SPACE_MAXADDR, /* highaddr */ 1272 NULL, NULL, /* filtfunc, filtfuncarg */ 1273 CESA_MAX_REQUEST_SIZE, /* maxsize */ 1274 CESA_MAX_FRAGMENTS, /* nsegments */ 1275 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */ 1276 NULL, NULL, /* lockfunc, lockfuncarg */ 1277 &sc->sc_data_dtag); /* dmat */ 1278 if (error) 1279 goto err3; 1280 1281 /* Initialize data structures: TDMA Descriptors Pool */ 1282 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm, 1283 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc)); 1284 if (error) 1285 goto err4; 1286 1287 STAILQ_INIT(&sc->sc_free_tdesc); 1288 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) { 1289 sc->sc_tdesc[i].ctd_cthd = 1290 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i; 1291 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr + 1292 (i * sizeof(struct cesa_tdma_hdesc)); 1293 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i], 1294 ctd_stq); 1295 } 1296 1297 /* Initialize data structures: SA Descriptors Pool */ 1298 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm, 1299 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc)); 1300 if (error) 1301 goto err5; 1302 1303 STAILQ_INIT(&sc->sc_free_sdesc); 1304 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) { 1305 sc->sc_sdesc[i].csd_cshd = 1306 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i; 1307 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr + 1308 (i * sizeof(struct cesa_sa_hdesc)); 1309 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i], 1310 csd_stq); 1311 } 1312 1313 /* Initialize data structures: Requests Pool */ 1314 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm, 1315 CESA_REQUESTS * sizeof(struct cesa_sa_data)); 1316 if (error) 1317 goto err6; 1318 1319 STAILQ_INIT(&sc->sc_free_requests); 1320 STAILQ_INIT(&sc->sc_ready_requests); 1321 STAILQ_INIT(&sc->sc_queued_requests); 1322 for (i = 0; i < CESA_REQUESTS; i++) { 1323 sc->sc_requests[i].cr_csd = 1324 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i; 1325 sc->sc_requests[i].cr_csd_paddr = 1326 sc->sc_requests_cdm.cdm_paddr + 1327 (i * sizeof(struct cesa_sa_data)); 1328 1329 /* Preallocate DMA maps */ 1330 error = bus_dmamap_create(sc->sc_data_dtag, 0, 1331 &sc->sc_requests[i].cr_dmap); 1332 if (error && i > 0) { 1333 i--; 1334 do { 1335 bus_dmamap_destroy(sc->sc_data_dtag, 1336 sc->sc_requests[i].cr_dmap); 1337 } while (i--); 1338 1339 goto err7; 1340 } 1341 1342 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i], 1343 cr_stq); 1344 } 1345 1346 /* 1347 * Initialize TDMA: 1348 * - Burst limit: 128 bytes, 1349 * - Outstanding reads enabled, 1350 * - No byte-swap. 1351 */ 1352 val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 | 1353 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE; 1354 1355 if (sc->sc_soc_id == MV_DEV_88F6828 || 1356 sc->sc_soc_id == MV_DEV_88F6820 || 1357 sc->sc_soc_id == MV_DEV_88F6810) 1358 val |= CESA_TDMA_NUM_OUTSTAND; 1359 1360 CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val); 1361 1362 /* 1363 * Initialize SA: 1364 * - SA descriptor is present at beginning of CESA SRAM, 1365 * - Multi-packet chain mode, 1366 * - Cooperation with TDMA enabled. 1367 */ 1368 CESA_REG_WRITE(sc, CESA_SA_DPR, 0); 1369 CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA | 1370 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE); 1371 1372 /* Unmask interrupts */ 1373 CESA_REG_WRITE(sc, CESA_ICR, 0); 1374 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr); 1375 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1376 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS | 1377 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT | 1378 CESA_TDMA_EMR_DATA_ERROR); 1379 1380 /* Register in OCF */ 1381 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session), 1382 CRYPTOCAP_F_HARDWARE); 1383 if (sc->sc_cid < 0) { 1384 device_printf(dev, "could not get crypto driver id\n"); 1385 goto err8; 1386 } 1387 1388 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0); 1389 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0); 1390 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0); 1391 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0); 1392 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0); 1393 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0); 1394 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0); 1395 if (sc->sc_soc_id == MV_DEV_88F6828 || 1396 sc->sc_soc_id == MV_DEV_88F6820 || 1397 sc->sc_soc_id == MV_DEV_88F6810) 1398 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0); 1399 1400 return (0); 1401 err8: 1402 for (i = 0; i < CESA_REQUESTS; i++) 1403 bus_dmamap_destroy(sc->sc_data_dtag, 1404 sc->sc_requests[i].cr_dmap); 1405 err7: 1406 cesa_free_dma_mem(&sc->sc_requests_cdm); 1407 err6: 1408 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1409 err5: 1410 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1411 err4: 1412 bus_dma_tag_destroy(sc->sc_data_dtag); 1413 err3: 1414 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1415 err2: 1416 if (sc->sc_soc_id == MV_DEV_88F6828 || 1417 sc->sc_soc_id == MV_DEV_88F6820 || 1418 sc->sc_soc_id == MV_DEV_88F6810) 1419 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1420 err1: 1421 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1422 err0: 1423 mtx_destroy(&sc->sc_sessions_lock); 1424 mtx_destroy(&sc->sc_requests_lock); 1425 mtx_destroy(&sc->sc_sdesc_lock); 1426 mtx_destroy(&sc->sc_tdesc_lock); 1427 mtx_destroy(&sc->sc_sc_lock); 1428 return (ENXIO); 1429 } 1430 1431 static int 1432 cesa_detach(device_t dev) 1433 { 1434 struct cesa_softc *sc; 1435 int i; 1436 1437 sc = device_get_softc(dev); 1438 1439 /* TODO: Wait for queued requests completion before shutdown. */ 1440 1441 /* Mask interrupts */ 1442 CESA_REG_WRITE(sc, CESA_ICM, 0); 1443 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0); 1444 1445 /* Unregister from OCF */ 1446 crypto_unregister_all(sc->sc_cid); 1447 1448 /* Free DMA Maps */ 1449 for (i = 0; i < CESA_REQUESTS; i++) 1450 bus_dmamap_destroy(sc->sc_data_dtag, 1451 sc->sc_requests[i].cr_dmap); 1452 1453 /* Free DMA Memory */ 1454 cesa_free_dma_mem(&sc->sc_requests_cdm); 1455 cesa_free_dma_mem(&sc->sc_sdesc_cdm); 1456 cesa_free_dma_mem(&sc->sc_tdesc_cdm); 1457 1458 /* Free DMA Tag */ 1459 bus_dma_tag_destroy(sc->sc_data_dtag); 1460 1461 /* Stop interrupt */ 1462 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie); 1463 1464 /* Relase I/O and IRQ resources */ 1465 bus_release_resources(dev, cesa_res_spec, sc->sc_res); 1466 1467 /* Unmap SRAM memory */ 1468 if (sc->sc_soc_id == MV_DEV_88F6828 || 1469 sc->sc_soc_id == MV_DEV_88F6820 || 1470 sc->sc_soc_id == MV_DEV_88F6810) 1471 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size); 1472 1473 /* Destroy mutexes */ 1474 mtx_destroy(&sc->sc_sessions_lock); 1475 mtx_destroy(&sc->sc_requests_lock); 1476 mtx_destroy(&sc->sc_sdesc_lock); 1477 mtx_destroy(&sc->sc_tdesc_lock); 1478 mtx_destroy(&sc->sc_sc_lock); 1479 1480 return (0); 1481 } 1482 1483 static void 1484 cesa_intr(void *arg) 1485 { 1486 STAILQ_HEAD(, cesa_request) requests; 1487 struct cesa_request *cr, *tmp; 1488 struct cesa_softc *sc; 1489 uint32_t ecr, icr; 1490 int blocked; 1491 1492 sc = arg; 1493 1494 /* Ack interrupt */ 1495 ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR); 1496 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0); 1497 icr = CESA_REG_READ(sc, CESA_ICR); 1498 CESA_REG_WRITE(sc, CESA_ICR, 0); 1499 1500 /* Check for TDMA errors */ 1501 if (ecr & CESA_TDMA_ECR_MISS) { 1502 device_printf(sc->sc_dev, "TDMA Miss error detected!\n"); 1503 sc->sc_error = EIO; 1504 } 1505 1506 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) { 1507 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n"); 1508 sc->sc_error = EIO; 1509 } 1510 1511 if (ecr & CESA_TDMA_ECR_BOTH_HIT) { 1512 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n"); 1513 sc->sc_error = EIO; 1514 } 1515 1516 if (ecr & CESA_TDMA_ECR_DATA_ERROR) { 1517 device_printf(sc->sc_dev, "TDMA Data error detected!\n"); 1518 sc->sc_error = EIO; 1519 } 1520 1521 /* Check for CESA errors */ 1522 if (icr & sc->sc_tperr) { 1523 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n"); 1524 sc->sc_error = EIO; 1525 } 1526 1527 /* If there is nothing more to do, return */ 1528 if ((icr & CESA_ICR_ACCTDMA) == 0) 1529 return; 1530 1531 /* Get all finished requests */ 1532 CESA_LOCK(sc, requests); 1533 STAILQ_INIT(&requests); 1534 STAILQ_CONCAT(&requests, &sc->sc_queued_requests); 1535 STAILQ_INIT(&sc->sc_queued_requests); 1536 CESA_UNLOCK(sc, requests); 1537 1538 /* Execute all ready requests */ 1539 cesa_execute(sc); 1540 1541 /* Process completed requests */ 1542 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD | 1543 BUS_DMASYNC_POSTWRITE); 1544 1545 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) { 1546 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, 1547 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1548 1549 cr->cr_crp->crp_etype = sc->sc_error; 1550 if (cr->cr_mac) 1551 crypto_copyback(cr->cr_crp->crp_flags, 1552 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject, 1553 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash); 1554 1555 crypto_done(cr->cr_crp); 1556 cesa_free_request(sc, cr); 1557 } 1558 1559 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD | 1560 BUS_DMASYNC_PREWRITE); 1561 1562 sc->sc_error = 0; 1563 1564 /* Unblock driver if it ran out of resources */ 1565 CESA_LOCK(sc, sc); 1566 blocked = sc->sc_blocked; 1567 sc->sc_blocked = 0; 1568 CESA_UNLOCK(sc, sc); 1569 1570 if (blocked) 1571 crypto_unblock(sc->sc_cid, blocked); 1572 } 1573 1574 static int 1575 cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) 1576 { 1577 struct cesa_session *cs; 1578 struct cesa_softc *sc; 1579 struct cryptoini *enc; 1580 struct cryptoini *mac; 1581 int error; 1582 1583 sc = device_get_softc(dev); 1584 enc = NULL; 1585 mac = NULL; 1586 error = 0; 1587 1588 /* Check and parse input */ 1589 if (cesa_is_hash(cri->cri_alg)) 1590 mac = cri; 1591 else 1592 enc = cri; 1593 1594 cri = cri->cri_next; 1595 1596 if (cri) { 1597 if (!enc && !cesa_is_hash(cri->cri_alg)) 1598 enc = cri; 1599 1600 if (!mac && cesa_is_hash(cri->cri_alg)) 1601 mac = cri; 1602 1603 if (cri->cri_next || !(enc && mac)) 1604 return (EINVAL); 1605 } 1606 1607 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) || 1608 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN)) 1609 return (E2BIG); 1610 1611 /* Allocate session */ 1612 cs = crypto_get_driver_session(cses); 1613 1614 /* Prepare CESA configuration */ 1615 cs->cs_config = 0; 1616 cs->cs_ivlen = 1; 1617 cs->cs_mblen = 1; 1618 1619 if (enc) { 1620 switch (enc->cri_alg) { 1621 case CRYPTO_AES_CBC: 1622 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC; 1623 cs->cs_ivlen = AES_BLOCK_LEN; 1624 break; 1625 case CRYPTO_DES_CBC: 1626 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC; 1627 cs->cs_ivlen = DES_BLOCK_LEN; 1628 break; 1629 case CRYPTO_3DES_CBC: 1630 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE | 1631 CESA_CSHD_CBC; 1632 cs->cs_ivlen = DES3_BLOCK_LEN; 1633 break; 1634 default: 1635 error = EINVAL; 1636 break; 1637 } 1638 } 1639 1640 if (!error && mac) { 1641 switch (mac->cri_alg) { 1642 case CRYPTO_MD5: 1643 cs->cs_mblen = 1; 1644 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1645 mac->cri_mlen; 1646 cs->cs_config |= CESA_CSHD_MD5; 1647 break; 1648 case CRYPTO_MD5_HMAC: 1649 cs->cs_mblen = MD5_BLOCK_LEN; 1650 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN : 1651 mac->cri_mlen; 1652 cs->cs_config |= CESA_CSHD_MD5_HMAC; 1653 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1654 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1655 break; 1656 case CRYPTO_SHA1: 1657 cs->cs_mblen = 1; 1658 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1659 mac->cri_mlen; 1660 cs->cs_config |= CESA_CSHD_SHA1; 1661 break; 1662 case CRYPTO_SHA1_HMAC: 1663 cs->cs_mblen = SHA1_BLOCK_LEN; 1664 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN : 1665 mac->cri_mlen; 1666 cs->cs_config |= CESA_CSHD_SHA1_HMAC; 1667 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN) 1668 cs->cs_config |= CESA_CSHD_96_BIT_HMAC; 1669 break; 1670 case CRYPTO_SHA2_256_HMAC: 1671 cs->cs_mblen = SHA2_256_BLOCK_LEN; 1672 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN : 1673 mac->cri_mlen; 1674 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC; 1675 break; 1676 default: 1677 error = EINVAL; 1678 break; 1679 } 1680 } 1681 1682 /* Save cipher key */ 1683 if (!error && enc && enc->cri_key) { 1684 cs->cs_klen = enc->cri_klen / 8; 1685 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen); 1686 if (enc->cri_alg == CRYPTO_AES_CBC) 1687 error = cesa_prep_aes_key(cs); 1688 } 1689 1690 /* Save digest key */ 1691 if (!error && mac && mac->cri_key) 1692 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key, 1693 mac->cri_klen / 8); 1694 1695 if (error) 1696 return (error); 1697 1698 return (0); 1699 } 1700 1701 static int 1702 cesa_process(device_t dev, struct cryptop *crp, int hint) 1703 { 1704 struct cesa_request *cr; 1705 struct cesa_session *cs; 1706 struct cryptodesc *crd; 1707 struct cryptodesc *enc; 1708 struct cryptodesc *mac; 1709 struct cesa_softc *sc; 1710 int error; 1711 1712 sc = device_get_softc(dev); 1713 crd = crp->crp_desc; 1714 enc = NULL; 1715 mac = NULL; 1716 error = 0; 1717 1718 cs = crypto_get_driver_session(crp->crp_session); 1719 1720 /* Check and parse input */ 1721 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) { 1722 crp->crp_etype = E2BIG; 1723 crypto_done(crp); 1724 return (0); 1725 } 1726 1727 if (cesa_is_hash(crd->crd_alg)) 1728 mac = crd; 1729 else 1730 enc = crd; 1731 1732 crd = crd->crd_next; 1733 1734 if (crd) { 1735 if (!enc && !cesa_is_hash(crd->crd_alg)) 1736 enc = crd; 1737 1738 if (!mac && cesa_is_hash(crd->crd_alg)) 1739 mac = crd; 1740 1741 if (crd->crd_next || !(enc && mac)) { 1742 crp->crp_etype = EINVAL; 1743 crypto_done(crp); 1744 return (0); 1745 } 1746 } 1747 1748 /* 1749 * Get request descriptor. Block driver if there is no free 1750 * descriptors in pool. 1751 */ 1752 cr = cesa_alloc_request(sc); 1753 if (!cr) { 1754 CESA_LOCK(sc, sc); 1755 sc->sc_blocked = CRYPTO_SYMQ; 1756 CESA_UNLOCK(sc, sc); 1757 return (ERESTART); 1758 } 1759 1760 /* Prepare request */ 1761 cr->cr_crp = crp; 1762 cr->cr_enc = enc; 1763 cr->cr_mac = mac; 1764 cr->cr_cs = cs; 1765 1766 CESA_LOCK(sc, sessions); 1767 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1768 1769 if (enc && enc->crd_flags & CRD_F_ENCRYPT) { 1770 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1771 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1772 else 1773 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0); 1774 1775 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0) 1776 crypto_copyback(crp->crp_flags, crp->crp_buf, 1777 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1778 } else if (enc) { 1779 if (enc->crd_flags & CRD_F_IV_EXPLICIT) 1780 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen); 1781 else 1782 crypto_copydata(crp->crp_flags, crp->crp_buf, 1783 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv); 1784 } 1785 1786 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) { 1787 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) { 1788 cs->cs_klen = enc->crd_klen / 8; 1789 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen); 1790 if (enc->crd_alg == CRYPTO_AES_CBC) 1791 error = cesa_prep_aes_key(cs); 1792 } else 1793 error = E2BIG; 1794 } 1795 1796 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) { 1797 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN) 1798 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key, 1799 mac->crd_klen / 8); 1800 else 1801 error = E2BIG; 1802 } 1803 1804 /* Convert request to chain of TDMA and SA descriptors */ 1805 if (!error) 1806 error = cesa_create_chain(sc, cr); 1807 1808 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1809 CESA_UNLOCK(sc, sessions); 1810 1811 if (error) { 1812 cesa_free_request(sc, cr); 1813 crp->crp_etype = error; 1814 crypto_done(crp); 1815 return (0); 1816 } 1817 1818 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD | 1819 BUS_DMASYNC_PREWRITE); 1820 1821 /* Enqueue request to execution */ 1822 cesa_enqueue_request(sc, cr); 1823 1824 /* Start execution, if we have no more requests in queue */ 1825 if ((hint & CRYPTO_HINT_MORE) == 0) 1826 cesa_execute(sc); 1827 1828 return (0); 1829 } 1830