1 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.12 2003/06/04 17:56:59 sam Exp $ */ 2 /* $DragonFly: src/sys/dev/crypto/ubsec/ubsec.c,v 1.4 2003/11/20 22:07:25 dillon Exp $ */ 3 /* $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $ */ 4 5 /* 6 * Copyright (c) 2000 Jason L. Wright (jason@thought.net) 7 * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org) 8 * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com) 9 * 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by Jason L. Wright 23 * 4. The name of the author may not be used to endorse or promote products 24 * derived from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 27 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 28 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 29 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 30 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 31 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 32 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 34 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 35 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 * 38 * Effort sponsored in part by the Defense Advanced Research Projects 39 * Agency (DARPA) and Air Force Research Laboratory, Air Force 40 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 41 * 42 */ 43 44 /* 45 * uBsec 5[56]01, 58xx hardware crypto accelerator 46 */ 47 48 #include "opt_ubsec.h" 49 50 #include <sys/param.h> 51 #include <sys/systm.h> 52 #include <sys/proc.h> 53 #include <sys/errno.h> 54 #include <sys/malloc.h> 55 #include <sys/kernel.h> 56 #include <sys/mbuf.h> 57 #include <sys/sysctl.h> 58 #include <sys/endian.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 63 #include <machine/clock.h> 64 #include <machine/bus.h> 65 #include <machine/resource.h> 66 #include <sys/bus.h> 67 #include <sys/rman.h> 68 69 #include <crypto/sha1.h> 70 #include <opencrypto/cryptodev.h> 71 #include <opencrypto/cryptosoft.h> 72 #include <sys/md5.h> 73 #include <sys/random.h> 74 75 #include <bus/pci/pcivar.h> 76 #include <bus/pci/pcireg.h> 77 78 /* grr, #defines for gratuitous incompatibility in queue.h */ 79 #define SIMPLEQ_HEAD STAILQ_HEAD 80 #define SIMPLEQ_ENTRY STAILQ_ENTRY 81 #define SIMPLEQ_INIT STAILQ_INIT 82 #define SIMPLEQ_INSERT_TAIL STAILQ_INSERT_TAIL 83 #define SIMPLEQ_EMPTY STAILQ_EMPTY 84 #define SIMPLEQ_FIRST STAILQ_FIRST 85 #define SIMPLEQ_REMOVE_HEAD STAILQ_REMOVE_HEAD_UNTIL 86 #define SIMPLEQ_FOREACH STAILQ_FOREACH 87 /* ditto for endian.h */ 88 #define letoh16(x) le16toh(x) 89 #define letoh32(x) le32toh(x) 90 91 #ifdef UBSEC_RNDTEST 92 #include "../rndtest/rndtest.h" 93 #endif 94 #include "ubsecreg.h" 95 #include "ubsecvar.h" 96 97 /* 98 * Prototypes and count for the pci_device structure 99 */ 100 static int ubsec_probe(device_t); 101 static int ubsec_attach(device_t); 102 static int ubsec_detach(device_t); 103 static int ubsec_suspend(device_t); 104 static int ubsec_resume(device_t); 105 static void ubsec_shutdown(device_t); 106 107 static device_method_t ubsec_methods[] = { 108 /* Device interface */ 109 DEVMETHOD(device_probe, ubsec_probe), 110 DEVMETHOD(device_attach, ubsec_attach), 111 DEVMETHOD(device_detach, ubsec_detach), 112 DEVMETHOD(device_suspend, ubsec_suspend), 113 DEVMETHOD(device_resume, ubsec_resume), 114 DEVMETHOD(device_shutdown, ubsec_shutdown), 115 116 /* bus interface */ 117 DEVMETHOD(bus_print_child, bus_generic_print_child), 118 DEVMETHOD(bus_driver_added, bus_generic_driver_added), 119 120 { 0, 0 } 121 }; 122 static driver_t ubsec_driver = { 123 "ubsec", 124 ubsec_methods, 125 sizeof (struct ubsec_softc) 126 }; 127 static devclass_t ubsec_devclass; 128 129 DECLARE_DUMMY_MODULE(ubsec); 130 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0); 131 MODULE_DEPEND(ubsec, crypto, 1, 1, 1); 132 #ifdef UBSEC_RNDTEST 133 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1); 134 #endif 135 136 static void ubsec_intr(void *); 137 static int ubsec_newsession(void *, u_int32_t *, struct cryptoini *); 138 static int ubsec_freesession(void *, u_int64_t); 139 static int ubsec_process(void *, struct cryptop *, int); 140 static void ubsec_callback(struct ubsec_softc *, struct ubsec_q *); 141 static void ubsec_feed(struct ubsec_softc *); 142 static void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int); 143 static void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *); 144 static int ubsec_feed2(struct ubsec_softc *); 145 static void ubsec_rng(void *); 146 static int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t, 147 struct ubsec_dma_alloc *, int); 148 #define ubsec_dma_sync(_dma, _flags) \ 149 bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags)) 150 static void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *); 151 static int ubsec_dmamap_aligned(struct ubsec_operand *op); 152 153 static void ubsec_reset_board(struct ubsec_softc *sc); 154 static void ubsec_init_board(struct ubsec_softc *sc); 155 static void ubsec_init_pciregs(device_t dev); 156 static void ubsec_totalreset(struct ubsec_softc *sc); 157 158 static int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q); 159 160 static int ubsec_kprocess(void*, struct cryptkop *, int); 161 static int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int); 162 static int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int); 163 static int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int); 164 static void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *); 165 static int ubsec_ksigbits(struct crparam *); 166 static void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 167 static void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int); 168 169 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters"); 170 171 #ifdef UBSEC_DEBUG 172 static void ubsec_dump_pb(volatile struct ubsec_pktbuf *); 173 static void ubsec_dump_mcr(struct ubsec_mcr *); 174 static void ubsec_dump_ctx2(struct ubsec_ctx_keyop *); 175 176 static int ubsec_debug = 0; 177 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug, 178 0, "control debugging msgs"); 179 #endif 180 181 #define READ_REG(sc,r) \ 182 bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r)) 183 184 #define WRITE_REG(sc,reg,val) \ 185 bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val) 186 187 #define SWAP32(x) (x) = htole32(ntohl((x))) 188 #define HTOLE32(x) (x) = htole32(x) 189 190 191 struct ubsec_stats ubsecstats; 192 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats, 193 ubsec_stats, "driver statistics"); 194 195 static int 196 ubsec_probe(device_t dev) 197 { 198 if (pci_get_vendor(dev) == PCI_VENDOR_SUN && 199 (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 || 200 pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K)) 201 return (0); 202 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 203 (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 || 204 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)) 205 return (0); 206 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 207 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 || 208 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 209 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 || 210 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 || 211 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 212 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 213 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 214 )) 215 return (0); 216 return (ENXIO); 217 } 218 219 static const char* 220 ubsec_partname(struct ubsec_softc *sc) 221 { 222 /* XXX sprintf numbers when not decoded */ 223 switch (pci_get_vendor(sc->sc_dev)) { 224 case PCI_VENDOR_BROADCOM: 225 switch (pci_get_device(sc->sc_dev)) { 226 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801"; 227 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802"; 228 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805"; 229 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820"; 230 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821"; 231 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822"; 232 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823"; 233 } 234 return "Broadcom unknown-part"; 235 case PCI_VENDOR_BLUESTEEL: 236 switch (pci_get_device(sc->sc_dev)) { 237 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601"; 238 } 239 return "Bluesteel unknown-part"; 240 case PCI_VENDOR_SUN: 241 switch (pci_get_device(sc->sc_dev)) { 242 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821"; 243 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K"; 244 } 245 return "Sun unknown-part"; 246 } 247 return "Unknown-vendor unknown-part"; 248 } 249 250 static void 251 default_harvest(struct rndtest_state *rsp, void *buf, u_int count) 252 { 253 u_int32_t *p = (u_int32_t *)buf; 254 for (count /= sizeof (u_int32_t); count; count--) 255 add_true_randomness(*p++); 256 } 257 258 static int 259 ubsec_attach(device_t dev) 260 { 261 struct ubsec_softc *sc = device_get_softc(dev); 262 struct ubsec_dma *dmap; 263 u_int32_t cmd, i; 264 int rid; 265 266 KASSERT(sc != NULL, ("ubsec_attach: null software carrier!")); 267 bzero(sc, sizeof (*sc)); 268 sc->sc_dev = dev; 269 270 SIMPLEQ_INIT(&sc->sc_queue); 271 SIMPLEQ_INIT(&sc->sc_qchip); 272 SIMPLEQ_INIT(&sc->sc_queue2); 273 SIMPLEQ_INIT(&sc->sc_qchip2); 274 SIMPLEQ_INIT(&sc->sc_q2free); 275 276 /* XXX handle power management */ 277 278 sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR; 279 280 if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL && 281 pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601) 282 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 283 284 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 285 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 || 286 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805)) 287 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG; 288 289 if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 290 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820) 291 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 292 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 293 294 if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM && 295 (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 || 296 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 || 297 pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) || 298 (pci_get_vendor(dev) == PCI_VENDOR_SUN && 299 (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K || 300 pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) { 301 /* NB: the 5821/5822 defines some additional status bits */ 302 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY | 303 BS_STAT_MCR2_ALLEMPTY; 304 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG | 305 UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY; 306 } 307 308 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 309 cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN; 310 pci_write_config(dev, PCIR_COMMAND, cmd, 4); 311 cmd = pci_read_config(dev, PCIR_COMMAND, 4); 312 313 if (!(cmd & PCIM_CMD_MEMEN)) { 314 device_printf(dev, "failed to enable memory mapping\n"); 315 goto bad; 316 } 317 318 if (!(cmd & PCIM_CMD_BUSMASTEREN)) { 319 device_printf(dev, "failed to enable bus mastering\n"); 320 goto bad; 321 } 322 323 /* 324 * Setup memory-mapping of PCI registers. 325 */ 326 rid = BS_BAR; 327 sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 328 0, ~0, 1, RF_ACTIVE); 329 if (sc->sc_sr == NULL) { 330 device_printf(dev, "cannot map register space\n"); 331 goto bad; 332 } 333 sc->sc_st = rman_get_bustag(sc->sc_sr); 334 sc->sc_sh = rman_get_bushandle(sc->sc_sr); 335 336 /* 337 * Arrange interrupt line. 338 */ 339 rid = 0; 340 sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, 341 0, ~0, 1, RF_SHAREABLE|RF_ACTIVE); 342 if (sc->sc_irq == NULL) { 343 device_printf(dev, "could not map interrupt\n"); 344 goto bad1; 345 } 346 /* 347 * NB: Network code assumes we are blocked with splimp() 348 * so make sure the IRQ is mapped appropriately. 349 */ 350 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET, 351 ubsec_intr, sc, &sc->sc_ih)) { 352 device_printf(dev, "could not establish interrupt\n"); 353 goto bad2; 354 } 355 356 sc->sc_cid = crypto_get_driverid(0); 357 if (sc->sc_cid < 0) { 358 device_printf(dev, "could not get crypto driver id\n"); 359 goto bad3; 360 } 361 362 /* 363 * Setup DMA descriptor area. 364 */ 365 if (bus_dma_tag_create(NULL, /* parent */ 366 1, 0, /* alignment, bounds */ 367 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 368 BUS_SPACE_MAXADDR, /* highaddr */ 369 NULL, NULL, /* filter, filterarg */ 370 0x3ffff, /* maxsize */ 371 UBS_MAX_SCATTER, /* nsegments */ 372 0xffff, /* maxsegsize */ 373 BUS_DMA_ALLOCNOW, /* flags */ 374 &sc->sc_dmat)) { 375 device_printf(dev, "cannot allocate DMA tag\n"); 376 goto bad4; 377 } 378 SIMPLEQ_INIT(&sc->sc_freequeue); 379 dmap = sc->sc_dmaa; 380 for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) { 381 struct ubsec_q *q; 382 383 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q), 384 M_DEVBUF, M_NOWAIT); 385 if (q == NULL) { 386 device_printf(dev, "cannot allocate queue buffers\n"); 387 break; 388 } 389 390 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk), 391 &dmap->d_alloc, 0)) { 392 device_printf(dev, "cannot allocate dma buffers\n"); 393 free(q, M_DEVBUF); 394 break; 395 } 396 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr; 397 398 q->q_dma = dmap; 399 sc->sc_queuea[i] = q; 400 401 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 402 } 403 404 device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc)); 405 406 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0, 407 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 408 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0, 409 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 410 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0, 411 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 412 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0, 413 ubsec_newsession, ubsec_freesession, ubsec_process, sc); 414 415 /* 416 * Reset Broadcom chip 417 */ 418 ubsec_reset_board(sc); 419 420 /* 421 * Init Broadcom specific PCI settings 422 */ 423 ubsec_init_pciregs(dev); 424 425 /* 426 * Init Broadcom chip 427 */ 428 ubsec_init_board(sc); 429 430 #ifndef UBSEC_NO_RNG 431 if (sc->sc_flags & UBS_FLAGS_RNG) { 432 sc->sc_statmask |= BS_STAT_MCR2_DONE; 433 #ifdef UBSEC_RNDTEST 434 sc->sc_rndtest = rndtest_attach(dev); 435 if (sc->sc_rndtest) 436 sc->sc_harvest = rndtest_harvest; 437 else 438 sc->sc_harvest = default_harvest; 439 #else 440 sc->sc_harvest = default_harvest; 441 #endif 442 443 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 444 &sc->sc_rng.rng_q.q_mcr, 0)) 445 goto skip_rng; 446 447 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass), 448 &sc->sc_rng.rng_q.q_ctx, 0)) { 449 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 450 goto skip_rng; 451 } 452 453 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) * 454 UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) { 455 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 456 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 457 goto skip_rng; 458 } 459 460 if (hz >= 100) 461 sc->sc_rnghz = hz / 100; 462 else 463 sc->sc_rnghz = 1; 464 callout_init(&sc->sc_rngto); 465 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 466 skip_rng: 467 ; 468 } 469 #endif /* UBSEC_NO_RNG */ 470 471 if (sc->sc_flags & UBS_FLAGS_KEY) { 472 sc->sc_statmask |= BS_STAT_MCR2_DONE; 473 474 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0, 475 ubsec_kprocess, sc); 476 #if 0 477 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0, 478 ubsec_kprocess, sc); 479 #endif 480 } 481 return (0); 482 bad4: 483 crypto_unregister_all(sc->sc_cid); 484 bad3: 485 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 486 bad2: 487 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 488 bad1: 489 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 490 bad: 491 return (ENXIO); 492 } 493 494 /* 495 * Detach a device that successfully probed. 496 */ 497 static int 498 ubsec_detach(device_t dev) 499 { 500 struct ubsec_softc *sc = device_get_softc(dev); 501 int s; 502 503 KASSERT(sc != NULL, ("ubsec_detach: null software carrier")); 504 505 /* XXX wait/abort active ops */ 506 507 s = splimp(); 508 509 callout_stop(&sc->sc_rngto); 510 511 crypto_unregister_all(sc->sc_cid); 512 513 #ifdef UBSEC_RNDTEST 514 if (sc->sc_rndtest) 515 rndtest_detach(sc->sc_rndtest); 516 #endif 517 518 while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 519 struct ubsec_q *q; 520 521 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 522 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 523 ubsec_dma_free(sc, &q->q_dma->d_alloc); 524 free(q, M_DEVBUF); 525 } 526 #ifndef UBSEC_NO_RNG 527 if (sc->sc_flags & UBS_FLAGS_RNG) { 528 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr); 529 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx); 530 ubsec_dma_free(sc, &sc->sc_rng.rng_buf); 531 } 532 #endif /* UBSEC_NO_RNG */ 533 534 bus_generic_detach(dev); 535 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih); 536 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq); 537 538 bus_dma_tag_destroy(sc->sc_dmat); 539 bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr); 540 541 splx(s); 542 543 return (0); 544 } 545 546 /* 547 * Stop all chip i/o so that the kernel's probe routines don't 548 * get confused by errant DMAs when rebooting. 549 */ 550 static void 551 ubsec_shutdown(device_t dev) 552 { 553 #ifdef notyet 554 ubsec_stop(device_get_softc(dev)); 555 #endif 556 } 557 558 /* 559 * Device suspend routine. 560 */ 561 static int 562 ubsec_suspend(device_t dev) 563 { 564 struct ubsec_softc *sc = device_get_softc(dev); 565 566 KASSERT(sc != NULL, ("ubsec_suspend: null software carrier")); 567 #ifdef notyet 568 /* XXX stop the device and save PCI settings */ 569 #endif 570 sc->sc_suspended = 1; 571 572 return (0); 573 } 574 575 static int 576 ubsec_resume(device_t dev) 577 { 578 struct ubsec_softc *sc = device_get_softc(dev); 579 580 KASSERT(sc != NULL, ("ubsec_resume: null software carrier")); 581 #ifdef notyet 582 /* XXX retore PCI settings and start the device */ 583 #endif 584 sc->sc_suspended = 0; 585 return (0); 586 } 587 588 /* 589 * UBSEC Interrupt routine 590 */ 591 static void 592 ubsec_intr(void *arg) 593 { 594 struct ubsec_softc *sc = arg; 595 volatile u_int32_t stat; 596 struct ubsec_q *q; 597 struct ubsec_dma *dmap; 598 int npkts = 0, i; 599 600 stat = READ_REG(sc, BS_STAT); 601 stat &= sc->sc_statmask; 602 if (stat == 0) { 603 return; 604 } 605 606 WRITE_REG(sc, BS_STAT, stat); /* IACK */ 607 608 /* 609 * Check to see if we have any packets waiting for us 610 */ 611 if ((stat & BS_STAT_MCR1_DONE)) { 612 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 613 q = SIMPLEQ_FIRST(&sc->sc_qchip); 614 dmap = q->q_dma; 615 616 if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0) 617 break; 618 619 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 620 621 npkts = q->q_nstacked_mcrs; 622 sc->sc_nqchip -= 1+npkts; 623 /* 624 * search for further sc_qchip ubsec_q's that share 625 * the same MCR, and complete them too, they must be 626 * at the top. 627 */ 628 for (i = 0; i < npkts; i++) { 629 if(q->q_stacked_mcr[i]) { 630 ubsec_callback(sc, q->q_stacked_mcr[i]); 631 } else { 632 break; 633 } 634 } 635 ubsec_callback(sc, q); 636 } 637 638 /* 639 * Don't send any more packet to chip if there has been 640 * a DMAERR. 641 */ 642 if (!(stat & BS_STAT_DMAERR)) 643 ubsec_feed(sc); 644 } 645 646 /* 647 * Check to see if we have any key setups/rng's waiting for us 648 */ 649 if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) && 650 (stat & BS_STAT_MCR2_DONE)) { 651 struct ubsec_q2 *q2; 652 struct ubsec_mcr *mcr; 653 654 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) { 655 q2 = SIMPLEQ_FIRST(&sc->sc_qchip2); 656 657 ubsec_dma_sync(&q2->q_mcr, 658 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 659 660 mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr; 661 if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) { 662 ubsec_dma_sync(&q2->q_mcr, 663 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 664 break; 665 } 666 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next); 667 ubsec_callback2(sc, q2); 668 /* 669 * Don't send any more packet to chip if there has been 670 * a DMAERR. 671 */ 672 if (!(stat & BS_STAT_DMAERR)) 673 ubsec_feed2(sc); 674 } 675 } 676 677 /* 678 * Check to see if we got any DMA Error 679 */ 680 if (stat & BS_STAT_DMAERR) { 681 #ifdef UBSEC_DEBUG 682 if (ubsec_debug) { 683 volatile u_int32_t a = READ_REG(sc, BS_ERR); 684 685 printf("dmaerr %s@%08x\n", 686 (a & BS_ERR_READ) ? "read" : "write", 687 a & BS_ERR_ADDR); 688 } 689 #endif /* UBSEC_DEBUG */ 690 ubsecstats.hst_dmaerr++; 691 ubsec_totalreset(sc); 692 ubsec_feed(sc); 693 } 694 695 if (sc->sc_needwakeup) { /* XXX check high watermark */ 696 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ); 697 #ifdef UBSEC_DEBUG 698 if (ubsec_debug) 699 device_printf(sc->sc_dev, "wakeup crypto (%x)\n", 700 sc->sc_needwakeup); 701 #endif /* UBSEC_DEBUG */ 702 sc->sc_needwakeup &= ~wakeup; 703 crypto_unblock(sc->sc_cid, wakeup); 704 } 705 } 706 707 /* 708 * ubsec_feed() - aggregate and post requests to chip 709 */ 710 static void 711 ubsec_feed(struct ubsec_softc *sc) 712 { 713 struct ubsec_q *q, *q2; 714 int npkts, i; 715 void *v; 716 u_int32_t stat; 717 718 /* 719 * Decide how many ops to combine in a single MCR. We cannot 720 * aggregate more than UBS_MAX_AGGR because this is the number 721 * of slots defined in the data structure. Note that 722 * aggregation only happens if ops are marked batch'able. 723 * Aggregating ops reduces the number of interrupts to the host 724 * but also (potentially) increases the latency for processing 725 * completed ops as we only get an interrupt when all aggregated 726 * ops have completed. 727 */ 728 if (sc->sc_nqueue == 0) 729 return; 730 if (sc->sc_nqueue > 1) { 731 npkts = 0; 732 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) { 733 npkts++; 734 if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0) 735 break; 736 } 737 } else 738 npkts = 1; 739 /* 740 * Check device status before going any further. 741 */ 742 if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) { 743 if (stat & BS_STAT_DMAERR) { 744 ubsec_totalreset(sc); 745 ubsecstats.hst_dmaerr++; 746 } else 747 ubsecstats.hst_mcr1full++; 748 return; 749 } 750 if (sc->sc_nqueue > ubsecstats.hst_maxqueue) 751 ubsecstats.hst_maxqueue = sc->sc_nqueue; 752 if (npkts > UBS_MAX_AGGR) 753 npkts = UBS_MAX_AGGR; 754 if (npkts < 2) /* special case 1 op */ 755 goto feed1; 756 757 ubsecstats.hst_totbatch += npkts-1; 758 #ifdef UBSEC_DEBUG 759 if (ubsec_debug) 760 printf("merging %d records\n", npkts); 761 #endif /* UBSEC_DEBUG */ 762 763 q = SIMPLEQ_FIRST(&sc->sc_queue); 764 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 765 --sc->sc_nqueue; 766 767 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 768 if (q->q_dst_map != NULL) 769 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 770 771 q->q_nstacked_mcrs = npkts - 1; /* Number of packets stacked */ 772 773 for (i = 0; i < q->q_nstacked_mcrs; i++) { 774 q2 = SIMPLEQ_FIRST(&sc->sc_queue); 775 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map, 776 BUS_DMASYNC_PREWRITE); 777 if (q2->q_dst_map != NULL) 778 bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map, 779 BUS_DMASYNC_PREREAD); 780 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next); 781 --sc->sc_nqueue; 782 783 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) - 784 sizeof(struct ubsec_mcr_add)); 785 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add)); 786 q->q_stacked_mcr[i] = q2; 787 } 788 q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts); 789 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 790 sc->sc_nqchip += npkts; 791 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 792 ubsecstats.hst_maxqchip = sc->sc_nqchip; 793 ubsec_dma_sync(&q->q_dma->d_alloc, 794 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 795 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 796 offsetof(struct ubsec_dmachunk, d_mcr)); 797 return; 798 799 feed1: 800 q = SIMPLEQ_FIRST(&sc->sc_queue); 801 802 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE); 803 if (q->q_dst_map != NULL) 804 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD); 805 ubsec_dma_sync(&q->q_dma->d_alloc, 806 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 807 808 WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr + 809 offsetof(struct ubsec_dmachunk, d_mcr)); 810 #ifdef UBSEC_DEBUG 811 if (ubsec_debug) 812 printf("feed1: q->chip %p %08x stat %08x\n", 813 q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr), 814 stat); 815 #endif /* UBSEC_DEBUG */ 816 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next); 817 --sc->sc_nqueue; 818 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next); 819 sc->sc_nqchip++; 820 if (sc->sc_nqchip > ubsecstats.hst_maxqchip) 821 ubsecstats.hst_maxqchip = sc->sc_nqchip; 822 return; 823 } 824 825 /* 826 * Allocate a new 'session' and return an encoded session id. 'sidp' 827 * contains our registration id, and should contain an encoded session 828 * id on successful allocation. 829 */ 830 static int 831 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri) 832 { 833 struct cryptoini *c, *encini = NULL, *macini = NULL; 834 struct ubsec_softc *sc = arg; 835 struct ubsec_session *ses = NULL; 836 MD5_CTX md5ctx; 837 SHA1_CTX sha1ctx; 838 int i, sesn; 839 840 KASSERT(sc != NULL, ("ubsec_newsession: null softc")); 841 if (sidp == NULL || cri == NULL || sc == NULL) 842 return (EINVAL); 843 844 for (c = cri; c != NULL; c = c->cri_next) { 845 if (c->cri_alg == CRYPTO_MD5_HMAC || 846 c->cri_alg == CRYPTO_SHA1_HMAC) { 847 if (macini) 848 return (EINVAL); 849 macini = c; 850 } else if (c->cri_alg == CRYPTO_DES_CBC || 851 c->cri_alg == CRYPTO_3DES_CBC) { 852 if (encini) 853 return (EINVAL); 854 encini = c; 855 } else 856 return (EINVAL); 857 } 858 if (encini == NULL && macini == NULL) 859 return (EINVAL); 860 861 if (sc->sc_sessions == NULL) { 862 ses = sc->sc_sessions = (struct ubsec_session *)malloc( 863 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 864 if (ses == NULL) 865 return (ENOMEM); 866 sesn = 0; 867 sc->sc_nsessions = 1; 868 } else { 869 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) { 870 if (sc->sc_sessions[sesn].ses_used == 0) { 871 ses = &sc->sc_sessions[sesn]; 872 break; 873 } 874 } 875 876 if (ses == NULL) { 877 sesn = sc->sc_nsessions; 878 ses = (struct ubsec_session *)malloc((sesn + 1) * 879 sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT); 880 if (ses == NULL) 881 return (ENOMEM); 882 bcopy(sc->sc_sessions, ses, sesn * 883 sizeof(struct ubsec_session)); 884 bzero(sc->sc_sessions, sesn * 885 sizeof(struct ubsec_session)); 886 free(sc->sc_sessions, M_DEVBUF); 887 sc->sc_sessions = ses; 888 ses = &sc->sc_sessions[sesn]; 889 sc->sc_nsessions++; 890 } 891 } 892 893 bzero(ses, sizeof(struct ubsec_session)); 894 ses->ses_used = 1; 895 if (encini) { 896 /* get an IV, network byte order */ 897 /* XXX may read fewer than requested */ 898 read_random(ses->ses_iv, sizeof(ses->ses_iv)); 899 900 /* Go ahead and compute key in ubsec's byte order */ 901 if (encini->cri_alg == CRYPTO_DES_CBC) { 902 bcopy(encini->cri_key, &ses->ses_deskey[0], 8); 903 bcopy(encini->cri_key, &ses->ses_deskey[2], 8); 904 bcopy(encini->cri_key, &ses->ses_deskey[4], 8); 905 } else 906 bcopy(encini->cri_key, ses->ses_deskey, 24); 907 908 SWAP32(ses->ses_deskey[0]); 909 SWAP32(ses->ses_deskey[1]); 910 SWAP32(ses->ses_deskey[2]); 911 SWAP32(ses->ses_deskey[3]); 912 SWAP32(ses->ses_deskey[4]); 913 SWAP32(ses->ses_deskey[5]); 914 } 915 916 if (macini) { 917 for (i = 0; i < macini->cri_klen / 8; i++) 918 macini->cri_key[i] ^= HMAC_IPAD_VAL; 919 920 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 921 MD5Init(&md5ctx); 922 MD5Update(&md5ctx, macini->cri_key, 923 macini->cri_klen / 8); 924 MD5Update(&md5ctx, hmac_ipad_buffer, 925 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 926 bcopy(md5ctx.state, ses->ses_hminner, 927 sizeof(md5ctx.state)); 928 } else { 929 SHA1Init(&sha1ctx); 930 SHA1Update(&sha1ctx, macini->cri_key, 931 macini->cri_klen / 8); 932 SHA1Update(&sha1ctx, hmac_ipad_buffer, 933 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 934 bcopy(sha1ctx.h.b32, ses->ses_hminner, 935 sizeof(sha1ctx.h.b32)); 936 } 937 938 for (i = 0; i < macini->cri_klen / 8; i++) 939 macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 940 941 if (macini->cri_alg == CRYPTO_MD5_HMAC) { 942 MD5Init(&md5ctx); 943 MD5Update(&md5ctx, macini->cri_key, 944 macini->cri_klen / 8); 945 MD5Update(&md5ctx, hmac_opad_buffer, 946 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 947 bcopy(md5ctx.state, ses->ses_hmouter, 948 sizeof(md5ctx.state)); 949 } else { 950 SHA1Init(&sha1ctx); 951 SHA1Update(&sha1ctx, macini->cri_key, 952 macini->cri_klen / 8); 953 SHA1Update(&sha1ctx, hmac_opad_buffer, 954 HMAC_BLOCK_LEN - (macini->cri_klen / 8)); 955 bcopy(sha1ctx.h.b32, ses->ses_hmouter, 956 sizeof(sha1ctx.h.b32)); 957 } 958 959 for (i = 0; i < macini->cri_klen / 8; i++) 960 macini->cri_key[i] ^= HMAC_OPAD_VAL; 961 } 962 963 *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn); 964 return (0); 965 } 966 967 /* 968 * Deallocate a session. 969 */ 970 static int 971 ubsec_freesession(void *arg, u_int64_t tid) 972 { 973 struct ubsec_softc *sc = arg; 974 int session; 975 u_int32_t sid = ((u_int32_t) tid) & 0xffffffff; 976 977 KASSERT(sc != NULL, ("ubsec_freesession: null softc")); 978 if (sc == NULL) 979 return (EINVAL); 980 981 session = UBSEC_SESSION(sid); 982 if (session >= sc->sc_nsessions) 983 return (EINVAL); 984 985 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session])); 986 return (0); 987 } 988 989 static void 990 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) 991 { 992 struct ubsec_operand *op = arg; 993 994 KASSERT(nsegs <= UBS_MAX_SCATTER, 995 ("Too many DMA segments returned when mapping operand")); 996 #ifdef UBSEC_DEBUG 997 if (ubsec_debug) 998 printf("ubsec_op_cb: mapsize %u nsegs %d\n", 999 (u_int) mapsize, nsegs); 1000 #endif 1001 op->mapsize = mapsize; 1002 op->nsegs = nsegs; 1003 bcopy(seg, op->segs, nsegs * sizeof (seg[0])); 1004 } 1005 1006 static int 1007 ubsec_process(void *arg, struct cryptop *crp, int hint) 1008 { 1009 struct ubsec_q *q = NULL; 1010 int err = 0, i, j, s, nicealign; 1011 struct ubsec_softc *sc = arg; 1012 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd; 1013 int encoffset = 0, macoffset = 0, cpskip, cpoffset; 1014 int sskip, dskip, stheend, dtheend; 1015 int16_t coffset; 1016 struct ubsec_session *ses; 1017 struct ubsec_pktctx ctx; 1018 struct ubsec_dma *dmap = NULL; 1019 1020 if (crp == NULL || crp->crp_callback == NULL || sc == NULL) { 1021 ubsecstats.hst_invalid++; 1022 return (EINVAL); 1023 } 1024 if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) { 1025 ubsecstats.hst_badsession++; 1026 return (EINVAL); 1027 } 1028 1029 s = splimp(); 1030 1031 if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) { 1032 ubsecstats.hst_queuefull++; 1033 sc->sc_needwakeup |= CRYPTO_SYMQ; 1034 splx(s); 1035 return (ERESTART); 1036 } 1037 q = SIMPLEQ_FIRST(&sc->sc_freequeue); 1038 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next); 1039 splx(s); 1040 1041 dmap = q->q_dma; /* Save dma pointer */ 1042 bzero(q, sizeof(struct ubsec_q)); 1043 bzero(&ctx, sizeof(ctx)); 1044 1045 q->q_sesn = UBSEC_SESSION(crp->crp_sid); 1046 q->q_dma = dmap; 1047 ses = &sc->sc_sessions[q->q_sesn]; 1048 1049 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1050 q->q_src_m = (struct mbuf *)crp->crp_buf; 1051 q->q_dst_m = (struct mbuf *)crp->crp_buf; 1052 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1053 q->q_src_io = (struct uio *)crp->crp_buf; 1054 q->q_dst_io = (struct uio *)crp->crp_buf; 1055 } else { 1056 ubsecstats.hst_badflags++; 1057 err = EINVAL; 1058 goto errout; /* XXX we don't handle contiguous blocks! */ 1059 } 1060 1061 bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr)); 1062 1063 dmap->d_dma->d_mcr.mcr_pkts = htole16(1); 1064 dmap->d_dma->d_mcr.mcr_flags = 0; 1065 q->q_crp = crp; 1066 1067 crd1 = crp->crp_desc; 1068 if (crd1 == NULL) { 1069 ubsecstats.hst_nodesc++; 1070 err = EINVAL; 1071 goto errout; 1072 } 1073 crd2 = crd1->crd_next; 1074 1075 if (crd2 == NULL) { 1076 if (crd1->crd_alg == CRYPTO_MD5_HMAC || 1077 crd1->crd_alg == CRYPTO_SHA1_HMAC) { 1078 maccrd = crd1; 1079 enccrd = NULL; 1080 } else if (crd1->crd_alg == CRYPTO_DES_CBC || 1081 crd1->crd_alg == CRYPTO_3DES_CBC) { 1082 maccrd = NULL; 1083 enccrd = crd1; 1084 } else { 1085 ubsecstats.hst_badalg++; 1086 err = EINVAL; 1087 goto errout; 1088 } 1089 } else { 1090 if ((crd1->crd_alg == CRYPTO_MD5_HMAC || 1091 crd1->crd_alg == CRYPTO_SHA1_HMAC) && 1092 (crd2->crd_alg == CRYPTO_DES_CBC || 1093 crd2->crd_alg == CRYPTO_3DES_CBC) && 1094 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) { 1095 maccrd = crd1; 1096 enccrd = crd2; 1097 } else if ((crd1->crd_alg == CRYPTO_DES_CBC || 1098 crd1->crd_alg == CRYPTO_3DES_CBC) && 1099 (crd2->crd_alg == CRYPTO_MD5_HMAC || 1100 crd2->crd_alg == CRYPTO_SHA1_HMAC) && 1101 (crd1->crd_flags & CRD_F_ENCRYPT)) { 1102 enccrd = crd1; 1103 maccrd = crd2; 1104 } else { 1105 /* 1106 * We cannot order the ubsec as requested 1107 */ 1108 ubsecstats.hst_badalg++; 1109 err = EINVAL; 1110 goto errout; 1111 } 1112 } 1113 1114 if (enccrd) { 1115 encoffset = enccrd->crd_skip; 1116 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES); 1117 1118 if (enccrd->crd_flags & CRD_F_ENCRYPT) { 1119 q->q_flags |= UBSEC_QFLAGS_COPYOUTIV; 1120 1121 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1122 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1123 else { 1124 ctx.pc_iv[0] = ses->ses_iv[0]; 1125 ctx.pc_iv[1] = ses->ses_iv[1]; 1126 } 1127 1128 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) { 1129 if (crp->crp_flags & CRYPTO_F_IMBUF) 1130 m_copyback(q->q_src_m, 1131 enccrd->crd_inject, 1132 8, (caddr_t)ctx.pc_iv); 1133 else if (crp->crp_flags & CRYPTO_F_IOV) 1134 cuio_copyback(q->q_src_io, 1135 enccrd->crd_inject, 1136 8, (caddr_t)ctx.pc_iv); 1137 } 1138 } else { 1139 ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND); 1140 1141 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) 1142 bcopy(enccrd->crd_iv, ctx.pc_iv, 8); 1143 else if (crp->crp_flags & CRYPTO_F_IMBUF) 1144 m_copydata(q->q_src_m, enccrd->crd_inject, 1145 8, (caddr_t)ctx.pc_iv); 1146 else if (crp->crp_flags & CRYPTO_F_IOV) 1147 cuio_copydata(q->q_src_io, 1148 enccrd->crd_inject, 8, 1149 (caddr_t)ctx.pc_iv); 1150 } 1151 1152 ctx.pc_deskey[0] = ses->ses_deskey[0]; 1153 ctx.pc_deskey[1] = ses->ses_deskey[1]; 1154 ctx.pc_deskey[2] = ses->ses_deskey[2]; 1155 ctx.pc_deskey[3] = ses->ses_deskey[3]; 1156 ctx.pc_deskey[4] = ses->ses_deskey[4]; 1157 ctx.pc_deskey[5] = ses->ses_deskey[5]; 1158 SWAP32(ctx.pc_iv[0]); 1159 SWAP32(ctx.pc_iv[1]); 1160 } 1161 1162 if (maccrd) { 1163 macoffset = maccrd->crd_skip; 1164 1165 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) 1166 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5); 1167 else 1168 ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1); 1169 1170 for (i = 0; i < 5; i++) { 1171 ctx.pc_hminner[i] = ses->ses_hminner[i]; 1172 ctx.pc_hmouter[i] = ses->ses_hmouter[i]; 1173 1174 HTOLE32(ctx.pc_hminner[i]); 1175 HTOLE32(ctx.pc_hmouter[i]); 1176 } 1177 } 1178 1179 if (enccrd && maccrd) { 1180 /* 1181 * ubsec cannot handle packets where the end of encryption 1182 * and authentication are not the same, or where the 1183 * encrypted part begins before the authenticated part. 1184 */ 1185 if ((encoffset + enccrd->crd_len) != 1186 (macoffset + maccrd->crd_len)) { 1187 ubsecstats.hst_lenmismatch++; 1188 err = EINVAL; 1189 goto errout; 1190 } 1191 if (enccrd->crd_skip < maccrd->crd_skip) { 1192 ubsecstats.hst_skipmismatch++; 1193 err = EINVAL; 1194 goto errout; 1195 } 1196 sskip = maccrd->crd_skip; 1197 cpskip = dskip = enccrd->crd_skip; 1198 stheend = maccrd->crd_len; 1199 dtheend = enccrd->crd_len; 1200 coffset = enccrd->crd_skip - maccrd->crd_skip; 1201 cpoffset = cpskip + dtheend; 1202 #ifdef UBSEC_DEBUG 1203 if (ubsec_debug) { 1204 printf("mac: skip %d, len %d, inject %d\n", 1205 maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject); 1206 printf("enc: skip %d, len %d, inject %d\n", 1207 enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject); 1208 printf("src: skip %d, len %d\n", sskip, stheend); 1209 printf("dst: skip %d, len %d\n", dskip, dtheend); 1210 printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n", 1211 coffset, stheend, cpskip, cpoffset); 1212 } 1213 #endif 1214 } else { 1215 cpskip = dskip = sskip = macoffset + encoffset; 1216 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len; 1217 cpoffset = cpskip + dtheend; 1218 coffset = 0; 1219 } 1220 ctx.pc_offset = htole16(coffset >> 2); 1221 1222 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) { 1223 ubsecstats.hst_nomap++; 1224 err = ENOMEM; 1225 goto errout; 1226 } 1227 if (crp->crp_flags & CRYPTO_F_IMBUF) { 1228 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map, 1229 q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1230 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1231 q->q_src_map = NULL; 1232 ubsecstats.hst_noload++; 1233 err = ENOMEM; 1234 goto errout; 1235 } 1236 } else if (crp->crp_flags & CRYPTO_F_IOV) { 1237 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map, 1238 q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) { 1239 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1240 q->q_src_map = NULL; 1241 ubsecstats.hst_noload++; 1242 err = ENOMEM; 1243 goto errout; 1244 } 1245 } 1246 nicealign = ubsec_dmamap_aligned(&q->q_src); 1247 1248 dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend); 1249 1250 #ifdef UBSEC_DEBUG 1251 if (ubsec_debug) 1252 printf("src skip: %d nicealign: %u\n", sskip, nicealign); 1253 #endif 1254 for (i = j = 0; i < q->q_src_nsegs; i++) { 1255 struct ubsec_pktbuf *pb; 1256 bus_size_t packl = q->q_src_segs[i].ds_len; 1257 bus_addr_t packp = q->q_src_segs[i].ds_addr; 1258 1259 if (sskip >= packl) { 1260 sskip -= packl; 1261 continue; 1262 } 1263 1264 packl -= sskip; 1265 packp += sskip; 1266 sskip = 0; 1267 1268 if (packl > 0xfffc) { 1269 err = EIO; 1270 goto errout; 1271 } 1272 1273 if (j == 0) 1274 pb = &dmap->d_dma->d_mcr.mcr_ipktbuf; 1275 else 1276 pb = &dmap->d_dma->d_sbuf[j - 1]; 1277 1278 pb->pb_addr = htole32(packp); 1279 1280 if (stheend) { 1281 if (packl > stheend) { 1282 pb->pb_len = htole32(stheend); 1283 stheend = 0; 1284 } else { 1285 pb->pb_len = htole32(packl); 1286 stheend -= packl; 1287 } 1288 } else 1289 pb->pb_len = htole32(packl); 1290 1291 if ((i + 1) == q->q_src_nsegs) 1292 pb->pb_next = 0; 1293 else 1294 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1295 offsetof(struct ubsec_dmachunk, d_sbuf[j])); 1296 j++; 1297 } 1298 1299 if (enccrd == NULL && maccrd != NULL) { 1300 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0; 1301 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0; 1302 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr + 1303 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1304 #ifdef UBSEC_DEBUG 1305 if (ubsec_debug) 1306 printf("opkt: %x %x %x\n", 1307 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr, 1308 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len, 1309 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next); 1310 #endif 1311 } else { 1312 if (crp->crp_flags & CRYPTO_F_IOV) { 1313 if (!nicealign) { 1314 ubsecstats.hst_iovmisaligned++; 1315 err = EINVAL; 1316 goto errout; 1317 } 1318 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, 1319 &q->q_dst_map)) { 1320 ubsecstats.hst_nomap++; 1321 err = ENOMEM; 1322 goto errout; 1323 } 1324 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map, 1325 q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) { 1326 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1327 q->q_dst_map = NULL; 1328 ubsecstats.hst_noload++; 1329 err = ENOMEM; 1330 goto errout; 1331 } 1332 } else if (crp->crp_flags & CRYPTO_F_IMBUF) { 1333 if (nicealign) { 1334 q->q_dst = q->q_src; 1335 } else { 1336 int totlen, len; 1337 struct mbuf *m, *top, **mp; 1338 1339 ubsecstats.hst_unaligned++; 1340 totlen = q->q_src_mapsize; 1341 if (q->q_src_m->m_flags & M_PKTHDR) { 1342 len = MHLEN; 1343 MGETHDR(m, M_DONTWAIT, MT_DATA); 1344 if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) { 1345 m_free(m); 1346 m = NULL; 1347 } 1348 } else { 1349 len = MLEN; 1350 MGET(m, M_DONTWAIT, MT_DATA); 1351 } 1352 if (m == NULL) { 1353 ubsecstats.hst_nombuf++; 1354 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1355 goto errout; 1356 } 1357 if (totlen >= MINCLSIZE) { 1358 MCLGET(m, M_DONTWAIT); 1359 if ((m->m_flags & M_EXT) == 0) { 1360 m_free(m); 1361 ubsecstats.hst_nomcl++; 1362 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1363 goto errout; 1364 } 1365 len = MCLBYTES; 1366 } 1367 m->m_len = len; 1368 top = NULL; 1369 mp = ⊤ 1370 1371 while (totlen > 0) { 1372 if (top) { 1373 MGET(m, M_DONTWAIT, MT_DATA); 1374 if (m == NULL) { 1375 m_freem(top); 1376 ubsecstats.hst_nombuf++; 1377 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1378 goto errout; 1379 } 1380 len = MLEN; 1381 } 1382 if (top && totlen >= MINCLSIZE) { 1383 MCLGET(m, M_DONTWAIT); 1384 if ((m->m_flags & M_EXT) == 0) { 1385 *mp = m; 1386 m_freem(top); 1387 ubsecstats.hst_nomcl++; 1388 err = sc->sc_nqueue ? ERESTART : ENOMEM; 1389 goto errout; 1390 } 1391 len = MCLBYTES; 1392 } 1393 m->m_len = len = min(totlen, len); 1394 totlen -= len; 1395 *mp = m; 1396 mp = &m->m_next; 1397 } 1398 q->q_dst_m = top; 1399 ubsec_mcopy(q->q_src_m, q->q_dst_m, 1400 cpskip, cpoffset); 1401 if (bus_dmamap_create(sc->sc_dmat, 1402 BUS_DMA_NOWAIT, &q->q_dst_map) != 0) { 1403 ubsecstats.hst_nomap++; 1404 err = ENOMEM; 1405 goto errout; 1406 } 1407 if (bus_dmamap_load_mbuf(sc->sc_dmat, 1408 q->q_dst_map, q->q_dst_m, 1409 ubsec_op_cb, &q->q_dst, 1410 BUS_DMA_NOWAIT) != 0) { 1411 bus_dmamap_destroy(sc->sc_dmat, 1412 q->q_dst_map); 1413 q->q_dst_map = NULL; 1414 ubsecstats.hst_noload++; 1415 err = ENOMEM; 1416 goto errout; 1417 } 1418 } 1419 } else { 1420 ubsecstats.hst_badflags++; 1421 err = EINVAL; 1422 goto errout; 1423 } 1424 1425 #ifdef UBSEC_DEBUG 1426 if (ubsec_debug) 1427 printf("dst skip: %d\n", dskip); 1428 #endif 1429 for (i = j = 0; i < q->q_dst_nsegs; i++) { 1430 struct ubsec_pktbuf *pb; 1431 bus_size_t packl = q->q_dst_segs[i].ds_len; 1432 bus_addr_t packp = q->q_dst_segs[i].ds_addr; 1433 1434 if (dskip >= packl) { 1435 dskip -= packl; 1436 continue; 1437 } 1438 1439 packl -= dskip; 1440 packp += dskip; 1441 dskip = 0; 1442 1443 if (packl > 0xfffc) { 1444 err = EIO; 1445 goto errout; 1446 } 1447 1448 if (j == 0) 1449 pb = &dmap->d_dma->d_mcr.mcr_opktbuf; 1450 else 1451 pb = &dmap->d_dma->d_dbuf[j - 1]; 1452 1453 pb->pb_addr = htole32(packp); 1454 1455 if (dtheend) { 1456 if (packl > dtheend) { 1457 pb->pb_len = htole32(dtheend); 1458 dtheend = 0; 1459 } else { 1460 pb->pb_len = htole32(packl); 1461 dtheend -= packl; 1462 } 1463 } else 1464 pb->pb_len = htole32(packl); 1465 1466 if ((i + 1) == q->q_dst_nsegs) { 1467 if (maccrd) 1468 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1469 offsetof(struct ubsec_dmachunk, d_macbuf[0])); 1470 else 1471 pb->pb_next = 0; 1472 } else 1473 pb->pb_next = htole32(dmap->d_alloc.dma_paddr + 1474 offsetof(struct ubsec_dmachunk, d_dbuf[j])); 1475 j++; 1476 } 1477 } 1478 1479 dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr + 1480 offsetof(struct ubsec_dmachunk, d_ctx)); 1481 1482 if (sc->sc_flags & UBS_FLAGS_LONGCTX) { 1483 struct ubsec_pktctx_long *ctxl; 1484 1485 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr + 1486 offsetof(struct ubsec_dmachunk, d_ctx)); 1487 1488 /* transform small context into long context */ 1489 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long)); 1490 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC); 1491 ctxl->pc_flags = ctx.pc_flags; 1492 ctxl->pc_offset = ctx.pc_offset; 1493 for (i = 0; i < 6; i++) 1494 ctxl->pc_deskey[i] = ctx.pc_deskey[i]; 1495 for (i = 0; i < 5; i++) 1496 ctxl->pc_hminner[i] = ctx.pc_hminner[i]; 1497 for (i = 0; i < 5; i++) 1498 ctxl->pc_hmouter[i] = ctx.pc_hmouter[i]; 1499 ctxl->pc_iv[0] = ctx.pc_iv[0]; 1500 ctxl->pc_iv[1] = ctx.pc_iv[1]; 1501 } else 1502 bcopy(&ctx, dmap->d_alloc.dma_vaddr + 1503 offsetof(struct ubsec_dmachunk, d_ctx), 1504 sizeof(struct ubsec_pktctx)); 1505 1506 s = splimp(); 1507 SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next); 1508 sc->sc_nqueue++; 1509 ubsecstats.hst_ipackets++; 1510 ubsecstats.hst_ibytes += dmap->d_alloc.dma_size; 1511 if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR) 1512 ubsec_feed(sc); 1513 splx(s); 1514 return (0); 1515 1516 errout: 1517 if (q != NULL) { 1518 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 1519 m_freem(q->q_dst_m); 1520 1521 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1522 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1523 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1524 } 1525 if (q->q_src_map != NULL) { 1526 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1527 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1528 } 1529 1530 s = splimp(); 1531 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1532 splx(s); 1533 } 1534 if (err != ERESTART) { 1535 crp->crp_etype = err; 1536 crypto_done(crp); 1537 } else { 1538 sc->sc_needwakeup |= CRYPTO_SYMQ; 1539 } 1540 return (err); 1541 } 1542 1543 static void 1544 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q) 1545 { 1546 struct cryptop *crp = (struct cryptop *)q->q_crp; 1547 struct cryptodesc *crd; 1548 struct ubsec_dma *dmap = q->q_dma; 1549 1550 ubsecstats.hst_opackets++; 1551 ubsecstats.hst_obytes += dmap->d_alloc.dma_size; 1552 1553 ubsec_dma_sync(&dmap->d_alloc, 1554 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE); 1555 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) { 1556 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, 1557 BUS_DMASYNC_POSTREAD); 1558 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map); 1559 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map); 1560 } 1561 bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE); 1562 bus_dmamap_unload(sc->sc_dmat, q->q_src_map); 1563 bus_dmamap_destroy(sc->sc_dmat, q->q_src_map); 1564 1565 if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) { 1566 m_freem(q->q_src_m); 1567 crp->crp_buf = (caddr_t)q->q_dst_m; 1568 } 1569 ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len; 1570 1571 /* copy out IV for future use */ 1572 if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) { 1573 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1574 if (crd->crd_alg != CRYPTO_DES_CBC && 1575 crd->crd_alg != CRYPTO_3DES_CBC) 1576 continue; 1577 if (crp->crp_flags & CRYPTO_F_IMBUF) 1578 m_copydata((struct mbuf *)crp->crp_buf, 1579 crd->crd_skip + crd->crd_len - 8, 8, 1580 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1581 else if (crp->crp_flags & CRYPTO_F_IOV) { 1582 cuio_copydata((struct uio *)crp->crp_buf, 1583 crd->crd_skip + crd->crd_len - 8, 8, 1584 (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv); 1585 } 1586 break; 1587 } 1588 } 1589 1590 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1591 if (crd->crd_alg != CRYPTO_MD5_HMAC && 1592 crd->crd_alg != CRYPTO_SHA1_HMAC) 1593 continue; 1594 if (crp->crp_flags & CRYPTO_F_IMBUF) 1595 m_copyback((struct mbuf *)crp->crp_buf, 1596 crd->crd_inject, 12, 1597 (caddr_t)dmap->d_dma->d_macbuf); 1598 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac) 1599 bcopy((caddr_t)dmap->d_dma->d_macbuf, 1600 crp->crp_mac, 12); 1601 break; 1602 } 1603 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 1604 crypto_done(crp); 1605 } 1606 1607 static void 1608 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset) 1609 { 1610 int i, j, dlen, slen; 1611 caddr_t dptr, sptr; 1612 1613 j = 0; 1614 sptr = srcm->m_data; 1615 slen = srcm->m_len; 1616 dptr = dstm->m_data; 1617 dlen = dstm->m_len; 1618 1619 while (1) { 1620 for (i = 0; i < min(slen, dlen); i++) { 1621 if (j < hoffset || j >= toffset) 1622 *dptr++ = *sptr++; 1623 slen--; 1624 dlen--; 1625 j++; 1626 } 1627 if (slen == 0) { 1628 srcm = srcm->m_next; 1629 if (srcm == NULL) 1630 return; 1631 sptr = srcm->m_data; 1632 slen = srcm->m_len; 1633 } 1634 if (dlen == 0) { 1635 dstm = dstm->m_next; 1636 if (dstm == NULL) 1637 return; 1638 dptr = dstm->m_data; 1639 dlen = dstm->m_len; 1640 } 1641 } 1642 } 1643 1644 /* 1645 * feed the key generator, must be called at splimp() or higher. 1646 */ 1647 static int 1648 ubsec_feed2(struct ubsec_softc *sc) 1649 { 1650 struct ubsec_q2 *q; 1651 1652 while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) { 1653 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL) 1654 break; 1655 q = SIMPLEQ_FIRST(&sc->sc_queue2); 1656 1657 ubsec_dma_sync(&q->q_mcr, 1658 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1659 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE); 1660 1661 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr); 1662 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next); 1663 --sc->sc_nqueue2; 1664 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next); 1665 } 1666 return (0); 1667 } 1668 1669 /* 1670 * Callback for handling random numbers 1671 */ 1672 static void 1673 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q) 1674 { 1675 struct cryptkop *krp; 1676 struct ubsec_ctx_keyop *ctx; 1677 1678 ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr; 1679 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE); 1680 1681 switch (q->q_type) { 1682 #ifndef UBSEC_NO_RNG 1683 case UBS_CTXOP_RNGBYPASS: { 1684 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q; 1685 1686 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD); 1687 (*sc->sc_harvest)(sc->sc_rndtest, 1688 rng->rng_buf.dma_vaddr, 1689 UBSEC_RNG_BUFSIZ*sizeof (u_int32_t)); 1690 rng->rng_used = 0; 1691 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1692 break; 1693 } 1694 #endif 1695 case UBS_CTXOP_MODEXP: { 1696 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 1697 u_int rlen, clen; 1698 1699 krp = me->me_krp; 1700 rlen = (me->me_modbits + 7) / 8; 1701 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8; 1702 1703 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE); 1704 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE); 1705 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD); 1706 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE); 1707 1708 if (clen < rlen) 1709 krp->krp_status = E2BIG; 1710 else { 1711 if (sc->sc_flags & UBS_FLAGS_HWNORM) { 1712 bzero(krp->krp_param[krp->krp_iparams].crp_p, 1713 (krp->krp_param[krp->krp_iparams].crp_nbits 1714 + 7) / 8); 1715 bcopy(me->me_C.dma_vaddr, 1716 krp->krp_param[krp->krp_iparams].crp_p, 1717 (me->me_modbits + 7) / 8); 1718 } else 1719 ubsec_kshift_l(me->me_shiftbits, 1720 me->me_C.dma_vaddr, me->me_normbits, 1721 krp->krp_param[krp->krp_iparams].crp_p, 1722 krp->krp_param[krp->krp_iparams].crp_nbits); 1723 } 1724 1725 crypto_kdone(krp); 1726 1727 /* bzero all potentially sensitive data */ 1728 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 1729 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 1730 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 1731 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 1732 1733 /* Can't free here, so put us on the free list. */ 1734 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next); 1735 break; 1736 } 1737 case UBS_CTXOP_RSAPRIV: { 1738 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 1739 u_int len; 1740 1741 krp = rp->rpr_krp; 1742 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE); 1743 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD); 1744 1745 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8; 1746 bcopy(rp->rpr_msgout.dma_vaddr, 1747 krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len); 1748 1749 crypto_kdone(krp); 1750 1751 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 1752 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 1753 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size); 1754 1755 /* Can't free here, so put us on the free list. */ 1756 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next); 1757 break; 1758 } 1759 default: 1760 device_printf(sc->sc_dev, "unknown ctx op: %x\n", 1761 letoh16(ctx->ctx_op)); 1762 break; 1763 } 1764 } 1765 1766 #ifndef UBSEC_NO_RNG 1767 static void 1768 ubsec_rng(void *vsc) 1769 { 1770 struct ubsec_softc *sc = vsc; 1771 struct ubsec_q2_rng *rng = &sc->sc_rng; 1772 struct ubsec_mcr *mcr; 1773 struct ubsec_ctx_rngbypass *ctx; 1774 int s; 1775 1776 s = splimp(); 1777 if (rng->rng_used) { 1778 splx(s); 1779 return; 1780 } 1781 sc->sc_nqueue2++; 1782 if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE) 1783 goto out; 1784 1785 mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr; 1786 ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr; 1787 1788 mcr->mcr_pkts = htole16(1); 1789 mcr->mcr_flags = 0; 1790 mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr); 1791 mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0; 1792 mcr->mcr_ipktbuf.pb_len = 0; 1793 mcr->mcr_reserved = mcr->mcr_pktlen = 0; 1794 mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr); 1795 mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) & 1796 UBS_PKTBUF_LEN); 1797 mcr->mcr_opktbuf.pb_next = 0; 1798 1799 ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass)); 1800 ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS); 1801 rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS; 1802 1803 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD); 1804 1805 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next); 1806 rng->rng_used = 1; 1807 ubsec_feed2(sc); 1808 ubsecstats.hst_rng++; 1809 splx(s); 1810 1811 return; 1812 1813 out: 1814 /* 1815 * Something weird happened, generate our own call back. 1816 */ 1817 sc->sc_nqueue2--; 1818 splx(s); 1819 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc); 1820 } 1821 #endif /* UBSEC_NO_RNG */ 1822 1823 static void 1824 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1825 { 1826 bus_addr_t *paddr = (bus_addr_t*) arg; 1827 *paddr = segs->ds_addr; 1828 } 1829 1830 static int 1831 ubsec_dma_malloc( 1832 struct ubsec_softc *sc, 1833 bus_size_t size, 1834 struct ubsec_dma_alloc *dma, 1835 int mapflags 1836 ) 1837 { 1838 int r; 1839 1840 /* XXX could specify sc_dmat as parent but that just adds overhead */ 1841 r = bus_dma_tag_create(NULL, /* parent */ 1842 1, 0, /* alignment, bounds */ 1843 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ 1844 BUS_SPACE_MAXADDR, /* highaddr */ 1845 NULL, NULL, /* filter, filterarg */ 1846 size, /* maxsize */ 1847 1, /* nsegments */ 1848 size, /* maxsegsize */ 1849 BUS_DMA_ALLOCNOW, /* flags */ 1850 &dma->dma_tag); 1851 if (r != 0) { 1852 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1853 "bus_dma_tag_create failed; error %u\n", r); 1854 goto fail_0; 1855 } 1856 1857 r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); 1858 if (r != 0) { 1859 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1860 "bus_dmamap_create failed; error %u\n", r); 1861 goto fail_1; 1862 } 1863 1864 r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 1865 BUS_DMA_NOWAIT, &dma->dma_map); 1866 if (r != 0) { 1867 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1868 "bus_dmammem_alloc failed; size %u, error %u\n", 1869 size, r); 1870 goto fail_2; 1871 } 1872 1873 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1874 size, 1875 ubsec_dmamap_cb, 1876 &dma->dma_paddr, 1877 mapflags | BUS_DMA_NOWAIT); 1878 if (r != 0) { 1879 device_printf(sc->sc_dev, "ubsec_dma_malloc: " 1880 "bus_dmamap_load failed; error %u\n", r); 1881 goto fail_3; 1882 } 1883 1884 dma->dma_size = size; 1885 return (0); 1886 1887 fail_3: 1888 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1889 fail_2: 1890 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1891 fail_1: 1892 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1893 bus_dma_tag_destroy(dma->dma_tag); 1894 fail_0: 1895 dma->dma_map = NULL; 1896 dma->dma_tag = NULL; 1897 return (r); 1898 } 1899 1900 static void 1901 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma) 1902 { 1903 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1904 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1905 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1906 bus_dma_tag_destroy(dma->dma_tag); 1907 } 1908 1909 /* 1910 * Resets the board. Values in the regesters are left as is 1911 * from the reset (i.e. initial values are assigned elsewhere). 1912 */ 1913 static void 1914 ubsec_reset_board(struct ubsec_softc *sc) 1915 { 1916 volatile u_int32_t ctrl; 1917 1918 ctrl = READ_REG(sc, BS_CTRL); 1919 ctrl |= BS_CTRL_RESET; 1920 WRITE_REG(sc, BS_CTRL, ctrl); 1921 1922 /* 1923 * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us 1924 */ 1925 DELAY(10); 1926 } 1927 1928 /* 1929 * Init Broadcom registers 1930 */ 1931 static void 1932 ubsec_init_board(struct ubsec_softc *sc) 1933 { 1934 u_int32_t ctrl; 1935 1936 ctrl = READ_REG(sc, BS_CTRL); 1937 ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64); 1938 ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT; 1939 1940 if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) 1941 ctrl |= BS_CTRL_MCR2INT; 1942 else 1943 ctrl &= ~BS_CTRL_MCR2INT; 1944 1945 if (sc->sc_flags & UBS_FLAGS_HWNORM) 1946 ctrl &= ~BS_CTRL_SWNORM; 1947 1948 WRITE_REG(sc, BS_CTRL, ctrl); 1949 } 1950 1951 /* 1952 * Init Broadcom PCI registers 1953 */ 1954 static void 1955 ubsec_init_pciregs(device_t dev) 1956 { 1957 #if 0 1958 u_int32_t misc; 1959 1960 misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT); 1961 misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT)) 1962 | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT); 1963 misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT)) 1964 | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT); 1965 pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc); 1966 #endif 1967 1968 /* 1969 * This will set the cache line size to 1, this will 1970 * force the BCM58xx chip just to do burst read/writes. 1971 * Cache line read/writes are to slow 1972 */ 1973 pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1); 1974 } 1975 1976 /* 1977 * Clean up after a chip crash. 1978 * It is assumed that the caller in splimp() 1979 */ 1980 static void 1981 ubsec_cleanchip(struct ubsec_softc *sc) 1982 { 1983 struct ubsec_q *q; 1984 1985 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) { 1986 q = SIMPLEQ_FIRST(&sc->sc_qchip); 1987 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next); 1988 ubsec_free_q(sc, q); 1989 } 1990 sc->sc_nqchip = 0; 1991 } 1992 1993 /* 1994 * free a ubsec_q 1995 * It is assumed that the caller is within spimp() 1996 */ 1997 static int 1998 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q) 1999 { 2000 struct ubsec_q *q2; 2001 struct cryptop *crp; 2002 int npkts; 2003 int i; 2004 2005 npkts = q->q_nstacked_mcrs; 2006 2007 for (i = 0; i < npkts; i++) { 2008 if(q->q_stacked_mcr[i]) { 2009 q2 = q->q_stacked_mcr[i]; 2010 2011 if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m)) 2012 m_freem(q2->q_dst_m); 2013 2014 crp = (struct cryptop *)q2->q_crp; 2015 2016 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next); 2017 2018 crp->crp_etype = EFAULT; 2019 crypto_done(crp); 2020 } else { 2021 break; 2022 } 2023 } 2024 2025 /* 2026 * Free header MCR 2027 */ 2028 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m)) 2029 m_freem(q->q_dst_m); 2030 2031 crp = (struct cryptop *)q->q_crp; 2032 2033 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next); 2034 2035 crp->crp_etype = EFAULT; 2036 crypto_done(crp); 2037 return(0); 2038 } 2039 2040 /* 2041 * Routine to reset the chip and clean up. 2042 * It is assumed that the caller is in splimp() 2043 */ 2044 static void 2045 ubsec_totalreset(struct ubsec_softc *sc) 2046 { 2047 ubsec_reset_board(sc); 2048 ubsec_init_board(sc); 2049 ubsec_cleanchip(sc); 2050 } 2051 2052 static int 2053 ubsec_dmamap_aligned(struct ubsec_operand *op) 2054 { 2055 int i; 2056 2057 for (i = 0; i < op->nsegs; i++) { 2058 if (op->segs[i].ds_addr & 3) 2059 return (0); 2060 if ((i != (op->nsegs - 1)) && 2061 (op->segs[i].ds_len & 3)) 2062 return (0); 2063 } 2064 return (1); 2065 } 2066 2067 static void 2068 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q) 2069 { 2070 switch (q->q_type) { 2071 case UBS_CTXOP_MODEXP: { 2072 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q; 2073 2074 ubsec_dma_free(sc, &me->me_q.q_mcr); 2075 ubsec_dma_free(sc, &me->me_q.q_ctx); 2076 ubsec_dma_free(sc, &me->me_M); 2077 ubsec_dma_free(sc, &me->me_E); 2078 ubsec_dma_free(sc, &me->me_C); 2079 ubsec_dma_free(sc, &me->me_epb); 2080 free(me, M_DEVBUF); 2081 break; 2082 } 2083 case UBS_CTXOP_RSAPRIV: { 2084 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q; 2085 2086 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2087 ubsec_dma_free(sc, &rp->rpr_q.q_ctx); 2088 ubsec_dma_free(sc, &rp->rpr_msgin); 2089 ubsec_dma_free(sc, &rp->rpr_msgout); 2090 free(rp, M_DEVBUF); 2091 break; 2092 } 2093 default: 2094 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type); 2095 break; 2096 } 2097 } 2098 2099 static int 2100 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint) 2101 { 2102 struct ubsec_softc *sc = arg; 2103 int r; 2104 2105 if (krp == NULL || krp->krp_callback == NULL) 2106 return (EINVAL); 2107 2108 while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) { 2109 struct ubsec_q2 *q; 2110 2111 q = SIMPLEQ_FIRST(&sc->sc_q2free); 2112 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next); 2113 ubsec_kfree(sc, q); 2114 } 2115 2116 switch (krp->krp_op) { 2117 case CRK_MOD_EXP: 2118 if (sc->sc_flags & UBS_FLAGS_HWNORM) 2119 r = ubsec_kprocess_modexp_hw(sc, krp, hint); 2120 else 2121 r = ubsec_kprocess_modexp_sw(sc, krp, hint); 2122 break; 2123 case CRK_MOD_EXP_CRT: 2124 return (ubsec_kprocess_rsapriv(sc, krp, hint)); 2125 default: 2126 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n", 2127 krp->krp_op); 2128 krp->krp_status = EOPNOTSUPP; 2129 crypto_kdone(krp); 2130 return (0); 2131 } 2132 return (0); /* silence compiler */ 2133 } 2134 2135 /* 2136 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization) 2137 */ 2138 static int 2139 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2140 { 2141 struct ubsec_q2_modexp *me; 2142 struct ubsec_mcr *mcr; 2143 struct ubsec_ctx_modexp *ctx; 2144 struct ubsec_pktbuf *epb; 2145 int s, err = 0; 2146 u_int nbits, normbits, mbits, shiftbits, ebits; 2147 2148 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2149 if (me == NULL) { 2150 err = ENOMEM; 2151 goto errout; 2152 } 2153 bzero(me, sizeof *me); 2154 me->me_krp = krp; 2155 me->me_q.q_type = UBS_CTXOP_MODEXP; 2156 2157 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2158 if (nbits <= 512) 2159 normbits = 512; 2160 else if (nbits <= 768) 2161 normbits = 768; 2162 else if (nbits <= 1024) 2163 normbits = 1024; 2164 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2165 normbits = 1536; 2166 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2167 normbits = 2048; 2168 else { 2169 err = E2BIG; 2170 goto errout; 2171 } 2172 2173 shiftbits = normbits - nbits; 2174 2175 me->me_modbits = nbits; 2176 me->me_shiftbits = shiftbits; 2177 me->me_normbits = normbits; 2178 2179 /* Sanity check: result bits must be >= true modulus bits. */ 2180 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2181 err = ERANGE; 2182 goto errout; 2183 } 2184 2185 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2186 &me->me_q.q_mcr, 0)) { 2187 err = ENOMEM; 2188 goto errout; 2189 } 2190 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2191 2192 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2193 &me->me_q.q_ctx, 0)) { 2194 err = ENOMEM; 2195 goto errout; 2196 } 2197 2198 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2199 if (mbits > nbits) { 2200 err = E2BIG; 2201 goto errout; 2202 } 2203 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2204 err = ENOMEM; 2205 goto errout; 2206 } 2207 ubsec_kshift_r(shiftbits, 2208 krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits, 2209 me->me_M.dma_vaddr, normbits); 2210 2211 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2212 err = ENOMEM; 2213 goto errout; 2214 } 2215 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2216 2217 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2218 if (ebits > nbits) { 2219 err = E2BIG; 2220 goto errout; 2221 } 2222 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2223 err = ENOMEM; 2224 goto errout; 2225 } 2226 ubsec_kshift_r(shiftbits, 2227 krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits, 2228 me->me_E.dma_vaddr, normbits); 2229 2230 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2231 &me->me_epb, 0)) { 2232 err = ENOMEM; 2233 goto errout; 2234 } 2235 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2236 epb->pb_addr = htole32(me->me_E.dma_paddr); 2237 epb->pb_next = 0; 2238 epb->pb_len = htole32(normbits / 8); 2239 2240 #ifdef UBSEC_DEBUG 2241 if (ubsec_debug) { 2242 printf("Epb "); 2243 ubsec_dump_pb(epb); 2244 } 2245 #endif 2246 2247 mcr->mcr_pkts = htole16(1); 2248 mcr->mcr_flags = 0; 2249 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2250 mcr->mcr_reserved = 0; 2251 mcr->mcr_pktlen = 0; 2252 2253 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2254 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2255 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2256 2257 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2258 mcr->mcr_opktbuf.pb_next = 0; 2259 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2260 2261 #ifdef DIAGNOSTIC 2262 /* Misaligned output buffer will hang the chip. */ 2263 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2264 panic("%s: modexp invalid addr 0x%x\n", 2265 device_get_nameunit(sc->sc_dev), 2266 letoh32(mcr->mcr_opktbuf.pb_addr)); 2267 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2268 panic("%s: modexp invalid len 0x%x\n", 2269 device_get_nameunit(sc->sc_dev), 2270 letoh32(mcr->mcr_opktbuf.pb_len)); 2271 #endif 2272 2273 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2274 bzero(ctx, sizeof(*ctx)); 2275 ubsec_kshift_r(shiftbits, 2276 krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits, 2277 ctx->me_N, normbits); 2278 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2279 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2280 ctx->me_E_len = htole16(nbits); 2281 ctx->me_N_len = htole16(nbits); 2282 2283 #ifdef UBSEC_DEBUG 2284 if (ubsec_debug) { 2285 ubsec_dump_mcr(mcr); 2286 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2287 } 2288 #endif 2289 2290 /* 2291 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2292 * everything else. 2293 */ 2294 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2295 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2296 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2297 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2298 2299 /* Enqueue and we're done... */ 2300 s = splimp(); 2301 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2302 ubsec_feed2(sc); 2303 ubsecstats.hst_modexp++; 2304 splx(s); 2305 2306 return (0); 2307 2308 errout: 2309 if (me != NULL) { 2310 if (me->me_q.q_mcr.dma_map != NULL) 2311 ubsec_dma_free(sc, &me->me_q.q_mcr); 2312 if (me->me_q.q_ctx.dma_map != NULL) { 2313 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2314 ubsec_dma_free(sc, &me->me_q.q_ctx); 2315 } 2316 if (me->me_M.dma_map != NULL) { 2317 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2318 ubsec_dma_free(sc, &me->me_M); 2319 } 2320 if (me->me_E.dma_map != NULL) { 2321 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2322 ubsec_dma_free(sc, &me->me_E); 2323 } 2324 if (me->me_C.dma_map != NULL) { 2325 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2326 ubsec_dma_free(sc, &me->me_C); 2327 } 2328 if (me->me_epb.dma_map != NULL) 2329 ubsec_dma_free(sc, &me->me_epb); 2330 free(me, M_DEVBUF); 2331 } 2332 krp->krp_status = err; 2333 crypto_kdone(krp); 2334 return (0); 2335 } 2336 2337 /* 2338 * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization) 2339 */ 2340 static int 2341 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2342 { 2343 struct ubsec_q2_modexp *me; 2344 struct ubsec_mcr *mcr; 2345 struct ubsec_ctx_modexp *ctx; 2346 struct ubsec_pktbuf *epb; 2347 int s, err = 0; 2348 u_int nbits, normbits, mbits, shiftbits, ebits; 2349 2350 me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT); 2351 if (me == NULL) { 2352 err = ENOMEM; 2353 goto errout; 2354 } 2355 bzero(me, sizeof *me); 2356 me->me_krp = krp; 2357 me->me_q.q_type = UBS_CTXOP_MODEXP; 2358 2359 nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]); 2360 if (nbits <= 512) 2361 normbits = 512; 2362 else if (nbits <= 768) 2363 normbits = 768; 2364 else if (nbits <= 1024) 2365 normbits = 1024; 2366 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536) 2367 normbits = 1536; 2368 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048) 2369 normbits = 2048; 2370 else { 2371 err = E2BIG; 2372 goto errout; 2373 } 2374 2375 shiftbits = normbits - nbits; 2376 2377 /* XXX ??? */ 2378 me->me_modbits = nbits; 2379 me->me_shiftbits = shiftbits; 2380 me->me_normbits = normbits; 2381 2382 /* Sanity check: result bits must be >= true modulus bits. */ 2383 if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) { 2384 err = ERANGE; 2385 goto errout; 2386 } 2387 2388 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2389 &me->me_q.q_mcr, 0)) { 2390 err = ENOMEM; 2391 goto errout; 2392 } 2393 mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr; 2394 2395 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp), 2396 &me->me_q.q_ctx, 0)) { 2397 err = ENOMEM; 2398 goto errout; 2399 } 2400 2401 mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]); 2402 if (mbits > nbits) { 2403 err = E2BIG; 2404 goto errout; 2405 } 2406 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) { 2407 err = ENOMEM; 2408 goto errout; 2409 } 2410 bzero(me->me_M.dma_vaddr, normbits / 8); 2411 bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p, 2412 me->me_M.dma_vaddr, (mbits + 7) / 8); 2413 2414 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) { 2415 err = ENOMEM; 2416 goto errout; 2417 } 2418 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2419 2420 ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]); 2421 if (ebits > nbits) { 2422 err = E2BIG; 2423 goto errout; 2424 } 2425 if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) { 2426 err = ENOMEM; 2427 goto errout; 2428 } 2429 bzero(me->me_E.dma_vaddr, normbits / 8); 2430 bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p, 2431 me->me_E.dma_vaddr, (ebits + 7) / 8); 2432 2433 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf), 2434 &me->me_epb, 0)) { 2435 err = ENOMEM; 2436 goto errout; 2437 } 2438 epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr; 2439 epb->pb_addr = htole32(me->me_E.dma_paddr); 2440 epb->pb_next = 0; 2441 epb->pb_len = htole32((ebits + 7) / 8); 2442 2443 #ifdef UBSEC_DEBUG 2444 if (ubsec_debug) { 2445 printf("Epb "); 2446 ubsec_dump_pb(epb); 2447 } 2448 #endif 2449 2450 mcr->mcr_pkts = htole16(1); 2451 mcr->mcr_flags = 0; 2452 mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr); 2453 mcr->mcr_reserved = 0; 2454 mcr->mcr_pktlen = 0; 2455 2456 mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr); 2457 mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8); 2458 mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr); 2459 2460 mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr); 2461 mcr->mcr_opktbuf.pb_next = 0; 2462 mcr->mcr_opktbuf.pb_len = htole32(normbits / 8); 2463 2464 #ifdef DIAGNOSTIC 2465 /* Misaligned output buffer will hang the chip. */ 2466 if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0) 2467 panic("%s: modexp invalid addr 0x%x\n", 2468 device_get_nameunit(sc->sc_dev), 2469 letoh32(mcr->mcr_opktbuf.pb_addr)); 2470 if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0) 2471 panic("%s: modexp invalid len 0x%x\n", 2472 device_get_nameunit(sc->sc_dev), 2473 letoh32(mcr->mcr_opktbuf.pb_len)); 2474 #endif 2475 2476 ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr; 2477 bzero(ctx, sizeof(*ctx)); 2478 bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N, 2479 (nbits + 7) / 8); 2480 ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t))); 2481 ctx->me_op = htole16(UBS_CTXOP_MODEXP); 2482 ctx->me_E_len = htole16(ebits); 2483 ctx->me_N_len = htole16(nbits); 2484 2485 #ifdef UBSEC_DEBUG 2486 if (ubsec_debug) { 2487 ubsec_dump_mcr(mcr); 2488 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx); 2489 } 2490 #endif 2491 2492 /* 2493 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2494 * everything else. 2495 */ 2496 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE); 2497 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE); 2498 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD); 2499 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE); 2500 2501 /* Enqueue and we're done... */ 2502 s = splimp(); 2503 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next); 2504 ubsec_feed2(sc); 2505 splx(s); 2506 2507 return (0); 2508 2509 errout: 2510 if (me != NULL) { 2511 if (me->me_q.q_mcr.dma_map != NULL) 2512 ubsec_dma_free(sc, &me->me_q.q_mcr); 2513 if (me->me_q.q_ctx.dma_map != NULL) { 2514 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size); 2515 ubsec_dma_free(sc, &me->me_q.q_ctx); 2516 } 2517 if (me->me_M.dma_map != NULL) { 2518 bzero(me->me_M.dma_vaddr, me->me_M.dma_size); 2519 ubsec_dma_free(sc, &me->me_M); 2520 } 2521 if (me->me_E.dma_map != NULL) { 2522 bzero(me->me_E.dma_vaddr, me->me_E.dma_size); 2523 ubsec_dma_free(sc, &me->me_E); 2524 } 2525 if (me->me_C.dma_map != NULL) { 2526 bzero(me->me_C.dma_vaddr, me->me_C.dma_size); 2527 ubsec_dma_free(sc, &me->me_C); 2528 } 2529 if (me->me_epb.dma_map != NULL) 2530 ubsec_dma_free(sc, &me->me_epb); 2531 free(me, M_DEVBUF); 2532 } 2533 krp->krp_status = err; 2534 crypto_kdone(krp); 2535 return (0); 2536 } 2537 2538 static int 2539 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint) 2540 { 2541 struct ubsec_q2_rsapriv *rp = NULL; 2542 struct ubsec_mcr *mcr; 2543 struct ubsec_ctx_rsapriv *ctx; 2544 int s, err = 0; 2545 u_int padlen, msglen; 2546 2547 msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]); 2548 padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]); 2549 if (msglen > padlen) 2550 padlen = msglen; 2551 2552 if (padlen <= 256) 2553 padlen = 256; 2554 else if (padlen <= 384) 2555 padlen = 384; 2556 else if (padlen <= 512) 2557 padlen = 512; 2558 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768) 2559 padlen = 768; 2560 else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024) 2561 padlen = 1024; 2562 else { 2563 err = E2BIG; 2564 goto errout; 2565 } 2566 2567 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) { 2568 err = E2BIG; 2569 goto errout; 2570 } 2571 2572 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) { 2573 err = E2BIG; 2574 goto errout; 2575 } 2576 2577 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) { 2578 err = E2BIG; 2579 goto errout; 2580 } 2581 2582 rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT); 2583 if (rp == NULL) 2584 return (ENOMEM); 2585 bzero(rp, sizeof *rp); 2586 rp->rpr_krp = krp; 2587 rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV; 2588 2589 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr), 2590 &rp->rpr_q.q_mcr, 0)) { 2591 err = ENOMEM; 2592 goto errout; 2593 } 2594 mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr; 2595 2596 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv), 2597 &rp->rpr_q.q_ctx, 0)) { 2598 err = ENOMEM; 2599 goto errout; 2600 } 2601 ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr; 2602 bzero(ctx, sizeof *ctx); 2603 2604 /* Copy in p */ 2605 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p, 2606 &ctx->rpr_buf[0 * (padlen / 8)], 2607 (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8); 2608 2609 /* Copy in q */ 2610 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p, 2611 &ctx->rpr_buf[1 * (padlen / 8)], 2612 (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8); 2613 2614 /* Copy in dp */ 2615 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p, 2616 &ctx->rpr_buf[2 * (padlen / 8)], 2617 (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8); 2618 2619 /* Copy in dq */ 2620 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p, 2621 &ctx->rpr_buf[3 * (padlen / 8)], 2622 (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8); 2623 2624 /* Copy in pinv */ 2625 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p, 2626 &ctx->rpr_buf[4 * (padlen / 8)], 2627 (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8); 2628 2629 msglen = padlen * 2; 2630 2631 /* Copy in input message (aligned buffer/length). */ 2632 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) { 2633 /* Is this likely? */ 2634 err = E2BIG; 2635 goto errout; 2636 } 2637 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) { 2638 err = ENOMEM; 2639 goto errout; 2640 } 2641 bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8); 2642 bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p, 2643 rp->rpr_msgin.dma_vaddr, 2644 (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8); 2645 2646 /* Prepare space for output message (aligned buffer/length). */ 2647 if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) { 2648 /* Is this likely? */ 2649 err = E2BIG; 2650 goto errout; 2651 } 2652 if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) { 2653 err = ENOMEM; 2654 goto errout; 2655 } 2656 bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8); 2657 2658 mcr->mcr_pkts = htole16(1); 2659 mcr->mcr_flags = 0; 2660 mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr); 2661 mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr); 2662 mcr->mcr_ipktbuf.pb_next = 0; 2663 mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size); 2664 mcr->mcr_reserved = 0; 2665 mcr->mcr_pktlen = htole16(msglen); 2666 mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr); 2667 mcr->mcr_opktbuf.pb_next = 0; 2668 mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size); 2669 2670 #ifdef DIAGNOSTIC 2671 if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) { 2672 panic("%s: rsapriv: invalid msgin %x(0x%x)", 2673 device_get_nameunit(sc->sc_dev), 2674 rp->rpr_msgin.dma_paddr, rp->rpr_msgin.dma_size); 2675 } 2676 if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) { 2677 panic("%s: rsapriv: invalid msgout %x(0x%x)", 2678 device_get_nameunit(sc->sc_dev), 2679 rp->rpr_msgout.dma_paddr, rp->rpr_msgout.dma_size); 2680 } 2681 #endif 2682 2683 ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8)); 2684 ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV); 2685 ctx->rpr_q_len = htole16(padlen); 2686 ctx->rpr_p_len = htole16(padlen); 2687 2688 /* 2689 * ubsec_feed2 will sync mcr and ctx, we just need to sync 2690 * everything else. 2691 */ 2692 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE); 2693 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD); 2694 2695 /* Enqueue and we're done... */ 2696 s = splimp(); 2697 SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next); 2698 ubsec_feed2(sc); 2699 ubsecstats.hst_modexpcrt++; 2700 splx(s); 2701 return (0); 2702 2703 errout: 2704 if (rp != NULL) { 2705 if (rp->rpr_q.q_mcr.dma_map != NULL) 2706 ubsec_dma_free(sc, &rp->rpr_q.q_mcr); 2707 if (rp->rpr_msgin.dma_map != NULL) { 2708 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size); 2709 ubsec_dma_free(sc, &rp->rpr_msgin); 2710 } 2711 if (rp->rpr_msgout.dma_map != NULL) { 2712 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size); 2713 ubsec_dma_free(sc, &rp->rpr_msgout); 2714 } 2715 free(rp, M_DEVBUF); 2716 } 2717 krp->krp_status = err; 2718 crypto_kdone(krp); 2719 return (0); 2720 } 2721 2722 #ifdef UBSEC_DEBUG 2723 static void 2724 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb) 2725 { 2726 printf("addr 0x%x (0x%x) next 0x%x\n", 2727 pb->pb_addr, pb->pb_len, pb->pb_next); 2728 } 2729 2730 static void 2731 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c) 2732 { 2733 printf("CTX (0x%x):\n", c->ctx_len); 2734 switch (letoh16(c->ctx_op)) { 2735 case UBS_CTXOP_RNGBYPASS: 2736 case UBS_CTXOP_RNGSHA1: 2737 break; 2738 case UBS_CTXOP_MODEXP: 2739 { 2740 struct ubsec_ctx_modexp *cx = (void *)c; 2741 int i, len; 2742 2743 printf(" Elen %u, Nlen %u\n", 2744 letoh16(cx->me_E_len), letoh16(cx->me_N_len)); 2745 len = (cx->me_N_len + 7)/8; 2746 for (i = 0; i < len; i++) 2747 printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]); 2748 printf("\n"); 2749 break; 2750 } 2751 default: 2752 printf("unknown context: %x\n", c->ctx_op); 2753 } 2754 printf("END CTX\n"); 2755 } 2756 2757 static void 2758 ubsec_dump_mcr(struct ubsec_mcr *mcr) 2759 { 2760 volatile struct ubsec_mcr_add *ma; 2761 int i; 2762 2763 printf("MCR:\n"); 2764 printf(" pkts: %u, flags 0x%x\n", 2765 letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags)); 2766 ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp; 2767 for (i = 0; i < letoh16(mcr->mcr_pkts); i++) { 2768 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i, 2769 letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen), 2770 letoh16(ma->mcr_reserved)); 2771 printf(" %d: ipkt ", i); 2772 ubsec_dump_pb(&ma->mcr_ipktbuf); 2773 printf(" %d: opkt ", i); 2774 ubsec_dump_pb(&ma->mcr_opktbuf); 2775 ma++; 2776 } 2777 printf("END MCR\n"); 2778 } 2779 #endif /* UBSEC_DEBUG */ 2780 2781 /* 2782 * Return the number of significant bits of a big number. 2783 */ 2784 static int 2785 ubsec_ksigbits(struct crparam *cr) 2786 { 2787 u_int plen = (cr->crp_nbits + 7) / 8; 2788 int i, sig = plen * 8; 2789 u_int8_t c, *p = cr->crp_p; 2790 2791 for (i = plen - 1; i >= 0; i--) { 2792 c = p[i]; 2793 if (c != 0) { 2794 while ((c & 0x80) == 0) { 2795 sig--; 2796 c <<= 1; 2797 } 2798 break; 2799 } 2800 sig -= 8; 2801 } 2802 return (sig); 2803 } 2804 2805 static void 2806 ubsec_kshift_r( 2807 u_int shiftbits, 2808 u_int8_t *src, u_int srcbits, 2809 u_int8_t *dst, u_int dstbits) 2810 { 2811 u_int slen, dlen; 2812 int i, si, di, n; 2813 2814 slen = (srcbits + 7) / 8; 2815 dlen = (dstbits + 7) / 8; 2816 2817 for (i = 0; i < slen; i++) 2818 dst[i] = src[i]; 2819 for (i = 0; i < dlen - slen; i++) 2820 dst[slen + i] = 0; 2821 2822 n = shiftbits / 8; 2823 if (n != 0) { 2824 si = dlen - n - 1; 2825 di = dlen - 1; 2826 while (si >= 0) 2827 dst[di--] = dst[si--]; 2828 while (di >= 0) 2829 dst[di--] = 0; 2830 } 2831 2832 n = shiftbits % 8; 2833 if (n != 0) { 2834 for (i = dlen - 1; i > 0; i--) 2835 dst[i] = (dst[i] << n) | 2836 (dst[i - 1] >> (8 - n)); 2837 dst[0] = dst[0] << n; 2838 } 2839 } 2840 2841 static void 2842 ubsec_kshift_l( 2843 u_int shiftbits, 2844 u_int8_t *src, u_int srcbits, 2845 u_int8_t *dst, u_int dstbits) 2846 { 2847 int slen, dlen, i, n; 2848 2849 slen = (srcbits + 7) / 8; 2850 dlen = (dstbits + 7) / 8; 2851 2852 n = shiftbits / 8; 2853 for (i = 0; i < slen; i++) 2854 dst[i] = src[i + n]; 2855 for (i = 0; i < dlen - slen; i++) 2856 dst[slen + i] = 0; 2857 2858 n = shiftbits % 8; 2859 if (n != 0) { 2860 for (i = 0; i < (dlen - 1); i++) 2861 dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n)); 2862 dst[dlen - 1] = dst[dlen - 1] >> n; 2863 } 2864 } 2865