xref: /dragonfly/sys/dev/crypto/ubsec/ubsec.c (revision 030b0c8c)
1 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.12 2003/06/04 17:56:59 sam Exp $ */
2 /*	$OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $	*/
3 
4 /*
5  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
6  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
7  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
8  *
9  * All rights reserved.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  * 3. All advertising materials mentioning features or use of this software
20  *    must display the following acknowledgement:
21  *	This product includes software developed by Jason L. Wright
22  * 4. The name of the author may not be used to endorse or promote products
23  *    derived from this software without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
27  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
28  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
29  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
30  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
31  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
34  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35  * POSSIBILITY OF SUCH DAMAGE.
36  *
37  * Effort sponsored in part by the Defense Advanced Research Projects
38  * Agency (DARPA) and Air Force Research Laboratory, Air Force
39  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
40  *
41  */
42 
43 /*
44  * uBsec 5[56]01, 58xx hardware crypto accelerator
45  */
46 
47 #include "opt_ubsec.h"
48 
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/proc.h>
52 #include <sys/errno.h>
53 #include <sys/malloc.h>
54 #include <sys/kernel.h>
55 #include <sys/mbuf.h>
56 #include <sys/sysctl.h>
57 #include <sys/endian.h>
58 #include <sys/bus.h>
59 #include <sys/rman.h>
60 #include <sys/md5.h>
61 #include <sys/random.h>
62 #include <sys/thread2.h>
63 
64 #include <vm/vm.h>
65 #include <vm/pmap.h>
66 
67 #include <machine/clock.h>
68 
69 #include <crypto/sha1.h>
70 #include <opencrypto/cryptodev.h>
71 #include <opencrypto/cryptosoft.h>
72 
73 #include "cryptodev_if.h"
74 
75 #include <bus/pci/pcivar.h>
76 #include <bus/pci/pcireg.h>
77 
78 /* grr, #defines for gratuitous incompatibility in queue.h */
79 #define	SIMPLEQ_HEAD		STAILQ_HEAD
80 #define	SIMPLEQ_ENTRY		STAILQ_ENTRY
81 #define	SIMPLEQ_INIT		STAILQ_INIT
82 #define	SIMPLEQ_INSERT_TAIL	STAILQ_INSERT_TAIL
83 #define	SIMPLEQ_EMPTY		STAILQ_EMPTY
84 #define	SIMPLEQ_FIRST		STAILQ_FIRST
85 #define	SIMPLEQ_REMOVE_HEAD	STAILQ_REMOVE_HEAD
86 #define	SIMPLEQ_FOREACH		STAILQ_FOREACH
87 /* ditto for endian.h */
88 #define	letoh16(x)		le16toh(x)
89 #define	letoh32(x)		le32toh(x)
90 
91 #ifdef UBSEC_RNDTEST
92 #include "../rndtest/rndtest.h"
93 #endif
94 #include "ubsecreg.h"
95 #include "ubsecvar.h"
96 
97 /*
98  * Prototypes and count for the pci_device structure
99  */
100 static	int ubsec_probe(device_t);
101 static	int ubsec_attach(device_t);
102 static	int ubsec_detach(device_t);
103 static	int ubsec_suspend(device_t);
104 static	int ubsec_resume(device_t);
105 static	void ubsec_shutdown(device_t);
106 static	void ubsec_intr(void *);
107 static	int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
108 static	int ubsec_freesession(device_t, u_int64_t);
109 static	int ubsec_process(device_t, struct cryptop *, int);
110 static	void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
111 static	void ubsec_feed(struct ubsec_softc *);
112 static	void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
113 static	void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
114 static	int ubsec_feed2(struct ubsec_softc *);
115 static	void ubsec_rng(void *);
116 static	int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
117 			     struct ubsec_dma_alloc *, int);
118 #define	ubsec_dma_sync(_dma, _flags) \
119 	bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
120 static	void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
121 static	int ubsec_dmamap_aligned(struct ubsec_operand *op);
122 
123 static	void ubsec_reset_board(struct ubsec_softc *sc);
124 static	void ubsec_init_board(struct ubsec_softc *sc);
125 static	void ubsec_init_pciregs(device_t dev);
126 static	void ubsec_totalreset(struct ubsec_softc *sc);
127 
128 static	int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
129 
130 static	int ubsec_kprocess(device_t, struct cryptkop *, int);
131 static	int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
132 static	int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
133 static	int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
134 static	void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
135 static	int ubsec_ksigbits(struct crparam *);
136 static	void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
137 static	void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
138 
139 
140 static device_method_t ubsec_methods[] = {
141 	/* Device interface */
142 	DEVMETHOD(device_probe,		ubsec_probe),
143 	DEVMETHOD(device_attach,	ubsec_attach),
144 	DEVMETHOD(device_detach,	ubsec_detach),
145 	DEVMETHOD(device_suspend,	ubsec_suspend),
146 	DEVMETHOD(device_resume,	ubsec_resume),
147 	DEVMETHOD(device_shutdown,	ubsec_shutdown),
148 
149 	/* bus interface */
150 	DEVMETHOD(bus_print_child,	bus_generic_print_child),
151 	DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
152 
153 	/* crypto device methods */
154 	DEVMETHOD(cryptodev_newsession,	ubsec_newsession),
155 	DEVMETHOD(cryptodev_freesession,ubsec_freesession),
156 	DEVMETHOD(cryptodev_process,	ubsec_process),
157 	DEVMETHOD(cryptodev_kprocess,	ubsec_kprocess),
158 
159 	DEVMETHOD_END
160 };
161 static driver_t ubsec_driver = {
162 	"ubsec",
163 	ubsec_methods,
164 	sizeof (struct ubsec_softc)
165 };
166 static devclass_t ubsec_devclass;
167 
168 DECLARE_DUMMY_MODULE(ubsec);
169 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, NULL, NULL);
170 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
171 #ifdef UBSEC_RNDTEST
172 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
173 #endif
174 
175 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
176 
177 #ifdef UBSEC_DEBUG
178 static	void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
179 static	void ubsec_dump_mcr(struct ubsec_mcr *);
180 static	void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
181 
182 static	int ubsec_debug = 0;
183 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
184 	    0, "control debugging msgs");
185 #endif
186 
187 #define	READ_REG(sc,r) \
188 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
189 
190 #define WRITE_REG(sc,reg,val) \
191 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
192 
193 #define	SWAP32(x) (x) = htole32(ntohl((x)))
194 #define	HTOLE32(x) (x) = htole32(x)
195 
196 
197 struct ubsec_stats ubsecstats;
198 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
199 	    ubsec_stats, "driver statistics");
200 
201 static int
ubsec_probe(device_t dev)202 ubsec_probe(device_t dev)
203 {
204 	if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
205 	    (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
206 	     pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
207 		return (0);
208 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
209 	    (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
210 	     pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
211 		return (0);
212 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
213 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
214 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
215 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
216 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
217 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
218 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
219 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
220 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
221 	     ))
222 		return (0);
223 	return (ENXIO);
224 }
225 
226 static const char*
ubsec_partname(struct ubsec_softc * sc)227 ubsec_partname(struct ubsec_softc *sc)
228 {
229 	/* XXX sprintf numbers when not decoded */
230 	switch (pci_get_vendor(sc->sc_dev)) {
231 	case PCI_VENDOR_BROADCOM:
232 		switch (pci_get_device(sc->sc_dev)) {
233 		case PCI_PRODUCT_BROADCOM_5801:	return "Broadcom 5801";
234 		case PCI_PRODUCT_BROADCOM_5802:	return "Broadcom 5802";
235 		case PCI_PRODUCT_BROADCOM_5805:	return "Broadcom 5805";
236 		case PCI_PRODUCT_BROADCOM_5820:	return "Broadcom 5820";
237 		case PCI_PRODUCT_BROADCOM_5821:	return "Broadcom 5821";
238 		case PCI_PRODUCT_BROADCOM_5822:	return "Broadcom 5822";
239 		case PCI_PRODUCT_BROADCOM_5823:	return "Broadcom 5823";
240 		case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825";
241 		}
242 		return "Broadcom unknown-part";
243 	case PCI_VENDOR_BLUESTEEL:
244 		switch (pci_get_device(sc->sc_dev)) {
245 		case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
246 		}
247 		return "Bluesteel unknown-part";
248 	case PCI_VENDOR_SUN:
249 		switch (pci_get_device(sc->sc_dev)) {
250 		case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
251 		case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
252 		}
253 		return "Sun unknown-part";
254 	}
255 	return "Unknown-vendor unknown-part";
256 }
257 
258 static void
default_harvest(struct rndtest_state * rsp __unused,void * buf,u_int count)259 default_harvest(struct rndtest_state *rsp __unused, void *buf, u_int count)
260 {
261 	add_buffer_randomness_src(buf, count, RAND_SRC_UBSEC);
262 }
263 
264 static int
ubsec_attach(device_t dev)265 ubsec_attach(device_t dev)
266 {
267 	struct ubsec_softc *sc = device_get_softc(dev);
268 	struct ubsec_dma *dmap;
269 	u_int32_t cmd, i;
270 	int rid;
271 
272 	KASSERT(sc != NULL, ("ubsec_attach: null software carrier!"));
273 	bzero(sc, sizeof (*sc));
274 	sc->sc_dev = dev;
275 
276 	SIMPLEQ_INIT(&sc->sc_queue);
277 	SIMPLEQ_INIT(&sc->sc_qchip);
278 	SIMPLEQ_INIT(&sc->sc_queue2);
279 	SIMPLEQ_INIT(&sc->sc_qchip2);
280 	SIMPLEQ_INIT(&sc->sc_q2free);
281 
282 	/* XXX handle power management */
283 
284 	sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
285 
286 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
287 	    pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
288 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
289 
290 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
291 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
292 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
293 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
294 
295 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
296 	    pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
297 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
298 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
299 
300 	if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
301 	     (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
302 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
303 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
304 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
305 	    (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
306 	     (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
307 	      pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
308 		/* NB: the 5821/5822 defines some additional status bits */
309 		sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
310 		    BS_STAT_MCR2_ALLEMPTY;
311 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
312 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
313 	}
314 
315 	cmd = pci_read_config(dev, PCIR_COMMAND, 4);
316 	cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
317 	pci_write_config(dev, PCIR_COMMAND, cmd, 4);
318 	cmd = pci_read_config(dev, PCIR_COMMAND, 4);
319 
320 	if (!(cmd & PCIM_CMD_MEMEN)) {
321 		device_printf(dev, "failed to enable memory mapping\n");
322 		goto bad;
323 	}
324 
325 	if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
326 		device_printf(dev, "failed to enable bus mastering\n");
327 		goto bad;
328 	}
329 
330 	/*
331 	 * Setup memory-mapping of PCI registers.
332 	 */
333 	rid = BS_BAR;
334 	sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
335 				       0, ~0, 1, RF_ACTIVE);
336 	if (sc->sc_sr == NULL) {
337 		device_printf(dev, "cannot map register space\n");
338 		goto bad;
339 	}
340 	sc->sc_st = rman_get_bustag(sc->sc_sr);
341 	sc->sc_sh = rman_get_bushandle(sc->sc_sr);
342 
343 	/*
344 	 * Arrange interrupt line.
345 	 */
346 	rid = 0;
347 	sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
348 					0, ~0, 1, RF_SHAREABLE|RF_ACTIVE);
349 	if (sc->sc_irq == NULL) {
350 		device_printf(dev, "could not map interrupt\n");
351 		goto bad1;
352 	}
353 	/*
354 	 * NB: Network code assumes we are blocked with splimp()
355 	 *     so make sure the IRQ is mapped appropriately.
356 	 */
357 	if (bus_setup_intr(dev, sc->sc_irq, 0,
358 			   ubsec_intr, sc,
359 			   &sc->sc_ih, NULL)) {
360 		device_printf(dev, "could not establish interrupt\n");
361 		goto bad2;
362 	}
363 
364 	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
365 	if (sc->sc_cid < 0) {
366 		device_printf(dev, "could not get crypto driver id\n");
367 		goto bad3;
368 	}
369 
370 	/*
371 	 * Setup DMA descriptor area.
372 	 */
373 	if (bus_dma_tag_create(NULL,			/* parent */
374 			       1, 0,			/* alignment, bounds */
375 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
376 			       BUS_SPACE_MAXADDR,	/* highaddr */
377 			       0x3ffff,			/* maxsize */
378 			       UBS_MAX_SCATTER,		/* nsegments */
379 			       0xffff,			/* maxsegsize */
380 			       BUS_DMA_ALLOCNOW,	/* flags */
381 			       &sc->sc_dmat)) {
382 		device_printf(dev, "cannot allocate DMA tag\n");
383 		goto bad4;
384 	}
385 	SIMPLEQ_INIT(&sc->sc_freequeue);
386 	dmap = sc->sc_dmaa;
387 	for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
388 		struct ubsec_q *q;
389 
390 		q = kmalloc(sizeof(struct ubsec_q), M_DEVBUF, M_WAITOK);
391 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
392 		    &dmap->d_alloc, 0)) {
393 			device_printf(dev, "cannot allocate dma buffers\n");
394 			kfree(q, M_DEVBUF);
395 			break;
396 		}
397 		dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
398 
399 		q->q_dma = dmap;
400 		sc->sc_queuea[i] = q;
401 
402 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
403 	}
404 
405 	device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
406 
407 	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
408 	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
409 	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
410 	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
411 
412 	/*
413 	 * Reset Broadcom chip
414 	 */
415 	ubsec_reset_board(sc);
416 
417 	/*
418 	 * Init Broadcom specific PCI settings
419 	 */
420 	ubsec_init_pciregs(dev);
421 
422 	/*
423 	 * Init Broadcom chip
424 	 */
425 	ubsec_init_board(sc);
426 
427 #ifndef UBSEC_NO_RNG
428 	if (sc->sc_flags & UBS_FLAGS_RNG) {
429 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
430 #ifdef UBSEC_RNDTEST
431 		sc->sc_rndtest = rndtest_attach(dev);
432 		if (sc->sc_rndtest)
433 			sc->sc_harvest = rndtest_harvest;
434 		else
435 			sc->sc_harvest = default_harvest;
436 #else
437 		sc->sc_harvest = default_harvest;
438 #endif
439 
440 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
441 		    &sc->sc_rng.rng_q.q_mcr, 0))
442 			goto skip_rng;
443 
444 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
445 		    &sc->sc_rng.rng_q.q_ctx, 0)) {
446 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
447 			goto skip_rng;
448 		}
449 
450 		if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
451 		    UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
452 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
453 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
454 			goto skip_rng;
455 		}
456 
457 		if (hz >= 100)
458 			sc->sc_rnghz = hz / 100;
459 		else
460 			sc->sc_rnghz = 1;
461 		callout_init(&sc->sc_rngto);
462 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
463 skip_rng:
464 	;
465 	}
466 #endif /* UBSEC_NO_RNG */
467 
468 	if (sc->sc_flags & UBS_FLAGS_KEY) {
469 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
470 
471 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
472 #if 0
473 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
474 #endif
475 	}
476 	return (0);
477 bad4:
478 	crypto_unregister_all(sc->sc_cid);
479 bad3:
480 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
481 bad2:
482 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
483 bad1:
484 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
485 bad:
486 	return (ENXIO);
487 }
488 
489 /*
490  * Detach a device that successfully probed.
491  */
492 static int
ubsec_detach(device_t dev)493 ubsec_detach(device_t dev)
494 {
495 	struct ubsec_softc *sc = device_get_softc(dev);
496 
497 	KASSERT(sc != NULL, ("ubsec_detach: null software carrier"));
498 
499 	/* XXX wait/abort active ops */
500 
501 	crit_enter();
502 
503 	callout_stop(&sc->sc_rngto);
504 
505 	crypto_unregister_all(sc->sc_cid);
506 
507 #ifdef UBSEC_RNDTEST
508 	if (sc->sc_rndtest)
509 		rndtest_detach(sc->sc_rndtest);
510 #endif
511 
512 	while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
513 		struct ubsec_q *q;
514 
515 		q = SIMPLEQ_FIRST(&sc->sc_freequeue);
516 		SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
517 		ubsec_dma_free(sc, &q->q_dma->d_alloc);
518 		kfree(q, M_DEVBUF);
519 	}
520 #ifndef UBSEC_NO_RNG
521 	if (sc->sc_flags & UBS_FLAGS_RNG) {
522 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
523 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
524 		ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
525 	}
526 #endif /* UBSEC_NO_RNG */
527 
528 	bus_generic_detach(dev);
529 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
530 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
531 
532 	bus_dma_tag_destroy(sc->sc_dmat);
533 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
534 
535 	crit_exit();
536 
537 	return (0);
538 }
539 
540 /*
541  * Stop all chip i/o so that the kernel's probe routines don't
542  * get confused by errant DMAs when rebooting.
543  */
544 static void
ubsec_shutdown(device_t dev)545 ubsec_shutdown(device_t dev)
546 {
547 #ifdef notyet
548 	ubsec_stop(device_get_softc(dev));
549 #endif
550 }
551 
552 /*
553  * Device suspend routine.
554  */
555 static int
ubsec_suspend(device_t dev)556 ubsec_suspend(device_t dev)
557 {
558 	struct ubsec_softc *sc = device_get_softc(dev);
559 
560 	KASSERT(sc != NULL, ("ubsec_suspend: null software carrier"));
561 #ifdef notyet
562 	/* XXX stop the device and save PCI settings */
563 #endif
564 	sc->sc_suspended = 1;
565 
566 	return (0);
567 }
568 
569 static int
ubsec_resume(device_t dev)570 ubsec_resume(device_t dev)
571 {
572 	struct ubsec_softc *sc = device_get_softc(dev);
573 
574 	KASSERT(sc != NULL, ("ubsec_resume: null software carrier"));
575 #ifdef notyet
576 	/* XXX retore PCI settings and start the device */
577 #endif
578 	sc->sc_suspended = 0;
579 	return (0);
580 }
581 
582 /*
583  * UBSEC Interrupt routine
584  */
585 static void
ubsec_intr(void * arg)586 ubsec_intr(void *arg)
587 {
588 	struct ubsec_softc *sc = arg;
589 	volatile u_int32_t stat;
590 	struct ubsec_q *q;
591 	struct ubsec_dma *dmap;
592 	int npkts = 0, i;
593 
594 	stat = READ_REG(sc, BS_STAT);
595 	stat &= sc->sc_statmask;
596 	if (stat == 0) {
597 		return;
598 	}
599 
600 	WRITE_REG(sc, BS_STAT, stat);		/* IACK */
601 
602 	/*
603 	 * Check to see if we have any packets waiting for us
604 	 */
605 	if ((stat & BS_STAT_MCR1_DONE)) {
606 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
607 			q = SIMPLEQ_FIRST(&sc->sc_qchip);
608 			dmap = q->q_dma;
609 
610 			if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
611 				break;
612 
613 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
614 
615 			npkts = q->q_nstacked_mcrs;
616 			sc->sc_nqchip -= 1+npkts;
617 			/*
618 			 * search for further sc_qchip ubsec_q's that share
619 			 * the same MCR, and complete them too, they must be
620 			 * at the top.
621 			 */
622 			for (i = 0; i < npkts; i++) {
623 				if(q->q_stacked_mcr[i]) {
624 					ubsec_callback(sc, q->q_stacked_mcr[i]);
625 				} else {
626 					break;
627 				}
628 			}
629 			ubsec_callback(sc, q);
630 		}
631 
632 		/*
633 		 * Don't send any more packet to chip if there has been
634 		 * a DMAERR.
635 		 */
636 		if (!(stat & BS_STAT_DMAERR))
637 			ubsec_feed(sc);
638 	}
639 
640 	/*
641 	 * Check to see if we have any key setups/rng's waiting for us
642 	 */
643 	if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
644 	    (stat & BS_STAT_MCR2_DONE)) {
645 		struct ubsec_q2 *q2;
646 		struct ubsec_mcr *mcr;
647 
648 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
649 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
650 
651 			ubsec_dma_sync(&q2->q_mcr,
652 			    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
653 
654 			mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
655 			if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
656 				ubsec_dma_sync(&q2->q_mcr,
657 				    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
658 				break;
659 			}
660 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
661 			ubsec_callback2(sc, q2);
662 			/*
663 			 * Don't send any more packet to chip if there has been
664 			 * a DMAERR.
665 			 */
666 			if (!(stat & BS_STAT_DMAERR))
667 				ubsec_feed2(sc);
668 		}
669 	}
670 
671 	/*
672 	 * Check to see if we got any DMA Error
673 	 */
674 	if (stat & BS_STAT_DMAERR) {
675 #ifdef UBSEC_DEBUG
676 		if (ubsec_debug) {
677 			volatile u_int32_t a = READ_REG(sc, BS_ERR);
678 
679 			kprintf("dmaerr %s@%08x\n",
680 			    (a & BS_ERR_READ) ? "read" : "write",
681 			    a & BS_ERR_ADDR);
682 		}
683 #endif /* UBSEC_DEBUG */
684 		ubsecstats.hst_dmaerr++;
685 		ubsec_totalreset(sc);
686 		ubsec_feed(sc);
687 	}
688 
689 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
690 		int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
691 #ifdef UBSEC_DEBUG
692 		if (ubsec_debug)
693 			device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
694 				sc->sc_needwakeup);
695 #endif /* UBSEC_DEBUG */
696 		sc->sc_needwakeup &= ~wakeup;
697 		crypto_unblock(sc->sc_cid, wakeup);
698 	}
699 }
700 
701 /*
702  * ubsec_feed() - aggregate and post requests to chip
703  */
704 static void
ubsec_feed(struct ubsec_softc * sc)705 ubsec_feed(struct ubsec_softc *sc)
706 {
707 	struct ubsec_q *q, *q2;
708 	int npkts, i;
709 	void *v;
710 	u_int32_t stat;
711 
712 	/*
713 	 * Decide how many ops to combine in a single MCR.  We cannot
714 	 * aggregate more than UBS_MAX_AGGR because this is the number
715 	 * of slots defined in the data structure.  Note that
716 	 * aggregation only happens if ops are marked batch'able.
717 	 * Aggregating ops reduces the number of interrupts to the host
718 	 * but also (potentially) increases the latency for processing
719 	 * completed ops as we only get an interrupt when all aggregated
720 	 * ops have completed.
721 	 */
722 	if (sc->sc_nqueue == 0)
723 		return;
724 	if (sc->sc_nqueue > 1) {
725 		npkts = 0;
726 		SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
727 			npkts++;
728 			if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
729 				break;
730 		}
731 	} else
732 		npkts = 1;
733 	/*
734 	 * Check device status before going any further.
735 	 */
736 	if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
737 		if (stat & BS_STAT_DMAERR) {
738 			ubsec_totalreset(sc);
739 			ubsecstats.hst_dmaerr++;
740 		} else
741 			ubsecstats.hst_mcr1full++;
742 		return;
743 	}
744 	if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
745 		ubsecstats.hst_maxqueue = sc->sc_nqueue;
746 	if (npkts > UBS_MAX_AGGR)
747 		npkts = UBS_MAX_AGGR;
748 	if (npkts < 2)				/* special case 1 op */
749 		goto feed1;
750 
751 	ubsecstats.hst_totbatch += npkts-1;
752 #ifdef UBSEC_DEBUG
753 	if (ubsec_debug)
754 		kprintf("merging %d records\n", npkts);
755 #endif /* UBSEC_DEBUG */
756 
757 	q = SIMPLEQ_FIRST(&sc->sc_queue);
758 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
759 	--sc->sc_nqueue;
760 
761 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
762 	if (q->q_dst_map != NULL)
763 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
764 
765 	q->q_nstacked_mcrs = npkts - 1;		/* Number of packets stacked */
766 
767 	for (i = 0; i < q->q_nstacked_mcrs; i++) {
768 		q2 = SIMPLEQ_FIRST(&sc->sc_queue);
769 		bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
770 		    BUS_DMASYNC_PREWRITE);
771 		if (q2->q_dst_map != NULL)
772 			bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
773 			    BUS_DMASYNC_PREREAD);
774 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
775 		--sc->sc_nqueue;
776 
777 		v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
778 		    sizeof(struct ubsec_mcr_add));
779 		bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
780 		q->q_stacked_mcr[i] = q2;
781 	}
782 	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
783 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
784 	sc->sc_nqchip += npkts;
785 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
786 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
787 	ubsec_dma_sync(&q->q_dma->d_alloc,
788 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
789 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
790 	    offsetof(struct ubsec_dmachunk, d_mcr));
791 	return;
792 
793 feed1:
794 	q = SIMPLEQ_FIRST(&sc->sc_queue);
795 
796 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
797 	if (q->q_dst_map != NULL)
798 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
799 	ubsec_dma_sync(&q->q_dma->d_alloc,
800 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
801 
802 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
803 	    offsetof(struct ubsec_dmachunk, d_mcr));
804 #ifdef UBSEC_DEBUG
805 	if (ubsec_debug)
806 		kprintf("feed1: q->chip %p %08x stat %08x\n",
807 		      q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
808 		      stat);
809 #endif /* UBSEC_DEBUG */
810 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
811 	--sc->sc_nqueue;
812 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
813 	sc->sc_nqchip++;
814 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
815 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
816 	return;
817 }
818 
819 static void
ubsec_setup_enckey(struct ubsec_session * ses,int algo,caddr_t key)820 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
821 {
822 
823 	/* Go ahead and compute key in ubsec's byte order */
824 	if (algo == CRYPTO_DES_CBC) {
825 		bcopy(key, &ses->ses_deskey[0], 8);
826 		bcopy(key, &ses->ses_deskey[2], 8);
827 		bcopy(key, &ses->ses_deskey[4], 8);
828 	} else
829 		bcopy(key, ses->ses_deskey, 24);
830 
831 	SWAP32(ses->ses_deskey[0]);
832 	SWAP32(ses->ses_deskey[1]);
833 	SWAP32(ses->ses_deskey[2]);
834 	SWAP32(ses->ses_deskey[3]);
835 	SWAP32(ses->ses_deskey[4]);
836 	SWAP32(ses->ses_deskey[5]);
837 }
838 
839 static void
ubsec_setup_mackey(struct ubsec_session * ses,int algo,caddr_t key,int klen)840 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
841 {
842 	MD5_CTX md5ctx;
843 	SHA1_CTX sha1ctx;
844 	int i;
845 
846 	for (i = 0; i < klen; i++)
847 		key[i] ^= HMAC_IPAD_VAL;
848 
849 	if (algo == CRYPTO_MD5_HMAC) {
850 		MD5Init(&md5ctx);
851 		MD5Update(&md5ctx, key, klen);
852 		MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
853 		/* gcc8 craps out on -Warray-bounds w/ optimized bcopy */
854 		_bcopy(&md5ctx.A, ses->ses_hminner, sizeof(md5ctx.A) * 4);
855 	} else {
856 		SHA1Init(&sha1ctx);
857 		SHA1Update(&sha1ctx, key, klen);
858 		SHA1Update(&sha1ctx, hmac_ipad_buffer,
859 		    SHA1_HMAC_BLOCK_LEN - klen);
860 		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
861 	}
862 
863 	for (i = 0; i < klen; i++)
864 		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
865 
866 	if (algo == CRYPTO_MD5_HMAC) {
867 		MD5Init(&md5ctx);
868 		MD5Update(&md5ctx, key, klen);
869 		MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
870 		/* gcc8 craps out on -Warray-bounds w/ optimized bcopy */
871 		_bcopy(&md5ctx.A, ses->ses_hmouter, sizeof(md5ctx.A) * 4);
872 	} else {
873 		SHA1Init(&sha1ctx);
874 		SHA1Update(&sha1ctx, key, klen);
875 		SHA1Update(&sha1ctx, hmac_opad_buffer,
876 		    SHA1_HMAC_BLOCK_LEN - klen);
877 		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
878 	}
879 
880 	for (i = 0; i < klen; i++)
881 		key[i] ^= HMAC_OPAD_VAL;
882 }
883 
884 /*
885  * Allocate a new 'session' and return an encoded session id.  'sidp'
886  * contains our registration id, and should contain an encoded session
887  * id on successful allocation.
888  */
889 static int
ubsec_newsession(device_t dev,u_int32_t * sidp,struct cryptoini * cri)890 ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
891 {
892 	struct ubsec_softc *sc = device_get_softc(dev);
893 	struct cryptoini *c, *encini = NULL, *macini = NULL;
894 	struct ubsec_session *ses = NULL;
895 	int sesn;
896 #if 0
897 	MD5_CTX md5ctx;
898 	SHA1_CTX sha1ctx;
899 	int i;
900 #endif
901 
902 	KASSERT(sc != NULL, ("ubsec_newsession: null softc"));
903 	if (sidp == NULL || cri == NULL || sc == NULL)
904 		return (EINVAL);
905 
906 	for (c = cri; c != NULL; c = c->cri_next) {
907 		if (c->cri_alg == CRYPTO_MD5_HMAC ||
908 		    c->cri_alg == CRYPTO_SHA1_HMAC) {
909 			if (macini)
910 				return (EINVAL);
911 			macini = c;
912 		} else if (c->cri_alg == CRYPTO_DES_CBC ||
913 		    c->cri_alg == CRYPTO_3DES_CBC) {
914 			if (encini)
915 				return (EINVAL);
916 			encini = c;
917 		} else
918 			return (EINVAL);
919 	}
920 	if (encini == NULL && macini == NULL)
921 		return (EINVAL);
922 
923 	if (sc->sc_sessions == NULL) {
924 		ses = sc->sc_sessions = kmalloc(sizeof(struct ubsec_session),
925 						M_DEVBUF, M_INTWAIT);
926 		sesn = 0;
927 		sc->sc_nsessions = 1;
928 	} else {
929 		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
930 			if (sc->sc_sessions[sesn].ses_used == 0) {
931 				ses = &sc->sc_sessions[sesn];
932 				break;
933 			}
934 		}
935 
936 		if (ses == NULL) {
937 			sesn = sc->sc_nsessions;
938 			ses = kmalloc((sesn + 1) * sizeof(struct ubsec_session),
939 					M_DEVBUF, M_INTWAIT);
940 			bcopy(sc->sc_sessions, ses, sesn *
941 			    sizeof(struct ubsec_session));
942 			bzero(sc->sc_sessions, sesn *
943 			    sizeof(struct ubsec_session));
944 			kfree(sc->sc_sessions, M_DEVBUF);
945 			sc->sc_sessions = ses;
946 			ses = &sc->sc_sessions[sesn];
947 			sc->sc_nsessions++;
948 		}
949 	}
950 
951 	bzero(ses, sizeof(struct ubsec_session));
952 	ses->ses_used = 1;
953 	if (encini) {
954 		read_random(ses->ses_iv, sizeof(ses->ses_iv), 0);
955 		if (encini->cri_key != NULL) {
956 			ubsec_setup_enckey(ses, encini->cri_alg,
957 			    encini->cri_key);
958 		}
959 	}
960 
961 	if (macini) {
962 		ses->ses_mlen = macini->cri_mlen;
963 		if (ses->ses_mlen == 0) {
964 			if (macini->cri_alg == CRYPTO_MD5_HMAC)
965 				ses->ses_mlen = MD5_HASH_LEN;
966 			else
967 				ses->ses_mlen = SHA1_HASH_LEN;
968 		}
969 
970 		if (macini->cri_key != NULL) {
971 			ubsec_setup_mackey(ses, macini->cri_alg,
972 			    macini->cri_key, macini->cri_klen/8);
973 		}
974 	}
975 
976 	*sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
977 	return (0);
978 }
979 
980 /*
981  * Deallocate a session.
982  */
983 static int
ubsec_freesession(device_t dev,u_int64_t tid)984 ubsec_freesession(device_t dev, u_int64_t tid)
985 {
986 	struct ubsec_softc *sc = device_get_softc(dev);
987 	int session;
988 	u_int32_t sid = CRYPTO_SESID2LID(tid);
989 
990 	KASSERT(sc != NULL, ("ubsec_freesession: null softc"));
991 	if (sc == NULL)
992 		return (EINVAL);
993 
994 	session = UBSEC_SESSION(sid);
995 	if (session >= sc->sc_nsessions)
996 		return (EINVAL);
997 
998 	bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
999 	return (0);
1000 }
1001 
1002 static void
ubsec_op_cb(void * arg,bus_dma_segment_t * seg,int nsegs,bus_size_t mapsize,int error)1003 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1004 {
1005 	struct ubsec_operand *op = arg;
1006 
1007 	KASSERT(nsegs <= UBS_MAX_SCATTER,
1008 		("Too many DMA segments returned when mapping operand"));
1009 #ifdef UBSEC_DEBUG
1010 	if (ubsec_debug)
1011 		kprintf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
1012 			(u_int) mapsize, nsegs, error);
1013 #endif
1014 	if (error != 0)
1015 		return;
1016 
1017 	op->mapsize = mapsize;
1018 	op->nsegs = nsegs;
1019 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1020 }
1021 
1022 static int
ubsec_process(device_t dev,struct cryptop * crp,int hint)1023 ubsec_process(device_t dev, struct cryptop *crp, int hint)
1024 {
1025 	struct ubsec_softc *sc = device_get_softc(dev);
1026 	struct ubsec_q *q = NULL;
1027 	int err = 0, i, j, nicealign;
1028 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1029 	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1030 	int sskip, dskip, stheend, dtheend;
1031 	int16_t coffset;
1032 	struct ubsec_session *ses;
1033 	struct ubsec_pktctx ctx;
1034 	struct ubsec_dma *dmap = NULL;
1035 
1036 	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1037 		ubsecstats.hst_invalid++;
1038 		return (EINVAL);
1039 	}
1040 	if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1041 		ubsecstats.hst_badsession++;
1042 		return (EINVAL);
1043 	}
1044 
1045 	crit_enter();
1046 
1047 	if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1048 		ubsecstats.hst_queuefull++;
1049 		sc->sc_needwakeup |= CRYPTO_SYMQ;
1050 		crit_exit();
1051 		return (ERESTART);
1052 	}
1053 	q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1054 	SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
1055 	crit_exit();
1056 
1057 	dmap = q->q_dma; /* Save dma pointer */
1058 	bzero(q, sizeof(struct ubsec_q));
1059 	bzero(&ctx, sizeof(ctx));
1060 
1061 	q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1062 	q->q_dma = dmap;
1063 	ses = &sc->sc_sessions[q->q_sesn];
1064 
1065 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1066 		q->q_src_m = (struct mbuf *)crp->crp_buf;
1067 		q->q_dst_m = (struct mbuf *)crp->crp_buf;
1068 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1069 		q->q_src_io = (struct uio *)crp->crp_buf;
1070 		q->q_dst_io = (struct uio *)crp->crp_buf;
1071 	} else {
1072 		ubsecstats.hst_badflags++;
1073 		err = EINVAL;
1074 		goto errout;	/* XXX we don't handle contiguous blocks! */
1075 	}
1076 
1077 	bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1078 
1079 	dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1080 	dmap->d_dma->d_mcr.mcr_flags = 0;
1081 	q->q_crp = crp;
1082 
1083 	crd1 = crp->crp_desc;
1084 	if (crd1 == NULL) {
1085 		ubsecstats.hst_nodesc++;
1086 		err = EINVAL;
1087 		goto errout;
1088 	}
1089 	crd2 = crd1->crd_next;
1090 
1091 	if (crd2 == NULL) {
1092 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1093 		    crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1094 			maccrd = crd1;
1095 			enccrd = NULL;
1096 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1097 		    crd1->crd_alg == CRYPTO_3DES_CBC) {
1098 			maccrd = NULL;
1099 			enccrd = crd1;
1100 		} else {
1101 			ubsecstats.hst_badalg++;
1102 			err = EINVAL;
1103 			goto errout;
1104 		}
1105 	} else {
1106 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1107 		    crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1108 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
1109 			crd2->crd_alg == CRYPTO_3DES_CBC) &&
1110 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1111 			maccrd = crd1;
1112 			enccrd = crd2;
1113 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1114 		    crd1->crd_alg == CRYPTO_3DES_CBC) &&
1115 		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1116 			crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1117 		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
1118 			enccrd = crd1;
1119 			maccrd = crd2;
1120 		} else {
1121 			/*
1122 			 * We cannot order the ubsec as requested
1123 			 */
1124 			ubsecstats.hst_badalg++;
1125 			err = EINVAL;
1126 			goto errout;
1127 		}
1128 	}
1129 
1130 	if (enccrd) {
1131 		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1132 			ubsec_setup_enckey(ses, enccrd->crd_alg,
1133 			    enccrd->crd_key);
1134 		}
1135 
1136 		encoffset = enccrd->crd_skip;
1137 		ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1138 
1139 		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1140 			q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1141 
1142 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1143 				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1144 			else {
1145 				ctx.pc_iv[0] = ses->ses_iv[0];
1146 				ctx.pc_iv[1] = ses->ses_iv[1];
1147 			}
1148 
1149 			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1150 				crypto_copyback(crp->crp_flags, crp->crp_buf,
1151 				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1152 			}
1153 		} else {
1154 			ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1155 
1156 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1157 				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1158 			else {
1159 				crypto_copydata(crp->crp_flags, crp->crp_buf,
1160 				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1161 			}
1162 		}
1163 
1164 		ctx.pc_deskey[0] = ses->ses_deskey[0];
1165 		ctx.pc_deskey[1] = ses->ses_deskey[1];
1166 		ctx.pc_deskey[2] = ses->ses_deskey[2];
1167 		ctx.pc_deskey[3] = ses->ses_deskey[3];
1168 		ctx.pc_deskey[4] = ses->ses_deskey[4];
1169 		ctx.pc_deskey[5] = ses->ses_deskey[5];
1170 		SWAP32(ctx.pc_iv[0]);
1171 		SWAP32(ctx.pc_iv[1]);
1172 	}
1173 
1174 	if (maccrd) {
1175 		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1176 			ubsec_setup_mackey(ses, maccrd->crd_alg,
1177 			    maccrd->crd_key, maccrd->crd_klen / 8);
1178 		}
1179 
1180 		macoffset = maccrd->crd_skip;
1181 
1182 		if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1183 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1184 		else
1185 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1186 
1187 		for (i = 0; i < 5; i++) {
1188 			ctx.pc_hminner[i] = ses->ses_hminner[i];
1189 			ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1190 
1191 			HTOLE32(ctx.pc_hminner[i]);
1192 			HTOLE32(ctx.pc_hmouter[i]);
1193 		}
1194 	}
1195 
1196 	if (enccrd && maccrd) {
1197 		/*
1198 		 * ubsec cannot handle packets where the end of encryption
1199 		 * and authentication are not the same, or where the
1200 		 * encrypted part begins before the authenticated part.
1201 		 */
1202 		if ((encoffset + enccrd->crd_len) !=
1203 		    (macoffset + maccrd->crd_len)) {
1204 			ubsecstats.hst_lenmismatch++;
1205 			err = EINVAL;
1206 			goto errout;
1207 		}
1208 		if (enccrd->crd_skip < maccrd->crd_skip) {
1209 			ubsecstats.hst_skipmismatch++;
1210 			err = EINVAL;
1211 			goto errout;
1212 		}
1213 		sskip = maccrd->crd_skip;
1214 		cpskip = dskip = enccrd->crd_skip;
1215 		stheend = maccrd->crd_len;
1216 		dtheend = enccrd->crd_len;
1217 		coffset = enccrd->crd_skip - maccrd->crd_skip;
1218 		cpoffset = cpskip + dtheend;
1219 #ifdef UBSEC_DEBUG
1220 		if (ubsec_debug) {
1221 			kprintf("mac: skip %d, len %d, inject %d\n",
1222 			    maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1223 			kprintf("enc: skip %d, len %d, inject %d\n",
1224 			    enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1225 			kprintf("src: skip %d, len %d\n", sskip, stheend);
1226 			kprintf("dst: skip %d, len %d\n", dskip, dtheend);
1227 			kprintf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1228 			    coffset, stheend, cpskip, cpoffset);
1229 		}
1230 #endif
1231 	} else {
1232 		cpskip = dskip = sskip = macoffset + encoffset;
1233 		dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1234 		cpoffset = cpskip + dtheend;
1235 		coffset = 0;
1236 	}
1237 	ctx.pc_offset = htole16(coffset >> 2);
1238 
1239 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1240 		ubsecstats.hst_nomap++;
1241 		err = ENOMEM;
1242 		goto errout;
1243 	}
1244 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
1245 		if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1246 		    q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1247 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1248 			q->q_src_map = NULL;
1249 			ubsecstats.hst_noload++;
1250 			err = ENOMEM;
1251 			goto errout;
1252 		}
1253 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
1254 		if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1255 		    q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1256 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1257 			q->q_src_map = NULL;
1258 			ubsecstats.hst_noload++;
1259 			err = ENOMEM;
1260 			goto errout;
1261 		}
1262 	}
1263 	nicealign = ubsec_dmamap_aligned(&q->q_src);
1264 
1265 	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1266 
1267 #ifdef UBSEC_DEBUG
1268 	if (ubsec_debug)
1269 		kprintf("src skip: %d nicealign: %u\n", sskip, nicealign);
1270 #endif
1271 	for (i = j = 0; i < q->q_src_nsegs; i++) {
1272 		struct ubsec_pktbuf *pb;
1273 		bus_size_t packl = q->q_src_segs[i].ds_len;
1274 		bus_addr_t packp = q->q_src_segs[i].ds_addr;
1275 
1276 		if (sskip >= packl) {
1277 			sskip -= packl;
1278 			continue;
1279 		}
1280 
1281 		packl -= sskip;
1282 		packp += sskip;
1283 		sskip = 0;
1284 
1285 		if (packl > 0xfffc) {
1286 			err = EIO;
1287 			goto errout;
1288 		}
1289 
1290 		if (j == 0)
1291 			pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1292 		else
1293 			pb = &dmap->d_dma->d_sbuf[j - 1];
1294 
1295 		pb->pb_addr = htole32(packp);
1296 
1297 		if (stheend) {
1298 			if (packl > stheend) {
1299 				pb->pb_len = htole32(stheend);
1300 				stheend = 0;
1301 			} else {
1302 				pb->pb_len = htole32(packl);
1303 				stheend -= packl;
1304 			}
1305 		} else
1306 			pb->pb_len = htole32(packl);
1307 
1308 		if ((i + 1) == q->q_src_nsegs)
1309 			pb->pb_next = 0;
1310 		else
1311 			pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1312 			    offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1313 		j++;
1314 	}
1315 
1316 	if (enccrd == NULL && maccrd != NULL) {
1317 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1318 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1319 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1320 		    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1321 #ifdef UBSEC_DEBUG
1322 		if (ubsec_debug)
1323 			kprintf("opkt: %x %x %x\n",
1324 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1325 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1326 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1327 #endif
1328 	} else {
1329 		if (crp->crp_flags & CRYPTO_F_IOV) {
1330 			if (!nicealign) {
1331 				ubsecstats.hst_iovmisaligned++;
1332 				err = EINVAL;
1333 				goto errout;
1334 			}
1335 			if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1336 			     &q->q_dst_map)) {
1337 				ubsecstats.hst_nomap++;
1338 				err = ENOMEM;
1339 				goto errout;
1340 			}
1341 			if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1342 			    q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1343 				bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1344 				q->q_dst_map = NULL;
1345 				ubsecstats.hst_noload++;
1346 				err = ENOMEM;
1347 				goto errout;
1348 			}
1349 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1350 			if (nicealign) {
1351 				q->q_dst = q->q_src;
1352 			} else {
1353 				int totlen, len;
1354 				struct mbuf *m, *top, **mp;
1355 
1356 				ubsecstats.hst_unaligned++;
1357 				totlen = q->q_src_mapsize;
1358 				if (q->q_src_m->m_flags & M_PKTHDR) {
1359 					len = MHLEN;
1360 					MGETHDR(m, M_NOWAIT, MT_DATA);
1361 					if (m && !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
1362 						m_free(m);
1363 						m = NULL;
1364 					}
1365 				} else {
1366 					len = MLEN;
1367 					MGET(m, M_NOWAIT, MT_DATA);
1368 				}
1369 				if (m == NULL) {
1370 					ubsecstats.hst_nombuf++;
1371 					err = sc->sc_nqueue ? ERESTART : ENOMEM;
1372 					goto errout;
1373 				}
1374 				if (totlen >= MINCLSIZE) {
1375 					MCLGET(m, M_NOWAIT);
1376 					if ((m->m_flags & M_EXT) == 0) {
1377 						m_free(m);
1378 						ubsecstats.hst_nomcl++;
1379 						err = sc->sc_nqueue ? ERESTART : ENOMEM;
1380 						goto errout;
1381 					}
1382 					len = MCLBYTES;
1383 				}
1384 				m->m_len = len;
1385 				top = NULL;
1386 				mp = &top;
1387 
1388 				while (totlen > 0) {
1389 					if (top) {
1390 						MGET(m, M_NOWAIT, MT_DATA);
1391 						if (m == NULL) {
1392 							m_freem(top);
1393 							ubsecstats.hst_nombuf++;
1394 							err = sc->sc_nqueue ? ERESTART : ENOMEM;
1395 							goto errout;
1396 						}
1397 						len = MLEN;
1398 					}
1399 					if (top && totlen >= MINCLSIZE) {
1400 						MCLGET(m, M_NOWAIT);
1401 						if ((m->m_flags & M_EXT) == 0) {
1402 							*mp = m;
1403 							m_freem(top);
1404 							ubsecstats.hst_nomcl++;
1405 							err = sc->sc_nqueue ? ERESTART : ENOMEM;
1406 							goto errout;
1407 						}
1408 						len = MCLBYTES;
1409 					}
1410 					m->m_len = len = min(totlen, len);
1411 					totlen -= len;
1412 					*mp = m;
1413 					mp = &m->m_next;
1414 				}
1415 				q->q_dst_m = top;
1416 				ubsec_mcopy(q->q_src_m, q->q_dst_m,
1417 				    cpskip, cpoffset);
1418 				if (bus_dmamap_create(sc->sc_dmat,
1419 				    BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1420 					ubsecstats.hst_nomap++;
1421 					err = ENOMEM;
1422 					goto errout;
1423 				}
1424 				if (bus_dmamap_load_mbuf(sc->sc_dmat,
1425 				    q->q_dst_map, q->q_dst_m,
1426 				    ubsec_op_cb, &q->q_dst,
1427 				    BUS_DMA_NOWAIT) != 0) {
1428 					bus_dmamap_destroy(sc->sc_dmat,
1429 					q->q_dst_map);
1430 					q->q_dst_map = NULL;
1431 					ubsecstats.hst_noload++;
1432 					err = ENOMEM;
1433 					goto errout;
1434 				}
1435 			}
1436 		} else {
1437 			ubsecstats.hst_badflags++;
1438 			err = EINVAL;
1439 			goto errout;
1440 		}
1441 
1442 #ifdef UBSEC_DEBUG
1443 		if (ubsec_debug)
1444 			kprintf("dst skip: %d\n", dskip);
1445 #endif
1446 		for (i = j = 0; i < q->q_dst_nsegs; i++) {
1447 			struct ubsec_pktbuf *pb;
1448 			bus_size_t packl = q->q_dst_segs[i].ds_len;
1449 			bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1450 
1451 			if (dskip >= packl) {
1452 				dskip -= packl;
1453 				continue;
1454 			}
1455 
1456 			packl -= dskip;
1457 			packp += dskip;
1458 			dskip = 0;
1459 
1460 			if (packl > 0xfffc) {
1461 				err = EIO;
1462 				goto errout;
1463 			}
1464 
1465 			if (j == 0)
1466 				pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1467 			else
1468 				pb = &dmap->d_dma->d_dbuf[j - 1];
1469 
1470 			pb->pb_addr = htole32(packp);
1471 
1472 			if (dtheend) {
1473 				if (packl > dtheend) {
1474 					pb->pb_len = htole32(dtheend);
1475 					dtheend = 0;
1476 				} else {
1477 					pb->pb_len = htole32(packl);
1478 					dtheend -= packl;
1479 				}
1480 			} else
1481 				pb->pb_len = htole32(packl);
1482 
1483 			if ((i + 1) == q->q_dst_nsegs) {
1484 				if (maccrd)
1485 					pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1486 					    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1487 				else
1488 					pb->pb_next = 0;
1489 			} else
1490 				pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1491 				    offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1492 			j++;
1493 		}
1494 	}
1495 
1496 	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1497 	    offsetof(struct ubsec_dmachunk, d_ctx));
1498 
1499 	if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1500 		struct ubsec_pktctx_long *ctxl;
1501 
1502 		ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1503 		    offsetof(struct ubsec_dmachunk, d_ctx));
1504 
1505 		/* transform small context into long context */
1506 		ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1507 		ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1508 		ctxl->pc_flags = ctx.pc_flags;
1509 		ctxl->pc_offset = ctx.pc_offset;
1510 		for (i = 0; i < 6; i++)
1511 			ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1512 		for (i = 0; i < 5; i++)
1513 			ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1514 		for (i = 0; i < 5; i++)
1515 			ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1516 		ctxl->pc_iv[0] = ctx.pc_iv[0];
1517 		ctxl->pc_iv[1] = ctx.pc_iv[1];
1518 	} else
1519 		bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1520 		    offsetof(struct ubsec_dmachunk, d_ctx),
1521 		    sizeof(struct ubsec_pktctx));
1522 
1523 	crit_enter();
1524 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1525 	sc->sc_nqueue++;
1526 	ubsecstats.hst_ipackets++;
1527 	ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1528 	if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1529 		ubsec_feed(sc);
1530 	crit_exit();
1531 	return (0);
1532 
1533 errout:
1534 	if (q != NULL) {
1535 		if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1536 			m_freem(q->q_dst_m);
1537 
1538 		if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1539 			bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1540 			bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1541 		}
1542 		if (q->q_src_map != NULL) {
1543 			bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1544 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1545 		}
1546 
1547 		crit_enter();
1548 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1549 		crit_exit();
1550 	}
1551 	if (err != ERESTART) {
1552 		crp->crp_etype = err;
1553 		crypto_done(crp);
1554 	} else {
1555 		sc->sc_needwakeup |= CRYPTO_SYMQ;
1556 	}
1557 	return (err);
1558 }
1559 
1560 static void
ubsec_callback(struct ubsec_softc * sc,struct ubsec_q * q)1561 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1562 {
1563 	struct cryptop *crp = (struct cryptop *)q->q_crp;
1564 	struct cryptodesc *crd;
1565 	struct ubsec_dma *dmap = q->q_dma;
1566 
1567 	ubsecstats.hst_opackets++;
1568 	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1569 
1570 	ubsec_dma_sync(&dmap->d_alloc,
1571 	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1572 	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1573 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1574 		    BUS_DMASYNC_POSTREAD);
1575 		bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1576 		bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1577 	}
1578 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1579 	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1580 	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1581 
1582 	if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1583 		m_freem(q->q_src_m);
1584 		crp->crp_buf = (caddr_t)q->q_dst_m;
1585 	}
1586 
1587 	/* copy out IV for future use */
1588 	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1589 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1590 			if (crd->crd_alg != CRYPTO_DES_CBC &&
1591 			    crd->crd_alg != CRYPTO_3DES_CBC)
1592 				continue;
1593 			crypto_copydata(crp->crp_flags, crp->crp_buf,
1594 			    crd->crd_skip + crd->crd_len - 8, 8,
1595 			    (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1596 			break;
1597 		}
1598 	}
1599 
1600 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1601 		if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1602 		    crd->crd_alg != CRYPTO_SHA1_HMAC)
1603 			continue;
1604 		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
1605 		    sc->sc_sessions[q->q_sesn].ses_mlen,
1606 		    (caddr_t)dmap->d_dma->d_macbuf);
1607 	}
1608 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1609 	crypto_done(crp);
1610 }
1611 
1612 static void
ubsec_mcopy(struct mbuf * srcm,struct mbuf * dstm,int hoffset,int toffset)1613 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1614 {
1615 	int i, j, dlen, slen;
1616 	caddr_t dptr, sptr;
1617 
1618 	j = 0;
1619 	sptr = srcm->m_data;
1620 	slen = srcm->m_len;
1621 	dptr = dstm->m_data;
1622 	dlen = dstm->m_len;
1623 
1624 	while (1) {
1625 		for (i = 0; i < min(slen, dlen); i++) {
1626 			if (j < hoffset || j >= toffset)
1627 				*dptr++ = *sptr++;
1628 			slen--;
1629 			dlen--;
1630 			j++;
1631 		}
1632 		if (slen == 0) {
1633 			srcm = srcm->m_next;
1634 			if (srcm == NULL)
1635 				return;
1636 			sptr = srcm->m_data;
1637 			slen = srcm->m_len;
1638 		}
1639 		if (dlen == 0) {
1640 			dstm = dstm->m_next;
1641 			if (dstm == NULL)
1642 				return;
1643 			dptr = dstm->m_data;
1644 			dlen = dstm->m_len;
1645 		}
1646 	}
1647 }
1648 
1649 /*
1650  * feed the key generator, must be called at splimp() or higher.
1651  */
1652 static int
ubsec_feed2(struct ubsec_softc * sc)1653 ubsec_feed2(struct ubsec_softc *sc)
1654 {
1655 	struct ubsec_q2 *q;
1656 
1657 	while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1658 		if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1659 			break;
1660 		q = SIMPLEQ_FIRST(&sc->sc_queue2);
1661 
1662 		ubsec_dma_sync(&q->q_mcr,
1663 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1664 		ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1665 
1666 		WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1667 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
1668 		--sc->sc_nqueue2;
1669 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1670 	}
1671 	return (0);
1672 }
1673 
1674 /*
1675  * Callback for handling random numbers
1676  */
1677 static void
ubsec_callback2(struct ubsec_softc * sc,struct ubsec_q2 * q)1678 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1679 {
1680 	struct cryptkop *krp;
1681 	struct ubsec_ctx_keyop *ctx;
1682 
1683 	ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1684 	ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1685 
1686 	switch (q->q_type) {
1687 #ifndef UBSEC_NO_RNG
1688 	case UBS_CTXOP_RNGBYPASS: {
1689 		struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1690 
1691 		ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1692 		(*sc->sc_harvest)(sc->sc_rndtest,
1693 			rng->rng_buf.dma_vaddr,
1694 			UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1695 		rng->rng_used = 0;
1696 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1697 		break;
1698 	}
1699 #endif
1700 	case UBS_CTXOP_MODEXP: {
1701 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1702 		u_int rlen, clen;
1703 
1704 		krp = me->me_krp;
1705 		rlen = (me->me_modbits + 7) / 8;
1706 		clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1707 
1708 		ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1709 		ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1710 		ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1711 		ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1712 
1713 		if (clen < rlen)
1714 			krp->krp_status = E2BIG;
1715 		else {
1716 			if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1717 				bzero(krp->krp_param[krp->krp_iparams].crp_p,
1718 				    (krp->krp_param[krp->krp_iparams].crp_nbits
1719 					+ 7) / 8);
1720 				bcopy(me->me_C.dma_vaddr,
1721 				    krp->krp_param[krp->krp_iparams].crp_p,
1722 				    (me->me_modbits + 7) / 8);
1723 			} else
1724 				ubsec_kshift_l(me->me_shiftbits,
1725 				    me->me_C.dma_vaddr, me->me_normbits,
1726 				    krp->krp_param[krp->krp_iparams].crp_p,
1727 				    krp->krp_param[krp->krp_iparams].crp_nbits);
1728 		}
1729 
1730 		crypto_kdone(krp);
1731 
1732 		/* bzero all potentially sensitive data */
1733 		bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1734 		bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1735 		bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1736 		bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1737 
1738 		/* Can't free here, so put us on the free list. */
1739 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1740 		break;
1741 	}
1742 	case UBS_CTXOP_RSAPRIV: {
1743 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1744 		u_int len;
1745 
1746 		krp = rp->rpr_krp;
1747 		ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1748 		ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1749 
1750 		len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1751 		bcopy(rp->rpr_msgout.dma_vaddr,
1752 		    krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1753 
1754 		crypto_kdone(krp);
1755 
1756 		bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1757 		bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1758 		bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1759 
1760 		/* Can't free here, so put us on the free list. */
1761 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1762 		break;
1763 	}
1764 	default:
1765 		device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1766 		    letoh16(ctx->ctx_op));
1767 		break;
1768 	}
1769 }
1770 
1771 #ifndef UBSEC_NO_RNG
1772 static void
ubsec_rng(void * vsc)1773 ubsec_rng(void *vsc)
1774 {
1775 	struct ubsec_softc *sc = vsc;
1776 	struct ubsec_q2_rng *rng = &sc->sc_rng;
1777 	struct ubsec_mcr *mcr;
1778 	struct ubsec_ctx_rngbypass *ctx;
1779 
1780 	crit_enter();
1781 	if (rng->rng_used) {
1782 		crit_exit();
1783 		return;
1784 	}
1785 	sc->sc_nqueue2++;
1786 	if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1787 		goto out;
1788 
1789 	mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1790 	ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1791 
1792 	mcr->mcr_pkts = htole16(1);
1793 	mcr->mcr_flags = 0;
1794 	mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1795 	mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1796 	mcr->mcr_ipktbuf.pb_len = 0;
1797 	mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1798 	mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1799 	mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1800 	    UBS_PKTBUF_LEN);
1801 	mcr->mcr_opktbuf.pb_next = 0;
1802 
1803 	ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1804 	ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1805 	rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1806 
1807 	ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1808 
1809 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1810 	rng->rng_used = 1;
1811 	ubsec_feed2(sc);
1812 	ubsecstats.hst_rng++;
1813 	crit_exit();
1814 
1815 	return;
1816 
1817 out:
1818 	/*
1819 	 * Something weird happened, generate our own call back.
1820 	 */
1821 	sc->sc_nqueue2--;
1822 	crit_exit();
1823 	callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1824 }
1825 #endif /* UBSEC_NO_RNG */
1826 
1827 static void
ubsec_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)1828 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1829 {
1830 	bus_addr_t *paddr = (bus_addr_t*) arg;
1831 	*paddr = segs->ds_addr;
1832 }
1833 
1834 static int
ubsec_dma_malloc(struct ubsec_softc * sc,bus_size_t size,struct ubsec_dma_alloc * dma,int mapflags)1835 ubsec_dma_malloc(
1836 	struct ubsec_softc *sc,
1837 	bus_size_t size,
1838 	struct ubsec_dma_alloc *dma,
1839 	int mapflags
1840 )
1841 {
1842 	int r;
1843 
1844 	/* XXX could specify sc_dmat as parent but that just adds overhead */
1845 	r = bus_dma_tag_create(NULL,			/* parent */
1846 			       1, 0,			/* alignment, bounds */
1847 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
1848 			       BUS_SPACE_MAXADDR,	/* highaddr */
1849 			       size,			/* maxsize */
1850 			       1,			/* nsegments */
1851 			       size,			/* maxsegsize */
1852 			       BUS_DMA_ALLOCNOW,	/* flags */
1853 			       &dma->dma_tag);
1854 	if (r != 0) {
1855 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1856 			"bus_dma_tag_create failed; error %u\n", r);
1857 		goto fail_0;
1858 	}
1859 
1860 	r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1861 	if (r != 0) {
1862 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1863 			"bus_dmamap_create failed; error %u\n", r);
1864 		goto fail_1;
1865 	}
1866 
1867 	r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1868 			     BUS_DMA_NOWAIT, &dma->dma_map);
1869 	if (r != 0) {
1870 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1871 			"bus_dmammem_alloc failed; size %ju, error %u\n",
1872 			(intmax_t)size, r);
1873 		goto fail_2;
1874 	}
1875 
1876 	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1877 		            size,
1878 			    ubsec_dmamap_cb,
1879 			    &dma->dma_paddr,
1880 			    mapflags | BUS_DMA_NOWAIT);
1881 	if (r != 0) {
1882 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1883 			"bus_dmamap_load failed; error %u\n", r);
1884 		goto fail_3;
1885 	}
1886 
1887 	dma->dma_size = size;
1888 	return (0);
1889 
1890 fail_3:
1891 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1892 fail_2:
1893 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1894 fail_1:
1895 	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1896 	bus_dma_tag_destroy(dma->dma_tag);
1897 fail_0:
1898 	dma->dma_map = NULL;
1899 	dma->dma_tag = NULL;
1900 	return (r);
1901 }
1902 
1903 static void
ubsec_dma_free(struct ubsec_softc * sc,struct ubsec_dma_alloc * dma)1904 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1905 {
1906 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1907 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1908 	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1909 	bus_dma_tag_destroy(dma->dma_tag);
1910 }
1911 
1912 /*
1913  * Resets the board.  Values in the regesters are left as is
1914  * from the reset (i.e. initial values are assigned elsewhere).
1915  */
1916 static void
ubsec_reset_board(struct ubsec_softc * sc)1917 ubsec_reset_board(struct ubsec_softc *sc)
1918 {
1919     volatile u_int32_t ctrl;
1920 
1921     ctrl = READ_REG(sc, BS_CTRL);
1922     ctrl |= BS_CTRL_RESET;
1923     WRITE_REG(sc, BS_CTRL, ctrl);
1924 
1925     /*
1926      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1927      */
1928     DELAY(10);
1929 }
1930 
1931 /*
1932  * Init Broadcom registers
1933  */
1934 static void
ubsec_init_board(struct ubsec_softc * sc)1935 ubsec_init_board(struct ubsec_softc *sc)
1936 {
1937 	u_int32_t ctrl;
1938 
1939 	ctrl = READ_REG(sc, BS_CTRL);
1940 	ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1941 	ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1942 
1943 	if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1944 		ctrl |= BS_CTRL_MCR2INT;
1945 	else
1946 		ctrl &= ~BS_CTRL_MCR2INT;
1947 
1948 	if (sc->sc_flags & UBS_FLAGS_HWNORM)
1949 		ctrl &= ~BS_CTRL_SWNORM;
1950 
1951 	WRITE_REG(sc, BS_CTRL, ctrl);
1952 }
1953 
1954 /*
1955  * Init Broadcom PCI registers
1956  */
1957 static void
ubsec_init_pciregs(device_t dev)1958 ubsec_init_pciregs(device_t dev)
1959 {
1960 #if 0
1961 	u_int32_t misc;
1962 
1963 	misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1964 	misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1965 	    | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1966 	misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1967 	    | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1968 	pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1969 #endif
1970 
1971 	/*
1972 	 * This will set the cache line size to 1, this will
1973 	 * force the BCM58xx chip just to do burst read/writes.
1974 	 * Cache line read/writes are to slow
1975 	 */
1976 	pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1977 }
1978 
1979 /*
1980  * Clean up after a chip crash.
1981  * It is assumed that the caller in splimp()
1982  */
1983 static void
ubsec_cleanchip(struct ubsec_softc * sc)1984 ubsec_cleanchip(struct ubsec_softc *sc)
1985 {
1986 	struct ubsec_q *q;
1987 
1988 	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1989 		q = SIMPLEQ_FIRST(&sc->sc_qchip);
1990 		SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
1991 		ubsec_free_q(sc, q);
1992 	}
1993 	sc->sc_nqchip = 0;
1994 }
1995 
1996 /*
1997  * free a ubsec_q
1998  * It is assumed that the caller is within spimp()
1999  */
2000 static int
ubsec_free_q(struct ubsec_softc * sc,struct ubsec_q * q)2001 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2002 {
2003 	struct ubsec_q *q2;
2004 	struct cryptop *crp;
2005 	int npkts;
2006 	int i;
2007 
2008 	npkts = q->q_nstacked_mcrs;
2009 
2010 	for (i = 0; i < npkts; i++) {
2011 		if(q->q_stacked_mcr[i]) {
2012 			q2 = q->q_stacked_mcr[i];
2013 
2014 			if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2015 				m_freem(q2->q_dst_m);
2016 
2017 			crp = (struct cryptop *)q2->q_crp;
2018 
2019 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2020 
2021 			crp->crp_etype = EFAULT;
2022 			crypto_done(crp);
2023 		} else {
2024 			break;
2025 		}
2026 	}
2027 
2028 	/*
2029 	 * Free header MCR
2030 	 */
2031 	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2032 		m_freem(q->q_dst_m);
2033 
2034 	crp = (struct cryptop *)q->q_crp;
2035 
2036 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2037 
2038 	crp->crp_etype = EFAULT;
2039 	crypto_done(crp);
2040 	return(0);
2041 }
2042 
2043 /*
2044  * Routine to reset the chip and clean up.
2045  * It is assumed that the caller is in splimp()
2046  */
2047 static void
ubsec_totalreset(struct ubsec_softc * sc)2048 ubsec_totalreset(struct ubsec_softc *sc)
2049 {
2050 	ubsec_reset_board(sc);
2051 	ubsec_init_board(sc);
2052 	ubsec_cleanchip(sc);
2053 }
2054 
2055 static int
ubsec_dmamap_aligned(struct ubsec_operand * op)2056 ubsec_dmamap_aligned(struct ubsec_operand *op)
2057 {
2058 	int i;
2059 
2060 	for (i = 0; i < op->nsegs; i++) {
2061 		if (op->segs[i].ds_addr & 3)
2062 			return (0);
2063 		if ((i != (op->nsegs - 1)) &&
2064 		    (op->segs[i].ds_len & 3))
2065 			return (0);
2066 	}
2067 	return (1);
2068 }
2069 
2070 static void
ubsec_kfree(struct ubsec_softc * sc,struct ubsec_q2 * q)2071 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2072 {
2073 	switch (q->q_type) {
2074 	case UBS_CTXOP_MODEXP: {
2075 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2076 
2077 		ubsec_dma_free(sc, &me->me_q.q_mcr);
2078 		ubsec_dma_free(sc, &me->me_q.q_ctx);
2079 		ubsec_dma_free(sc, &me->me_M);
2080 		ubsec_dma_free(sc, &me->me_E);
2081 		ubsec_dma_free(sc, &me->me_C);
2082 		ubsec_dma_free(sc, &me->me_epb);
2083 		kfree(me, M_DEVBUF);
2084 		break;
2085 	}
2086 	case UBS_CTXOP_RSAPRIV: {
2087 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2088 
2089 		ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2090 		ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2091 		ubsec_dma_free(sc, &rp->rpr_msgin);
2092 		ubsec_dma_free(sc, &rp->rpr_msgout);
2093 		kfree(rp, M_DEVBUF);
2094 		break;
2095 	}
2096 	default:
2097 		device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2098 		break;
2099 	}
2100 }
2101 
2102 static int
ubsec_kprocess(device_t dev,struct cryptkop * krp,int hint)2103 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
2104 {
2105 	struct ubsec_softc *sc = device_get_softc(dev);
2106 	int r;
2107 
2108 	if (krp == NULL || krp->krp_callback == NULL)
2109 		return (EINVAL);
2110 
2111 	while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2112 		struct ubsec_q2 *q;
2113 
2114 		q = SIMPLEQ_FIRST(&sc->sc_q2free);
2115 		SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
2116 		ubsec_kfree(sc, q);
2117 	}
2118 
2119 	switch (krp->krp_op) {
2120 	case CRK_MOD_EXP:
2121 		if (sc->sc_flags & UBS_FLAGS_HWNORM)
2122 			r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2123 		else
2124 			r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2125 		break;
2126 	case CRK_MOD_EXP_CRT:
2127 		return (ubsec_kprocess_rsapriv(sc, krp, hint));
2128 	default:
2129 		device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2130 		    krp->krp_op);
2131 		krp->krp_status = EOPNOTSUPP;
2132 		crypto_kdone(krp);
2133 		return (0);
2134 	}
2135 	return (0);			/* silence compiler */
2136 }
2137 
2138 /*
2139  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2140  */
2141 static int
ubsec_kprocess_modexp_sw(struct ubsec_softc * sc,struct cryptkop * krp,int hint)2142 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2143 {
2144 	struct ubsec_q2_modexp *me;
2145 	struct ubsec_mcr *mcr;
2146 	struct ubsec_ctx_modexp *ctx;
2147 	struct ubsec_pktbuf *epb;
2148 	int err = 0;
2149 	u_int nbits, normbits, mbits, shiftbits, ebits;
2150 
2151 	me = kmalloc(sizeof *me, M_DEVBUF, M_INTWAIT | M_ZERO);
2152 	me->me_krp = krp;
2153 	me->me_q.q_type = UBS_CTXOP_MODEXP;
2154 
2155 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2156 	if (nbits <= 512)
2157 		normbits = 512;
2158 	else if (nbits <= 768)
2159 		normbits = 768;
2160 	else if (nbits <= 1024)
2161 		normbits = 1024;
2162 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2163 		normbits = 1536;
2164 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2165 		normbits = 2048;
2166 	else {
2167 		err = E2BIG;
2168 		goto errout;
2169 	}
2170 
2171 	shiftbits = normbits - nbits;
2172 
2173 	me->me_modbits = nbits;
2174 	me->me_shiftbits = shiftbits;
2175 	me->me_normbits = normbits;
2176 
2177 	/* Sanity check: result bits must be >= true modulus bits. */
2178 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2179 		err = ERANGE;
2180 		goto errout;
2181 	}
2182 
2183 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2184 	    &me->me_q.q_mcr, 0)) {
2185 		err = ENOMEM;
2186 		goto errout;
2187 	}
2188 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2189 
2190 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2191 	    &me->me_q.q_ctx, 0)) {
2192 		err = ENOMEM;
2193 		goto errout;
2194 	}
2195 
2196 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2197 	if (mbits > nbits) {
2198 		err = E2BIG;
2199 		goto errout;
2200 	}
2201 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2202 		err = ENOMEM;
2203 		goto errout;
2204 	}
2205 	ubsec_kshift_r(shiftbits,
2206 	    krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2207 	    me->me_M.dma_vaddr, normbits);
2208 
2209 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2210 		err = ENOMEM;
2211 		goto errout;
2212 	}
2213 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2214 
2215 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2216 	if (ebits > nbits) {
2217 		err = E2BIG;
2218 		goto errout;
2219 	}
2220 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2221 		err = ENOMEM;
2222 		goto errout;
2223 	}
2224 	ubsec_kshift_r(shiftbits,
2225 	    krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2226 	    me->me_E.dma_vaddr, normbits);
2227 
2228 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2229 	    &me->me_epb, 0)) {
2230 		err = ENOMEM;
2231 		goto errout;
2232 	}
2233 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2234 	epb->pb_addr = htole32(me->me_E.dma_paddr);
2235 	epb->pb_next = 0;
2236 	epb->pb_len = htole32(normbits / 8);
2237 
2238 #ifdef UBSEC_DEBUG
2239 	if (ubsec_debug) {
2240 		kprintf("Epb ");
2241 		ubsec_dump_pb(epb);
2242 	}
2243 #endif
2244 
2245 	mcr->mcr_pkts = htole16(1);
2246 	mcr->mcr_flags = 0;
2247 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2248 	mcr->mcr_reserved = 0;
2249 	mcr->mcr_pktlen = 0;
2250 
2251 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2252 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2253 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2254 
2255 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2256 	mcr->mcr_opktbuf.pb_next = 0;
2257 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2258 
2259 #ifdef DIAGNOSTIC
2260 	/* Misaligned output buffer will hang the chip. */
2261 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2262 		panic("%s: modexp invalid addr 0x%x",
2263 		    device_get_nameunit(sc->sc_dev),
2264 		    letoh32(mcr->mcr_opktbuf.pb_addr));
2265 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2266 		panic("%s: modexp invalid len 0x%x",
2267 		    device_get_nameunit(sc->sc_dev),
2268 		    letoh32(mcr->mcr_opktbuf.pb_len));
2269 #endif
2270 
2271 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2272 	bzero(ctx, sizeof(*ctx));
2273 	ubsec_kshift_r(shiftbits,
2274 	    krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2275 	    ctx->me_N, normbits);
2276 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2277 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2278 	ctx->me_E_len = htole16(nbits);
2279 	ctx->me_N_len = htole16(nbits);
2280 
2281 #ifdef UBSEC_DEBUG
2282 	if (ubsec_debug) {
2283 		ubsec_dump_mcr(mcr);
2284 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2285 	}
2286 #endif
2287 
2288 	/*
2289 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2290 	 * everything else.
2291 	 */
2292 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2293 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2294 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2295 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2296 
2297 	/* Enqueue and we're done... */
2298 	crit_enter();
2299 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2300 	ubsec_feed2(sc);
2301 	ubsecstats.hst_modexp++;
2302 	crit_exit();
2303 
2304 	return (0);
2305 
2306 errout:
2307 	if (me != NULL) {
2308 		if (me->me_q.q_mcr.dma_map != NULL)
2309 			ubsec_dma_free(sc, &me->me_q.q_mcr);
2310 		if (me->me_q.q_ctx.dma_map != NULL) {
2311 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2312 			ubsec_dma_free(sc, &me->me_q.q_ctx);
2313 		}
2314 		if (me->me_M.dma_map != NULL) {
2315 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2316 			ubsec_dma_free(sc, &me->me_M);
2317 		}
2318 		if (me->me_E.dma_map != NULL) {
2319 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2320 			ubsec_dma_free(sc, &me->me_E);
2321 		}
2322 		if (me->me_C.dma_map != NULL) {
2323 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2324 			ubsec_dma_free(sc, &me->me_C);
2325 		}
2326 		if (me->me_epb.dma_map != NULL)
2327 			ubsec_dma_free(sc, &me->me_epb);
2328 		kfree(me, M_DEVBUF);
2329 	}
2330 	krp->krp_status = err;
2331 	crypto_kdone(krp);
2332 	return (0);
2333 }
2334 
2335 /*
2336  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2337  */
2338 static int
ubsec_kprocess_modexp_hw(struct ubsec_softc * sc,struct cryptkop * krp,int hint)2339 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2340 {
2341 	struct ubsec_q2_modexp *me;
2342 	struct ubsec_mcr *mcr;
2343 	struct ubsec_ctx_modexp *ctx;
2344 	struct ubsec_pktbuf *epb;
2345 	int err = 0;
2346 	u_int nbits, normbits, mbits, shiftbits, ebits;
2347 
2348 	me = kmalloc(sizeof *me, M_DEVBUF, M_INTWAIT | M_ZERO);
2349 	me->me_krp = krp;
2350 	me->me_q.q_type = UBS_CTXOP_MODEXP;
2351 
2352 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2353 	if (nbits <= 512)
2354 		normbits = 512;
2355 	else if (nbits <= 768)
2356 		normbits = 768;
2357 	else if (nbits <= 1024)
2358 		normbits = 1024;
2359 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2360 		normbits = 1536;
2361 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2362 		normbits = 2048;
2363 	else {
2364 		err = E2BIG;
2365 		goto errout;
2366 	}
2367 
2368 	shiftbits = normbits - nbits;
2369 
2370 	/* XXX ??? */
2371 	me->me_modbits = nbits;
2372 	me->me_shiftbits = shiftbits;
2373 	me->me_normbits = normbits;
2374 
2375 	/* Sanity check: result bits must be >= true modulus bits. */
2376 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2377 		err = ERANGE;
2378 		goto errout;
2379 	}
2380 
2381 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2382 	    &me->me_q.q_mcr, 0)) {
2383 		err = ENOMEM;
2384 		goto errout;
2385 	}
2386 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2387 
2388 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2389 	    &me->me_q.q_ctx, 0)) {
2390 		err = ENOMEM;
2391 		goto errout;
2392 	}
2393 
2394 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2395 	if (mbits > nbits) {
2396 		err = E2BIG;
2397 		goto errout;
2398 	}
2399 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2400 		err = ENOMEM;
2401 		goto errout;
2402 	}
2403 	bzero(me->me_M.dma_vaddr, normbits / 8);
2404 	bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2405 	    me->me_M.dma_vaddr, (mbits + 7) / 8);
2406 
2407 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2408 		err = ENOMEM;
2409 		goto errout;
2410 	}
2411 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2412 
2413 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2414 	if (ebits > nbits) {
2415 		err = E2BIG;
2416 		goto errout;
2417 	}
2418 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2419 		err = ENOMEM;
2420 		goto errout;
2421 	}
2422 	bzero(me->me_E.dma_vaddr, normbits / 8);
2423 	bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2424 	    me->me_E.dma_vaddr, (ebits + 7) / 8);
2425 
2426 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2427 	    &me->me_epb, 0)) {
2428 		err = ENOMEM;
2429 		goto errout;
2430 	}
2431 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2432 	epb->pb_addr = htole32(me->me_E.dma_paddr);
2433 	epb->pb_next = 0;
2434 	epb->pb_len = htole32((ebits + 7) / 8);
2435 
2436 #ifdef UBSEC_DEBUG
2437 	if (ubsec_debug) {
2438 		kprintf("Epb ");
2439 		ubsec_dump_pb(epb);
2440 	}
2441 #endif
2442 
2443 	mcr->mcr_pkts = htole16(1);
2444 	mcr->mcr_flags = 0;
2445 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2446 	mcr->mcr_reserved = 0;
2447 	mcr->mcr_pktlen = 0;
2448 
2449 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2450 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2451 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2452 
2453 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2454 	mcr->mcr_opktbuf.pb_next = 0;
2455 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2456 
2457 #ifdef DIAGNOSTIC
2458 	/* Misaligned output buffer will hang the chip. */
2459 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2460 		panic("%s: modexp invalid addr 0x%x",
2461 		    device_get_nameunit(sc->sc_dev),
2462 		    letoh32(mcr->mcr_opktbuf.pb_addr));
2463 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2464 		panic("%s: modexp invalid len 0x%x",
2465 		    device_get_nameunit(sc->sc_dev),
2466 		    letoh32(mcr->mcr_opktbuf.pb_len));
2467 #endif
2468 
2469 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2470 	bzero(ctx, sizeof(*ctx));
2471 	bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2472 	    (nbits + 7) / 8);
2473 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2474 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2475 	ctx->me_E_len = htole16(ebits);
2476 	ctx->me_N_len = htole16(nbits);
2477 
2478 #ifdef UBSEC_DEBUG
2479 	if (ubsec_debug) {
2480 		ubsec_dump_mcr(mcr);
2481 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2482 	}
2483 #endif
2484 
2485 	/*
2486 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2487 	 * everything else.
2488 	 */
2489 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2490 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2491 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2492 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2493 
2494 	/* Enqueue and we're done... */
2495 	crit_enter();
2496 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2497 	ubsec_feed2(sc);
2498 	crit_exit();
2499 
2500 	return (0);
2501 
2502 errout:
2503 	if (me != NULL) {
2504 		if (me->me_q.q_mcr.dma_map != NULL)
2505 			ubsec_dma_free(sc, &me->me_q.q_mcr);
2506 		if (me->me_q.q_ctx.dma_map != NULL) {
2507 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2508 			ubsec_dma_free(sc, &me->me_q.q_ctx);
2509 		}
2510 		if (me->me_M.dma_map != NULL) {
2511 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2512 			ubsec_dma_free(sc, &me->me_M);
2513 		}
2514 		if (me->me_E.dma_map != NULL) {
2515 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2516 			ubsec_dma_free(sc, &me->me_E);
2517 		}
2518 		if (me->me_C.dma_map != NULL) {
2519 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2520 			ubsec_dma_free(sc, &me->me_C);
2521 		}
2522 		if (me->me_epb.dma_map != NULL)
2523 			ubsec_dma_free(sc, &me->me_epb);
2524 		kfree(me, M_DEVBUF);
2525 	}
2526 	krp->krp_status = err;
2527 	crypto_kdone(krp);
2528 	return (0);
2529 }
2530 
2531 static int
ubsec_kprocess_rsapriv(struct ubsec_softc * sc,struct cryptkop * krp,int hint)2532 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2533 {
2534 	struct ubsec_q2_rsapriv *rp = NULL;
2535 	struct ubsec_mcr *mcr;
2536 	struct ubsec_ctx_rsapriv *ctx;
2537 	int err = 0;
2538 	u_int padlen, msglen;
2539 
2540 	msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2541 	padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2542 	if (msglen > padlen)
2543 		padlen = msglen;
2544 
2545 	if (padlen <= 256)
2546 		padlen = 256;
2547 	else if (padlen <= 384)
2548 		padlen = 384;
2549 	else if (padlen <= 512)
2550 		padlen = 512;
2551 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2552 		padlen = 768;
2553 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2554 		padlen = 1024;
2555 	else {
2556 		err = E2BIG;
2557 		goto errout;
2558 	}
2559 
2560 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2561 		err = E2BIG;
2562 		goto errout;
2563 	}
2564 
2565 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2566 		err = E2BIG;
2567 		goto errout;
2568 	}
2569 
2570 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2571 		err = E2BIG;
2572 		goto errout;
2573 	}
2574 
2575 	rp = kmalloc(sizeof *rp, M_DEVBUF, M_INTWAIT | M_ZERO);
2576 	rp->rpr_krp = krp;
2577 	rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2578 
2579 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2580 	    &rp->rpr_q.q_mcr, 0)) {
2581 		err = ENOMEM;
2582 		goto errout;
2583 	}
2584 	mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2585 
2586 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2587 	    &rp->rpr_q.q_ctx, 0)) {
2588 		err = ENOMEM;
2589 		goto errout;
2590 	}
2591 	ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2592 	bzero(ctx, sizeof *ctx);
2593 
2594 	/* Copy in p */
2595 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2596 	    &ctx->rpr_buf[0 * (padlen / 8)],
2597 	    (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2598 
2599 	/* Copy in q */
2600 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2601 	    &ctx->rpr_buf[1 * (padlen / 8)],
2602 	    (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2603 
2604 	/* Copy in dp */
2605 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2606 	    &ctx->rpr_buf[2 * (padlen / 8)],
2607 	    (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2608 
2609 	/* Copy in dq */
2610 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2611 	    &ctx->rpr_buf[3 * (padlen / 8)],
2612 	    (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2613 
2614 	/* Copy in pinv */
2615 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2616 	    &ctx->rpr_buf[4 * (padlen / 8)],
2617 	    (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2618 
2619 	msglen = padlen * 2;
2620 
2621 	/* Copy in input message (aligned buffer/length). */
2622 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2623 		/* Is this likely? */
2624 		err = E2BIG;
2625 		goto errout;
2626 	}
2627 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2628 		err = ENOMEM;
2629 		goto errout;
2630 	}
2631 	bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2632 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2633 	    rp->rpr_msgin.dma_vaddr,
2634 	    (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2635 
2636 	/* Prepare space for output message (aligned buffer/length). */
2637 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2638 		/* Is this likely? */
2639 		err = E2BIG;
2640 		goto errout;
2641 	}
2642 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2643 		err = ENOMEM;
2644 		goto errout;
2645 	}
2646 	bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2647 
2648 	mcr->mcr_pkts = htole16(1);
2649 	mcr->mcr_flags = 0;
2650 	mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2651 	mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2652 	mcr->mcr_ipktbuf.pb_next = 0;
2653 	mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2654 	mcr->mcr_reserved = 0;
2655 	mcr->mcr_pktlen = htole16(msglen);
2656 	mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2657 	mcr->mcr_opktbuf.pb_next = 0;
2658 	mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2659 
2660 #ifdef DIAGNOSTIC
2661 	if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2662 		panic("%s: rsapriv: invalid msgin %x(0x%jx)",
2663 		    device_get_nameunit(sc->sc_dev),
2664 		    rp->rpr_msgin.dma_paddr,
2665 		    (uintmax_t)rp->rpr_msgin.dma_size);
2666 	}
2667 	if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2668 		panic("%s: rsapriv: invalid msgout %x(0x%jx)",
2669 		    device_get_nameunit(sc->sc_dev),
2670 		    rp->rpr_msgout.dma_paddr,
2671 		    (uintmax_t)rp->rpr_msgout.dma_size);
2672 	}
2673 #endif
2674 
2675 	ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2676 	ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2677 	ctx->rpr_q_len = htole16(padlen);
2678 	ctx->rpr_p_len = htole16(padlen);
2679 
2680 	/*
2681 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
2682 	 * everything else.
2683 	 */
2684 	ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2685 	ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2686 
2687 	/* Enqueue and we're done... */
2688 	crit_enter();
2689 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2690 	ubsec_feed2(sc);
2691 	ubsecstats.hst_modexpcrt++;
2692 	crit_exit();
2693 	return (0);
2694 
2695 errout:
2696 	if (rp != NULL) {
2697 		if (rp->rpr_q.q_mcr.dma_map != NULL)
2698 			ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2699 		if (rp->rpr_msgin.dma_map != NULL) {
2700 			bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2701 			ubsec_dma_free(sc, &rp->rpr_msgin);
2702 		}
2703 		if (rp->rpr_msgout.dma_map != NULL) {
2704 			bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2705 			ubsec_dma_free(sc, &rp->rpr_msgout);
2706 		}
2707 		kfree(rp, M_DEVBUF);
2708 	}
2709 	krp->krp_status = err;
2710 	crypto_kdone(krp);
2711 	return (0);
2712 }
2713 
2714 #ifdef UBSEC_DEBUG
2715 static void
ubsec_dump_pb(volatile struct ubsec_pktbuf * pb)2716 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2717 {
2718 	kprintf("addr 0x%x (0x%x) next 0x%x\n",
2719 	    pb->pb_addr, pb->pb_len, pb->pb_next);
2720 }
2721 
2722 static void
ubsec_dump_ctx2(struct ubsec_ctx_keyop * c)2723 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2724 {
2725 	kprintf("CTX (0x%x):\n", c->ctx_len);
2726 	switch (letoh16(c->ctx_op)) {
2727 	case UBS_CTXOP_RNGBYPASS:
2728 	case UBS_CTXOP_RNGSHA1:
2729 		break;
2730 	case UBS_CTXOP_MODEXP:
2731 	{
2732 		struct ubsec_ctx_modexp *cx = (void *)c;
2733 		int i, len;
2734 
2735 		kprintf(" Elen %u, Nlen %u\n",
2736 		    letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2737 		len = (cx->me_N_len + 7)/8;
2738 		for (i = 0; i < len; i++)
2739 			kprintf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2740 		kprintf("\n");
2741 		break;
2742 	}
2743 	default:
2744 		kprintf("unknown context: %x\n", c->ctx_op);
2745 	}
2746 	kprintf("END CTX\n");
2747 }
2748 
2749 static void
ubsec_dump_mcr(struct ubsec_mcr * mcr)2750 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2751 {
2752 	volatile struct ubsec_mcr_add *ma;
2753 	int i;
2754 
2755 	kprintf("MCR:\n");
2756 	kprintf(" pkts: %u, flags 0x%x\n",
2757 	    letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2758 	ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2759 	for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2760 		kprintf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2761 		    letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2762 		    letoh16(ma->mcr_reserved));
2763 		kprintf(" %d: ipkt ", i);
2764 		ubsec_dump_pb(&ma->mcr_ipktbuf);
2765 		kprintf(" %d: opkt ", i);
2766 		ubsec_dump_pb(&ma->mcr_opktbuf);
2767 		ma++;
2768 	}
2769 	kprintf("END MCR\n");
2770 }
2771 #endif /* UBSEC_DEBUG */
2772 
2773 /*
2774  * Return the number of significant bits of a big number.
2775  */
2776 static int
ubsec_ksigbits(struct crparam * cr)2777 ubsec_ksigbits(struct crparam *cr)
2778 {
2779 	u_int plen = (cr->crp_nbits + 7) / 8;
2780 	int i, sig = plen * 8;
2781 	u_int8_t c, *p = cr->crp_p;
2782 
2783 	for (i = plen - 1; i >= 0; i--) {
2784 		c = p[i];
2785 		if (c != 0) {
2786 			while ((c & 0x80) == 0) {
2787 				sig--;
2788 				c <<= 1;
2789 			}
2790 			break;
2791 		}
2792 		sig -= 8;
2793 	}
2794 	return (sig);
2795 }
2796 
2797 static void
ubsec_kshift_r(u_int shiftbits,u_int8_t * src,u_int srcbits,u_int8_t * dst,u_int dstbits)2798 ubsec_kshift_r(
2799 	u_int shiftbits,
2800 	u_int8_t *src, u_int srcbits,
2801 	u_int8_t *dst, u_int dstbits)
2802 {
2803 	u_int slen, dlen;
2804 	int i, si, di, n;
2805 
2806 	slen = (srcbits + 7) / 8;
2807 	dlen = (dstbits + 7) / 8;
2808 
2809 	for (i = 0; i < slen; i++)
2810 		dst[i] = src[i];
2811 	for (i = 0; i < dlen - slen; i++)
2812 		dst[slen + i] = 0;
2813 
2814 	n = shiftbits / 8;
2815 	if (n != 0) {
2816 		si = dlen - n - 1;
2817 		di = dlen - 1;
2818 		while (si >= 0)
2819 			dst[di--] = dst[si--];
2820 		while (di >= 0)
2821 			dst[di--] = 0;
2822 	}
2823 
2824 	n = shiftbits % 8;
2825 	if (n != 0) {
2826 		for (i = dlen - 1; i > 0; i--)
2827 			dst[i] = (dst[i] << n) |
2828 			    (dst[i - 1] >> (8 - n));
2829 		dst[0] = dst[0] << n;
2830 	}
2831 }
2832 
2833 static void
ubsec_kshift_l(u_int shiftbits,u_int8_t * src,u_int srcbits,u_int8_t * dst,u_int dstbits)2834 ubsec_kshift_l(
2835 	u_int shiftbits,
2836 	u_int8_t *src, u_int srcbits,
2837 	u_int8_t *dst, u_int dstbits)
2838 {
2839 	int slen, dlen, i, n;
2840 
2841 	slen = (srcbits + 7) / 8;
2842 	dlen = (dstbits + 7) / 8;
2843 
2844 	n = shiftbits / 8;
2845 	for (i = 0; i < slen; i++)
2846 		dst[i] = src[i + n];
2847 	for (i = 0; i < dlen - slen; i++)
2848 		dst[slen + i] = 0;
2849 
2850 	n = shiftbits % 8;
2851 	if (n != 0) {
2852 		for (i = 0; i < (dlen - 1); i++)
2853 			dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2854 		dst[dlen - 1] = dst[dlen - 1] >> n;
2855 	}
2856 }
2857