1 /* $OpenBSD: glxsb.c,v 1.42 2023/01/30 10:49:05 jsg Exp $ */
2
3 /*
4 * Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
5 * Copyright (c) 2003, 2004 Theo de Raadt
6 * Copyright (c) 2003 Jason Wright
7 *
8 * Permission to use, copy, modify, and distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 */
20
21 /*
22 * Driver for the security block on the AMD Geode LX processors
23 * http://www.amd.com/files/connectivitysolutions/geode/geode_lx/33234d_lx_ds.pdf
24 */
25
26 #include <sys/param.h>
27 #include <sys/systm.h>
28 #include <sys/device.h>
29 #include <sys/malloc.h>
30 #include <sys/mbuf.h>
31 #include <sys/timeout.h>
32
33 #include <machine/bus.h>
34 #include <machine/cpufunc.h>
35
36 #include <dev/pci/pcivar.h>
37 #include <dev/pci/pcidevs.h>
38
39 #ifdef CRYPTO
40 #include <crypto/cryptodev.h>
41 #include <crypto/xform.h>
42 #include <crypto/cryptosoft.h>
43 #endif
44
45 #define SB_GLD_MSR_CAP 0x58002000 /* RO - Capabilities */
46 #define SB_GLD_MSR_CONFIG 0x58002001 /* RW - Master Config */
47 #define SB_GLD_MSR_SMI 0x58002002 /* RW - SMI */
48 #define SB_GLD_MSR_ERROR 0x58002003 /* RW - Error */
49 #define SB_GLD_MSR_PM 0x58002004 /* RW - Power Mgmt */
50 #define SB_GLD_MSR_DIAG 0x58002005 /* RW - Diagnostic */
51 #define SB_GLD_MSR_CTRL 0x58002006 /* RW - Security Block Cntrl */
52
53 /* For GLD_MSR_CTRL: */
54 #define SB_GMC_DIV0 0x0000 /* AES update divisor values */
55 #define SB_GMC_DIV1 0x0001
56 #define SB_GMC_DIV2 0x0002
57 #define SB_GMC_DIV3 0x0003
58 #define SB_GMC_DIV_MASK 0x0003
59 #define SB_GMC_SBI 0x0004 /* AES swap bits */
60 #define SB_GMC_SBY 0x0008 /* AES swap bytes */
61 #define SB_GMC_TW 0x0010 /* Time write (EEPROM) */
62 #define SB_GMC_T_SEL0 0x0000 /* RNG post-proc: none */
63 #define SB_GMC_T_SEL1 0x0100 /* RNG post-proc: LFSR */
64 #define SB_GMC_T_SEL2 0x0200 /* RNG post-proc: whitener */
65 #define SB_GMC_T_SEL3 0x0300 /* RNG LFSR+whitener */
66 #define SB_GMC_T_SEL_MASK 0x0300
67 #define SB_GMC_T_NE 0x0400 /* Noise (generator) Enable */
68 #define SB_GMC_T_TM 0x0800 /* RNG test mode */
69 /* (deterministic) */
70
71 /* Security Block configuration/control registers (offsets from base) */
72
73 #define SB_CTL_A 0x0000 /* RW - SB Control A */
74 #define SB_CTL_B 0x0004 /* RW - SB Control B */
75 #define SB_AES_INT 0x0008 /* RW - SB AES Interrupt */
76 #define SB_SOURCE_A 0x0010 /* RW - Source A */
77 #define SB_DEST_A 0x0014 /* RW - Destination A */
78 #define SB_LENGTH_A 0x0018 /* RW - Length A */
79 #define SB_SOURCE_B 0x0020 /* RW - Source B */
80 #define SB_DEST_B 0x0024 /* RW - Destination B */
81 #define SB_LENGTH_B 0x0028 /* RW - Length B */
82 #define SB_WKEY 0x0030 /* WO - Writable Key 0-3 */
83 #define SB_WKEY_0 0x0030 /* WO - Writable Key 0 */
84 #define SB_WKEY_1 0x0034 /* WO - Writable Key 1 */
85 #define SB_WKEY_2 0x0038 /* WO - Writable Key 2 */
86 #define SB_WKEY_3 0x003C /* WO - Writable Key 3 */
87 #define SB_CBC_IV 0x0040 /* RW - CBC IV 0-3 */
88 #define SB_CBC_IV_0 0x0040 /* RW - CBC IV 0 */
89 #define SB_CBC_IV_1 0x0044 /* RW - CBC IV 1 */
90 #define SB_CBC_IV_2 0x0048 /* RW - CBC IV 2 */
91 #define SB_CBC_IV_3 0x004C /* RW - CBC IV 3 */
92 #define SB_RANDOM_NUM 0x0050 /* RW - Random Number */
93 #define SB_RANDOM_NUM_STATUS 0x0054 /* RW - Random Number Status */
94 #define SB_EEPROM_COMM 0x0800 /* RW - EEPROM Command */
95 #define SB_EEPROM_ADDR 0x0804 /* RW - EEPROM Address */
96 #define SB_EEPROM_DATA 0x0808 /* RW - EEPROM Data */
97 #define SB_EEPROM_SEC_STATE 0x080C /* RW - EEPROM Security State */
98
99 /* For SB_CTL_A and _B */
100 #define SB_CTL_ST 0x0001 /* Start operation (enc/dec) */
101 #define SB_CTL_ENC 0x0002 /* Encrypt (0 is decrypt) */
102 #define SB_CTL_DEC 0x0000 /* Decrypt */
103 #define SB_CTL_WK 0x0004 /* Use writable key (we set) */
104 #define SB_CTL_DC 0x0008 /* Destination coherent */
105 #define SB_CTL_SC 0x0010 /* Source coherent */
106 #define SB_CTL_CBC 0x0020 /* CBC (0 is ECB) */
107
108 /* For SB_AES_INT */
109 #define SB_AI_DISABLE_AES_A 0x00001 /* Disable AES A compl int */
110 #define SB_AI_ENABLE_AES_A 0x00000 /* Enable AES A compl int */
111 #define SB_AI_DISABLE_AES_B 0x00002 /* Disable AES B compl int */
112 #define SB_AI_ENABLE_AES_B 0x00000 /* Enable AES B compl int */
113 #define SB_AI_DISABLE_EEPROM 0x00004 /* Disable EEPROM op comp int */
114 #define SB_AI_ENABLE_EEPROM 0x00000 /* Enable EEPROM op compl int */
115 #define SB_AI_AES_A_COMPLETE 0x10000 /* AES A operation complete */
116 #define SB_AI_AES_B_COMPLETE 0x20000 /* AES B operation complete */
117 #define SB_AI_EEPROM_COMPLETE 0x40000 /* EEPROM operation complete */
118
119 #define SB_RNS_TRNG_VALID 0x0001 /* in SB_RANDOM_NUM_STATUS */
120
121 #define SB_MEM_SIZE 0x0810 /* Size of memory block */
122
123 #define SB_AES_ALIGN 0x0010 /* Source and dest buffers */
124 /* must be 16-byte aligned */
125 #define SB_AES_BLOCK_SIZE 0x0010
126
127 /*
128 * The Geode LX security block AES acceleration doesn't perform scatter-
129 * gather: it just takes source and destination addresses. Therefore the
130 * plain- and ciphertexts need to be contiguous. To this end, we allocate
131 * a buffer for both, and accept the overhead of copying in and out. If
132 * the number of bytes in one operation is bigger than allowed for by the
133 * buffer (buffer is twice the size of the max length, as it has both input
134 * and output) then we have to perform multiple encryptions/decryptions.
135 */
136 #define GLXSB_MAX_AES_LEN 16384
137
138 #ifdef CRYPTO
139 struct glxsb_dma_map {
140 bus_dmamap_t dma_map;
141 bus_dma_segment_t dma_seg;
142 int dma_nsegs;
143 int dma_size;
144 caddr_t dma_vaddr;
145 uint32_t dma_paddr;
146 };
147 struct glxsb_session {
148 uint32_t ses_key[4];
149 int ses_klen;
150 int ses_used;
151 struct swcr_data *ses_swd_auth;
152 struct swcr_data *ses_swd_enc;
153 };
154 #endif /* CRYPTO */
155
156 struct glxsb_softc {
157 struct device sc_dev;
158 bus_space_tag_t sc_iot;
159 bus_space_handle_t sc_ioh;
160 struct timeout sc_to;
161
162 #ifdef CRYPTO
163 bus_dma_tag_t sc_dmat;
164 struct glxsb_dma_map sc_dma;
165 int32_t sc_cid;
166 int sc_nsessions;
167 struct glxsb_session *sc_sessions;
168 #endif /* CRYPTO */
169
170 uint64_t save_gld_msr;
171 };
172
173 int glxsb_match(struct device *, void *, void *);
174 void glxsb_attach(struct device *, struct device *, void *);
175 int glxsb_activate(struct device *, int);
176 void glxsb_rnd(void *);
177
178 const struct cfattach glxsb_ca = {
179 sizeof(struct glxsb_softc), glxsb_match, glxsb_attach, NULL,
180 glxsb_activate
181 };
182
183 struct cfdriver glxsb_cd = {
184 NULL, "glxsb", DV_DULL
185 };
186
187
188 #ifdef CRYPTO
189
190 #define GLXSB_SESSION(sid) ((sid) & 0x0fffffff)
191 #define GLXSB_SID(crd,ses) (((crd) << 28) | ((ses) & 0x0fffffff))
192
193 static struct glxsb_softc *glxsb_sc;
194
195 int glxsb_crypto_setup(struct glxsb_softc *);
196 int glxsb_crypto_newsession(uint32_t *, struct cryptoini *);
197 int glxsb_crypto_process(struct cryptop *);
198 int glxsb_crypto_freesession(uint64_t);
199 static __inline void glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t,
200 uint32_t, void *, int, void *);
201
202 int glxsb_dma_alloc(struct glxsb_softc *, int, struct glxsb_dma_map *);
203 void glxsb_dma_pre_op(struct glxsb_softc *, struct glxsb_dma_map *);
204 void glxsb_dma_post_op(struct glxsb_softc *, struct glxsb_dma_map *);
205 void glxsb_dma_free(struct glxsb_softc *, struct glxsb_dma_map *);
206
207 #endif /* CRYPTO */
208
209
210 int
glxsb_match(struct device * parent,void * match,void * aux)211 glxsb_match(struct device *parent, void *match, void *aux)
212 {
213 struct pci_attach_args *pa = aux;
214
215 if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_AMD &&
216 PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_AMD_GEODE_LX_CRYPTO)
217 return (1);
218
219 return (0);
220 }
221
222 void
glxsb_attach(struct device * parent,struct device * self,void * aux)223 glxsb_attach(struct device *parent, struct device *self, void *aux)
224 {
225 struct glxsb_softc *sc = (void *) self;
226 struct pci_attach_args *pa = aux;
227 bus_addr_t membase;
228 bus_size_t memsize;
229 uint64_t msr;
230 #ifdef CRYPTO
231 uint32_t intr;
232 #endif
233
234 msr = rdmsr(SB_GLD_MSR_CAP);
235 if ((msr & 0xFFFF00) != 0x130400) {
236 printf(": unknown ID 0x%x\n", (int) ((msr & 0xFFFF00) >> 16));
237 return;
238 }
239
240 /* printf(": revision %d", (int) (msr & 0xFF)); */
241
242 /* Map in the security block configuration/control registers */
243 if (pci_mapreg_map(pa, PCI_MAPREG_START,
244 PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT, 0, &sc->sc_iot,
245 &sc->sc_ioh, &membase, &memsize, SB_MEM_SIZE)) {
246 printf(": can't find mem space\n");
247 return;
248 }
249
250 /*
251 * Configure the Security Block.
252 *
253 * We want to enable the noise generator (T_NE), and enable the
254 * linear feedback shift register and whitener post-processing
255 * (T_SEL = 3). Also ensure that test mode (deterministic values)
256 * is disabled.
257 */
258 msr = rdmsr(SB_GLD_MSR_CTRL);
259 msr &= ~(SB_GMC_T_TM | SB_GMC_T_SEL_MASK);
260 msr |= SB_GMC_T_NE | SB_GMC_T_SEL3;
261 #if 0
262 msr |= SB_GMC_SBI | SB_GMC_SBY; /* for AES, if necessary */
263 #endif
264 wrmsr(SB_GLD_MSR_CTRL, msr);
265
266 /* Install a periodic collector for the "true" (AMD's word) RNG */
267 timeout_set(&sc->sc_to, glxsb_rnd, sc);
268 glxsb_rnd(sc);
269 printf(": RNG");
270
271 #ifdef CRYPTO
272 /* We don't have an interrupt handler, so disable completion INTs */
273 intr = SB_AI_DISABLE_AES_A | SB_AI_DISABLE_AES_B |
274 SB_AI_DISABLE_EEPROM | SB_AI_AES_A_COMPLETE |
275 SB_AI_AES_B_COMPLETE | SB_AI_EEPROM_COMPLETE;
276 bus_space_write_4(sc->sc_iot, sc->sc_ioh, SB_AES_INT, intr);
277
278 sc->sc_dmat = pa->pa_dmat;
279
280 if (glxsb_crypto_setup(sc))
281 printf(" AES");
282 #endif
283
284 printf("\n");
285 }
286
287 int
glxsb_activate(struct device * self,int act)288 glxsb_activate(struct device *self, int act)
289 {
290 struct glxsb_softc *sc = (struct glxsb_softc *)self;
291
292 switch (act) {
293 case DVACT_QUIESCE:
294 /* XXX should wait for current crypto op to finish */
295 break;
296 case DVACT_SUSPEND:
297 sc->save_gld_msr = rdmsr(SB_GLD_MSR_CTRL);
298 break;
299 case DVACT_RESUME:
300 wrmsr(SB_GLD_MSR_CTRL, sc->save_gld_msr);
301 break;
302 }
303 return (0);
304 }
305
306 void
glxsb_rnd(void * v)307 glxsb_rnd(void *v)
308 {
309 struct glxsb_softc *sc = v;
310 uint32_t status, value;
311
312 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, SB_RANDOM_NUM_STATUS);
313 if (status & SB_RNS_TRNG_VALID) {
314 value = bus_space_read_4(sc->sc_iot, sc->sc_ioh, SB_RANDOM_NUM);
315 enqueue_randomness(value);
316 }
317
318 timeout_add_msec(&sc->sc_to, 10);
319 }
320
321 #ifdef CRYPTO
322 int
glxsb_crypto_setup(struct glxsb_softc * sc)323 glxsb_crypto_setup(struct glxsb_softc *sc)
324 {
325 int algs[CRYPTO_ALGORITHM_MAX + 1];
326
327 /* Allocate a contiguous DMA-able buffer to work in */
328 if (glxsb_dma_alloc(sc, GLXSB_MAX_AES_LEN * 2, &sc->sc_dma) != 0)
329 return 0;
330
331 bzero(algs, sizeof(algs));
332 algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
333 algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
334 algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
335 algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
336 algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
337 algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
338 algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
339
340 sc->sc_cid = crypto_get_driverid(0);
341 if (sc->sc_cid < 0)
342 return 0;
343
344 crypto_register(sc->sc_cid, algs, glxsb_crypto_newsession,
345 glxsb_crypto_freesession, glxsb_crypto_process);
346
347 sc->sc_nsessions = 0;
348
349 glxsb_sc = sc;
350
351 return 1;
352 }
353
354 int
glxsb_crypto_newsession(uint32_t * sidp,struct cryptoini * cri)355 glxsb_crypto_newsession(uint32_t *sidp, struct cryptoini *cri)
356 {
357 struct glxsb_softc *sc = glxsb_sc;
358 struct glxsb_session *ses = NULL;
359 const struct auth_hash *axf;
360 const struct enc_xform *txf;
361 struct cryptoini *c;
362 struct swcr_data *swd;
363 int sesn, i;
364
365 if (sc == NULL || sidp == NULL || cri == NULL)
366 return (EINVAL);
367
368 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
369 if (sc->sc_sessions[sesn].ses_used == 0) {
370 ses = &sc->sc_sessions[sesn];
371 break;
372 }
373 }
374
375 if (ses == NULL) {
376 sesn = sc->sc_nsessions;
377 ses = mallocarray(sesn + 1, sizeof(*ses), M_DEVBUF,
378 M_NOWAIT);
379 if (ses == NULL)
380 return (ENOMEM);
381 if (sesn != 0) {
382 bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
383 explicit_bzero(sc->sc_sessions, sesn * sizeof(*ses));
384 free(sc->sc_sessions, M_DEVBUF, sesn * sizeof(*ses));
385 }
386 sc->sc_sessions = ses;
387 ses = &sc->sc_sessions[sesn];
388 sc->sc_nsessions++;
389 }
390
391 bzero(ses, sizeof(*ses));
392 ses->ses_used = 1;
393
394 for (c = cri; c != NULL; c = c->cri_next) {
395 switch (c->cri_alg) {
396 case CRYPTO_AES_CBC:
397
398 if (c->cri_klen != 128) {
399 swd = malloc(sizeof(struct swcr_data),
400 M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
401 if (swd == NULL) {
402 glxsb_crypto_freesession(sesn);
403 return (ENOMEM);
404 }
405 ses->ses_swd_enc = swd;
406 txf = &enc_xform_aes;
407 if (txf->ctxsize > 0) {
408 swd->sw_kschedule =
409 malloc(txf->ctxsize,
410 M_CRYPTO_DATA,
411 M_NOWAIT|M_ZERO);
412 if (swd->sw_kschedule == NULL) {
413 glxsb_crypto_freesession(sesn);
414 return (EINVAL);
415 }
416 }
417 if (txf->setkey(swd->sw_kschedule, c->cri_key,
418 c->cri_klen / 8) < 0) {
419 glxsb_crypto_freesession(sesn);
420 return (EINVAL);
421 }
422 swd->sw_exf = txf;
423 break;
424 }
425
426 ses->ses_klen = c->cri_klen;
427
428 /* Copy the key (Geode LX wants the primary key only) */
429 bcopy(c->cri_key, ses->ses_key, sizeof(ses->ses_key));
430 break;
431
432 case CRYPTO_MD5_HMAC:
433 axf = &auth_hash_hmac_md5_96;
434 goto authcommon;
435 case CRYPTO_SHA1_HMAC:
436 axf = &auth_hash_hmac_sha1_96;
437 goto authcommon;
438 case CRYPTO_RIPEMD160_HMAC:
439 axf = &auth_hash_hmac_ripemd_160_96;
440 goto authcommon;
441 case CRYPTO_SHA2_256_HMAC:
442 axf = &auth_hash_hmac_sha2_256_128;
443 goto authcommon;
444 case CRYPTO_SHA2_384_HMAC:
445 axf = &auth_hash_hmac_sha2_384_192;
446 goto authcommon;
447 case CRYPTO_SHA2_512_HMAC:
448 axf = &auth_hash_hmac_sha2_512_256;
449 authcommon:
450 swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
451 M_NOWAIT|M_ZERO);
452 if (swd == NULL) {
453 glxsb_crypto_freesession(sesn);
454 return (ENOMEM);
455 }
456 ses->ses_swd_auth = swd;
457
458 swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
459 M_NOWAIT);
460 if (swd->sw_ictx == NULL) {
461 glxsb_crypto_freesession(sesn);
462 return (ENOMEM);
463 }
464
465 swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
466 M_NOWAIT);
467 if (swd->sw_octx == NULL) {
468 glxsb_crypto_freesession(sesn);
469 return (ENOMEM);
470 }
471
472 for (i = 0; i < c->cri_klen / 8; i++)
473 c->cri_key[i] ^= HMAC_IPAD_VAL;
474
475 axf->Init(swd->sw_ictx);
476 axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
477 axf->Update(swd->sw_ictx, hmac_ipad_buffer,
478 axf->blocksize - (c->cri_klen / 8));
479
480 for (i = 0; i < c->cri_klen / 8; i++)
481 c->cri_key[i] ^= (HMAC_IPAD_VAL ^
482 HMAC_OPAD_VAL);
483
484 axf->Init(swd->sw_octx);
485 axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
486 axf->Update(swd->sw_octx, hmac_opad_buffer,
487 axf->blocksize - (c->cri_klen / 8));
488
489 for (i = 0; i < c->cri_klen / 8; i++)
490 c->cri_key[i] ^= HMAC_OPAD_VAL;
491
492 swd->sw_axf = axf;
493 swd->sw_alg = c->cri_alg;
494
495 break;
496 default:
497 glxsb_crypto_freesession(sesn);
498 return (EINVAL);
499 }
500 }
501
502 *sidp = GLXSB_SID(0, sesn);
503 return (0);
504 }
505
506 int
glxsb_crypto_freesession(uint64_t tid)507 glxsb_crypto_freesession(uint64_t tid)
508 {
509 struct glxsb_softc *sc = glxsb_sc;
510 struct swcr_data *swd;
511 const struct auth_hash *axf;
512 const struct enc_xform *txf;
513 int sesn;
514 uint32_t sid = ((uint32_t)tid) & 0xffffffff;
515
516 if (sc == NULL)
517 return (EINVAL);
518 sesn = GLXSB_SESSION(sid);
519 if (sesn >= sc->sc_nsessions)
520 return (EINVAL);
521 if ((swd = sc->sc_sessions[sesn].ses_swd_enc)) {
522 txf = swd->sw_exf;
523
524 if (swd->sw_kschedule) {
525 explicit_bzero(swd->sw_kschedule, txf->ctxsize);
526 free(swd->sw_kschedule, M_CRYPTO_DATA, txf->ctxsize);
527 }
528 free(swd, M_CRYPTO_DATA, sizeof(*swd));
529 }
530 if ((swd = sc->sc_sessions[sesn].ses_swd_auth)) {
531 axf = swd->sw_axf;
532
533 if (swd->sw_ictx) {
534 explicit_bzero(swd->sw_ictx, axf->ctxsize);
535 free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize);
536 }
537 if (swd->sw_octx) {
538 explicit_bzero(swd->sw_octx, axf->ctxsize);
539 free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize);
540 }
541 free(swd, M_CRYPTO_DATA, sizeof(*swd));
542 }
543 explicit_bzero(&sc->sc_sessions[sesn], sizeof(sc->sc_sessions[sesn]));
544 return (0);
545 }
546
547 /*
548 * Must be called at splnet() or higher
549 */
550 static __inline void
glxsb_aes(struct glxsb_softc * sc,uint32_t control,uint32_t psrc,uint32_t pdst,void * key,int len,void * iv)551 glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
552 uint32_t pdst, void *key, int len, void *iv)
553 {
554 uint32_t status;
555 int i;
556
557 if (len & 0xF) {
558 printf("%s: len must be a multiple of 16 (not %d)\n",
559 sc->sc_dev.dv_xname, len);
560 return;
561 }
562
563 /* Set the source */
564 bus_space_write_4(sc->sc_iot, sc->sc_ioh, SB_SOURCE_A, psrc);
565
566 /* Set the destination address */
567 bus_space_write_4(sc->sc_iot, sc->sc_ioh, SB_DEST_A, pdst);
568
569 /* Set the data length */
570 bus_space_write_4(sc->sc_iot, sc->sc_ioh, SB_LENGTH_A, len);
571
572 /* Set the IV */
573 if (iv != NULL) {
574 bus_space_write_region_4(sc->sc_iot, sc->sc_ioh,
575 SB_CBC_IV, iv, 4);
576 control |= SB_CTL_CBC;
577 }
578
579 /* Set the key */
580 bus_space_write_region_4(sc->sc_iot, sc->sc_ioh, SB_WKEY, key, 4);
581
582 /* Ask the security block to do it */
583 bus_space_write_4(sc->sc_iot, sc->sc_ioh, SB_CTL_A,
584 control | SB_CTL_WK | SB_CTL_DC | SB_CTL_SC | SB_CTL_ST);
585
586 /*
587 * Now wait until it is done.
588 *
589 * We do a busy wait. Obviously the number of iterations of
590 * the loop required to perform the AES operation depends upon
591 * the number of bytes to process.
592 *
593 * On a 500 MHz Geode LX we see
594 *
595 * length (bytes) typical max iterations
596 * 16 12
597 * 64 22
598 * 256 59
599 * 1024 212
600 * 8192 1,537
601 *
602 * Since we have a maximum size of operation defined in
603 * GLXSB_MAX_AES_LEN, we use this constant to decide how long
604 * to wait. Allow an order of magnitude longer than it should
605 * really take, just in case.
606 */
607 for (i = 0; i < GLXSB_MAX_AES_LEN * 10; i++) {
608 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, SB_CTL_A);
609
610 if ((status & SB_CTL_ST) == 0) /* Done */
611 return;
612 }
613
614 printf("%s: operation failed to complete\n", sc->sc_dev.dv_xname);
615 }
616
617 static int
glxsb_crypto_swauth(struct cryptop * crp,struct cryptodesc * crd,struct swcr_data * sw,caddr_t buf)618 glxsb_crypto_swauth(struct cryptop *crp, struct cryptodesc *crd,
619 struct swcr_data *sw, caddr_t buf)
620 {
621 int type;
622
623 if (crp->crp_flags & CRYPTO_F_IMBUF)
624 type = CRYPTO_BUF_MBUF;
625 else
626 type = CRYPTO_BUF_IOV;
627
628 return (swcr_authcompute(crp, crd, sw, buf, type));
629 }
630
631 static int
glxsb_crypto_swenc(struct cryptop * crp,struct cryptodesc * crd,struct swcr_data * sw,caddr_t buf)632 glxsb_crypto_swenc(struct cryptop *crp, struct cryptodesc *crd,
633 struct swcr_data *sw, caddr_t buf)
634 {
635 int type;
636
637 if (crp->crp_flags & CRYPTO_F_IMBUF)
638 type = CRYPTO_BUF_MBUF;
639 else
640 type = CRYPTO_BUF_IOV;
641
642 return (swcr_encdec(crd, sw, buf, type));
643 }
644
645 static int
glxsb_crypto_encdec(struct cryptop * crp,struct cryptodesc * crd,struct glxsb_session * ses,struct glxsb_softc * sc,caddr_t buf)646 glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
647 struct glxsb_session *ses, struct glxsb_softc *sc, caddr_t buf)
648 {
649 char *op_src, *op_dst;
650 uint32_t op_psrc, op_pdst;
651 uint8_t op_iv[SB_AES_BLOCK_SIZE];
652 int err = 0;
653 int len, tlen, xlen;
654 int offset;
655 uint32_t control;
656
657 if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0) {
658 err = EINVAL;
659 goto out;
660 }
661
662 /* How much of our buffer will we need to use? */
663 xlen = crd->crd_len > GLXSB_MAX_AES_LEN ?
664 GLXSB_MAX_AES_LEN : crd->crd_len;
665
666 /*
667 * XXX Check if we can have input == output on Geode LX.
668 * XXX In the meantime, use two separate (adjacent) buffers.
669 */
670 op_src = sc->sc_dma.dma_vaddr;
671 op_dst = sc->sc_dma.dma_vaddr + xlen;
672
673 op_psrc = sc->sc_dma.dma_paddr;
674 op_pdst = sc->sc_dma.dma_paddr + xlen;
675
676 if (crd->crd_flags & CRD_F_ENCRYPT) {
677 control = SB_CTL_ENC;
678 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
679 bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
680 else
681 arc4random_buf(op_iv, sizeof(op_iv));
682
683 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
684 if (crp->crp_flags & CRYPTO_F_IMBUF)
685 err = m_copyback((struct mbuf *)crp->crp_buf,
686 crd->crd_inject, sizeof(op_iv), op_iv,
687 M_NOWAIT);
688 else if (crp->crp_flags & CRYPTO_F_IOV)
689 cuio_copyback((struct uio *)crp->crp_buf,
690 crd->crd_inject, sizeof(op_iv), op_iv);
691 else
692 bcopy(op_iv,
693 crp->crp_buf + crd->crd_inject, sizeof(op_iv));
694 if (err)
695 goto out;
696 }
697 } else {
698 control = SB_CTL_DEC;
699 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
700 bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
701 else {
702 if (crp->crp_flags & CRYPTO_F_IMBUF)
703 m_copydata((struct mbuf *)crp->crp_buf,
704 crd->crd_inject, sizeof(op_iv), op_iv);
705 else if (crp->crp_flags & CRYPTO_F_IOV)
706 cuio_copydata((struct uio *)crp->crp_buf,
707 crd->crd_inject, sizeof(op_iv), op_iv);
708 else
709 bcopy(crp->crp_buf + crd->crd_inject,
710 op_iv, sizeof(op_iv));
711 }
712 }
713
714 offset = 0;
715 tlen = crd->crd_len;
716
717 /* Process the data in GLXSB_MAX_AES_LEN chunks */
718 while (tlen > 0) {
719 len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen;
720
721 if (crp->crp_flags & CRYPTO_F_IMBUF)
722 m_copydata((struct mbuf *)crp->crp_buf,
723 crd->crd_skip + offset, len, op_src);
724 else if (crp->crp_flags & CRYPTO_F_IOV)
725 cuio_copydata((struct uio *)crp->crp_buf,
726 crd->crd_skip + offset, len, op_src);
727 else
728 bcopy(crp->crp_buf + crd->crd_skip + offset, op_src,
729 len);
730
731 glxsb_dma_pre_op(sc, &sc->sc_dma);
732
733 glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key,
734 len, op_iv);
735
736 glxsb_dma_post_op(sc, &sc->sc_dma);
737
738 if (crp->crp_flags & CRYPTO_F_IMBUF)
739 err = m_copyback((struct mbuf *)crp->crp_buf,
740 crd->crd_skip + offset, len, op_dst, M_NOWAIT);
741 else if (crp->crp_flags & CRYPTO_F_IOV)
742 cuio_copyback((struct uio *)crp->crp_buf,
743 crd->crd_skip + offset, len, op_dst);
744 else
745 bcopy(op_dst, crp->crp_buf + crd->crd_skip + offset,
746 len);
747 if (err)
748 break;
749
750 offset += len;
751 tlen -= len;
752
753 if (tlen > 0) {
754 /* Copy out last block for use as next iteration */
755 if (crd->crd_flags & CRD_F_ENCRYPT)
756 bcopy(op_dst + len - sizeof(op_iv), op_iv,
757 sizeof(op_iv));
758 else
759 bcopy(op_src + len - sizeof(op_iv), op_iv,
760 sizeof(op_iv));
761 }
762 }
763
764 /* All AES processing has now been done. */
765 explicit_bzero(sc->sc_dma.dma_vaddr, xlen * 2);
766
767 out:
768 return (err);
769 }
770
771 int
glxsb_crypto_process(struct cryptop * crp)772 glxsb_crypto_process(struct cryptop *crp)
773 {
774 struct glxsb_softc *sc = glxsb_sc;
775 struct glxsb_session *ses;
776 struct cryptodesc *crd;
777 int sesn,err = 0;
778 int s, i;
779
780 s = splnet();
781
782 KASSERT(crp->crp_ndesc >= 1);
783
784 sesn = GLXSB_SESSION(crp->crp_sid);
785 if (sesn >= sc->sc_nsessions) {
786 err = EINVAL;
787 goto out;
788 }
789 ses = &sc->sc_sessions[sesn];
790 if (ses->ses_used == 0) {
791 err = EINVAL;
792 goto out;
793 }
794
795 for (i = 0; i < crp->crp_ndesc; i++) {
796 crd = &crp->crp_desc[i];
797 switch (crd->crd_alg) {
798 case CRYPTO_AES_CBC:
799 if (ses->ses_swd_enc) {
800 if ((err = glxsb_crypto_swenc(crp, crd, ses->ses_swd_enc,
801 crp->crp_buf)) != 0)
802 goto out;
803 } else if ((err = glxsb_crypto_encdec(crp, crd, ses, sc,
804 crp->crp_buf)) != 0)
805 goto out;
806 break;
807
808 case CRYPTO_MD5_HMAC:
809 case CRYPTO_SHA1_HMAC:
810 case CRYPTO_RIPEMD160_HMAC:
811 case CRYPTO_SHA2_256_HMAC:
812 case CRYPTO_SHA2_384_HMAC:
813 case CRYPTO_SHA2_512_HMAC:
814 if ((err = glxsb_crypto_swauth(crp, crd, ses->ses_swd_auth,
815 crp->crp_buf)) != 0)
816 goto out;
817 break;
818
819 default:
820 err = EINVAL;
821 goto out;
822 }
823 }
824
825 out:
826 splx(s);
827 return (err);
828 }
829
830 int
glxsb_dma_alloc(struct glxsb_softc * sc,int size,struct glxsb_dma_map * dma)831 glxsb_dma_alloc(struct glxsb_softc *sc, int size, struct glxsb_dma_map *dma)
832 {
833 int rc;
834
835 dma->dma_nsegs = 1;
836 dma->dma_size = size;
837
838 rc = bus_dmamap_create(sc->sc_dmat, size, dma->dma_nsegs, size,
839 0, BUS_DMA_NOWAIT, &dma->dma_map);
840 if (rc != 0) {
841 printf("%s: couldn't create DMA map for %d bytes (%d)\n",
842 sc->sc_dev.dv_xname, size, rc);
843
844 goto fail0;
845 }
846
847 rc = bus_dmamem_alloc(sc->sc_dmat, size, SB_AES_ALIGN, 0,
848 &dma->dma_seg, dma->dma_nsegs, &dma->dma_nsegs, BUS_DMA_NOWAIT);
849 if (rc != 0) {
850 printf("%s: couldn't allocate DMA memory of %d bytes (%d)\n",
851 sc->sc_dev.dv_xname, size, rc);
852
853 goto fail1;
854 }
855
856 rc = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, 1, size,
857 &dma->dma_vaddr, BUS_DMA_NOWAIT);
858 if (rc != 0) {
859 printf("%s: couldn't map DMA memory for %d bytes (%d)\n",
860 sc->sc_dev.dv_xname, size, rc);
861
862 goto fail2;
863 }
864
865 rc = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
866 size, NULL, BUS_DMA_NOWAIT);
867 if (rc != 0) {
868 printf("%s: couldn't load DMA memory for %d bytes (%d)\n",
869 sc->sc_dev.dv_xname, size, rc);
870
871 goto fail3;
872 }
873
874 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr;
875
876 return 0;
877
878 fail3:
879 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
880 fail2:
881 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nsegs);
882 fail1:
883 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
884 fail0:
885 return rc;
886 }
887
888 void
glxsb_dma_pre_op(struct glxsb_softc * sc,struct glxsb_dma_map * dma)889 glxsb_dma_pre_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
890 {
891 bus_dmamap_sync(sc->sc_dmat, dma->dma_map, 0, dma->dma_size,
892 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
893 }
894
895 void
glxsb_dma_post_op(struct glxsb_softc * sc,struct glxsb_dma_map * dma)896 glxsb_dma_post_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
897 {
898 bus_dmamap_sync(sc->sc_dmat, dma->dma_map, 0, dma->dma_size,
899 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
900 }
901
902 void
glxsb_dma_free(struct glxsb_softc * sc,struct glxsb_dma_map * dma)903 glxsb_dma_free(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
904 {
905 bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
906 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
907 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nsegs);
908 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
909 }
910
911 #endif /* CRYPTO */
912