1 /* $NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * sun8i_crypto -- Allwinner Crypto Engine driver
34 *
35 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64
36 * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at
37 * the moment, but in principle it could be wired up with opencrypto(9)
38 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1,
39 * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support
40 * AES keys in SRAM (for some kind of HDMI HDCP stuff?).
41 *
42 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf
43 */
44
45 #include <sys/cdefs.h>
46 __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $");
47
48 #include <sys/types.h>
49 #include <sys/param.h>
50 #include <sys/atomic.h>
51 #include <sys/bus.h>
52 #include <sys/callout.h>
53 #include <sys/conf.h>
54 #include <sys/cprng.h>
55 #include <sys/device.h>
56 #include <sys/kernel.h>
57 #include <sys/kmem.h>
58 #include <sys/mbuf.h>
59 #include <sys/mutex.h>
60 #include <sys/rndsource.h>
61 #include <sys/sdt.h>
62 #include <sys/sysctl.h>
63 #include <sys/workqueue.h>
64
65 #include <dev/fdt/fdtvar.h>
66
67 #include <opencrypto/cryptodev.h>
68
69 #include <arm/sunxi/sun8i_crypto.h>
70
71 #define SUN8I_CRYPTO_TIMEOUT hz
72 #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */
73 #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE
74
75 struct sun8i_crypto_config {
76 u_int mod_rate; /* module clock rate */
77 };
78
79 /*
80 * The module clock is set to 50 MHz on H3, 300 MHz otherwise.
81 * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c:
82 * Module clock is lower on H3 than other SoC due to some DMA
83 * timeout occurring with high value.
84 */
85 static const struct sun8i_crypto_config sun50i_a64_crypto_config = {
86 .mod_rate = 300*1000*1000,
87 };
88
89 static const struct sun8i_crypto_config sun50i_h5_crypto_config = {
90 .mod_rate = 300*1000*1000,
91 };
92
93 static const struct sun8i_crypto_config sun8i_h3_crypto_config = {
94 .mod_rate = 50*1000*1000,
95 };
96
97 struct sun8i_crypto_task;
98
99 struct sun8i_crypto_buf {
100 bus_dma_segment_t cb_seg[1];
101 int cb_nsegs;
102 void *cb_kva;
103 };
104
105 struct sun8i_crypto_softc {
106 device_t sc_dev;
107 bus_space_tag_t sc_bst;
108 bus_space_handle_t sc_bsh;
109 bus_dma_tag_t sc_dmat;
110 struct pool_cache *sc_taskpool;
111
112 const struct sun8i_crypto_config *sc_cfg;
113
114 struct workqueue *sc_wq;
115 void *sc_ih;
116 bool sc_polling;
117
118 kmutex_t sc_lock;
119 struct sun8i_crypto_chan {
120 struct sun8i_crypto_task *cc_task;
121 unsigned cc_starttime;
122 } sc_chan[SUN8I_CRYPTO_NCHAN];
123 struct callout sc_timeout;
124
125 kmutex_t sc_intr_lock;
126 uint32_t sc_done;
127 uint32_t sc_esr;
128 struct work sc_work;
129 bool sc_work_pending;
130
131 struct sun8i_crypto_rng {
132 struct sun8i_crypto_buf cr_buf;
133 struct sun8i_crypto_task *cr_task;
134 struct krndsource cr_rndsource;
135 bool cr_pending;
136 } sc_rng;
137 struct sun8i_crypto_selftest {
138 struct sun8i_crypto_buf cs_in;
139 struct sun8i_crypto_buf cs_key;
140 struct sun8i_crypto_buf cs_out;
141 struct sun8i_crypto_task *cs_task;
142 bool cs_pending;
143 bool cs_passed;
144 } sc_selftest;
145 struct sun8i_crypto_sysctl {
146 struct sysctllog *cy_log;
147 const struct sysctlnode *cy_root_node;
148 const struct sysctlnode *cy_trng_node;
149 } sc_sysctl;
150 struct sun8i_crypto_opencrypto {
151 uint32_t co_driverid;
152 } sc_opencrypto;
153 };
154
155 struct sun8i_crypto_task {
156 struct sun8i_crypto_buf ct_descbuf;
157 struct sun8i_crypto_taskdesc *ct_desc;
158 struct sun8i_crypto_buf ct_ivbuf;
159 void *ct_iv;
160 struct sun8i_crypto_buf ct_ctrbuf;
161 void *ct_ctr;
162 bus_dmamap_t ct_descmap;
163 bus_dmamap_t ct_keymap;
164 bus_dmamap_t ct_ivmap; /* IV input */
165 bus_dmamap_t ct_ctrmap; /* updated IV output */
166 bus_dmamap_t ct_srcmap;
167 bus_dmamap_t ct_dstmap;
168 uint32_t ct_nbytes;
169 int ct_flags;
170 #define TASK_KEY __BIT(0)
171 #define TASK_IV __BIT(1)
172 #define TASK_CTR __BIT(2)
173 #define TASK_SRC __BIT(3)
174 #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */
175 void (*ct_callback)(struct sun8i_crypto_softc *,
176 struct sun8i_crypto_task *, void *, int);
177 void *ct_cookie;
178 };
179
180 #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE
181 #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE
182
183 CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN);
184 CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN);
185
186 /*
187 * Forward declarations
188 */
189
190 static int sun8i_crypto_match(device_t, cfdata_t, void *);
191 static void sun8i_crypto_attach(device_t, device_t, void *);
192
193 static int sun8i_crypto_task_ctor(void *, void *, int);
194 static void sun8i_crypto_task_dtor(void *, void *);
195 static struct sun8i_crypto_task *
196 sun8i_crypto_task_get(struct sun8i_crypto_softc *,
197 void (*)(struct sun8i_crypto_softc *,
198 struct sun8i_crypto_task *, void *, int),
199 void *, int);
200 static void sun8i_crypto_task_put(struct sun8i_crypto_softc *,
201 struct sun8i_crypto_task *);
202
203 static int sun8i_crypto_task_load(struct sun8i_crypto_softc *,
204 struct sun8i_crypto_task *, uint32_t,
205 uint32_t, uint32_t, uint32_t);
206 static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *,
207 struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t);
208
209 static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *,
210 struct sun8i_crypto_task *, uint32_t);
211 static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *,
212 struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t);
213
214 static int sun8i_crypto_submit(struct sun8i_crypto_softc *,
215 struct sun8i_crypto_task *);
216
217 static void sun8i_crypto_timeout(void *);
218 static int sun8i_crypto_intr(void *);
219 static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *);
220 static void sun8i_crypto_worker(struct work *, void *);
221
222 static bool sun8i_crypto_poll(struct sun8i_crypto_softc *, uint32_t *,
223 uint32_t *);
224 static bool sun8i_crypto_done(struct sun8i_crypto_softc *, uint32_t,
225 uint32_t, unsigned);
226 static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned,
227 int);
228
229 static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t,
230 struct sun8i_crypto_buf *, int);
231 static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t,
232 struct sun8i_crypto_buf *);
233
234 static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *);
235 static void sun8i_crypto_rng_get(size_t, void *);
236 static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *,
237 struct sun8i_crypto_task *, void *, int);
238
239 static bool sun8i_crypto_selftest(struct sun8i_crypto_softc *);
240 static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *,
241 struct sun8i_crypto_task *, void *, int);
242
243 static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *);
244 static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS);
245 static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *,
246 struct sun8i_crypto_task *, void *, int);
247
248 static void sun8i_crypto_register(struct sun8i_crypto_softc *);
249 static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t);
250 static int sun8i_crypto_newsession(void *, uint32_t *,
251 struct cryptoini *);
252 static void sun8i_crypto_freesession(void *, uint64_t);
253 static u_int sun8i_crypto_ivlen(const struct cryptodesc *);
254 static int sun8i_crypto_process(void *, struct cryptop *, int);
255 static void sun8i_crypto_callback(struct sun8i_crypto_softc *,
256 struct sun8i_crypto_task *, void *, int);
257
258 /*
259 * Probes
260 */
261
262 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read,
263 "bus_size_t"/*reg*/,
264 "uint32_t"/*value*/);
265 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write,
266 "bus_size_t"/*reg*/,
267 "uint32_t"/*write*/);
268
269 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success,
270 "struct sun8i_crypto_task *"/*task*/);
271 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure,
272 "int"/*error*/);
273 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor,
274 "struct sun8i_crypto_task *"/*task*/);
275 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get,
276 "struct sun8i_crypto_task *"/*task*/);
277 SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put,
278 "struct sun8i_crypto_task *"/*task*/);
279
280 SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load,
281 "struct sun8i_crypto_task *"/*task*/,
282 "uint32_t"/*tdqc*/,
283 "uint32_t"/*tdqs*/,
284 "uint32_t"/*tdqa*/,
285 "struct sun8i_crypto_taskdesc *"/*desc*/,
286 "int"/*error*/);
287 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned,
288 "struct sun8i_crypto_task *"/*task*/,
289 "bus_addr_t"/*ds_addr*/,
290 "bus_size_t"/*ds_len*/);
291 SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done,
292 "struct sun8i_crypto_task *"/*task*/,
293 "int"/*error*/);
294
295 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure,
296 "struct sun8i_crypto_softc *"/*sc*/,
297 "struct sun8i_crypto_task *"/*task*/,
298 "int"/*error*/);
299 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success,
300 "struct sun8i_crypto_softc *"/*sc*/,
301 "struct sun8i_crypto_task *"/*task*/,
302 "unsigned"/*chan*/);
303 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr,
304 "struct sun8i_crypto_softc *"/*sc*/,
305 "uint32_t"/*isr*/,
306 "uint32_t"/*esr*/);
307 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done,
308 "struct sun8i_crypto_softc *"/*sc*/,
309 "unsigned"/*chan*/,
310 "int"/*error*/);
311
312 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry,
313 "struct sun8i_crypto_softc *"/*sc*/,
314 "struct cryptop *"/*crp*/,
315 "int"/*hint*/);
316 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy,
317 "struct sun8i_crypto_softc *"/*sc*/,
318 "struct cryptop *"/*crp*/,
319 "int"/*hint*/);
320 SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued,
321 "struct sun8i_crypto_softc *"/*sc*/,
322 "struct cryptop *"/*crp*/,
323 "int"/*hint*/,
324 "struct sun8i_crypto_task *"/*task*/);
325 SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done,
326 "struct sun8i_crypto_softc *"/*sc*/,
327 "struct cryptop *"/*crp*/,
328 "int"/*error*/);
329
330 /*
331 * Register access
332 */
333
334 static uint32_t
sun8i_crypto_read(struct sun8i_crypto_softc * sc,bus_size_t reg)335 sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg)
336 {
337 uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
338
339 SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v);
340 return v;
341 }
342
343 static void
sun8i_crypto_write(struct sun8i_crypto_softc * sc,bus_size_t reg,uint32_t v)344 sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v)
345 {
346
347 SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v);
348 bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v);
349 }
350
351 /*
352 * Autoconf goo
353 */
354
355 CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc),
356 sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL);
357
358 static const struct device_compatible_entry compat_data[] = {
359 { .compat = "allwinner,sun50i-a64-crypto",
360 .data = &sun50i_a64_crypto_config },
361 { .compat = "allwinner,sun50i-h5-crypto",
362 .data = &sun50i_h5_crypto_config },
363 { .compat = "allwinner,sun8i-h3-crypto",
364 .data = &sun8i_h3_crypto_config },
365 DEVICE_COMPAT_EOL
366 };
367
368 static int
sun8i_crypto_match(device_t parent,cfdata_t cf,void * aux)369 sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux)
370 {
371 const struct fdt_attach_args *const faa = aux;
372
373 return of_compatible_match(faa->faa_phandle, compat_data);
374 }
375
376 static void
sun8i_crypto_attach(device_t parent,device_t self,void * aux)377 sun8i_crypto_attach(device_t parent, device_t self, void *aux)
378 {
379 struct sun8i_crypto_softc *const sc = device_private(self);
380 const struct fdt_attach_args *const faa = aux;
381 bus_addr_t addr;
382 bus_size_t size;
383 const int phandle = faa->faa_phandle;
384 char intrstr[128];
385 struct clk *clk;
386 struct fdtbus_reset *rst;
387 u_int mod_rate;
388
389 sc->sc_dev = self;
390 sc->sc_dmat = faa->faa_dmat;
391 sc->sc_bst = faa->faa_bst;
392 sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task),
393 0, 0, 0, "sun8icry", NULL, IPL_SOFTSERIAL,
394 &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc);
395 sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data;
396 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
397 mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_VM);
398 callout_init(&sc->sc_timeout, CALLOUT_MPSAFE);
399 callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc);
400 if (workqueue_create(&sc->sc_wq, device_xname(self),
401 &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) {
402 aprint_error(": couldn't create workqueue\n");
403 return;
404 }
405
406 /*
407 * Prime the pool with enough tasks that each channel can be
408 * busy with a task as we prepare another task for when it's
409 * done.
410 */
411 pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN);
412
413 /* Get and map device registers. */
414 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
415 aprint_error(": couldn't get registers\n");
416 return;
417 }
418 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
419 aprint_error(": couldn't map registers\n");
420 return;
421 }
422
423 /* Get an interrupt handle. */
424 if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
425 aprint_error(": failed to decode interrupt\n");
426 return;
427 }
428
429 /* Enable the bus clock. */
430 if (fdtbus_clock_enable(phandle, "bus", true) != 0) {
431 aprint_error(": couldn't enable bus clock\n");
432 return;
433 }
434
435 /* Get the module clock and set it. */
436 mod_rate = sc->sc_cfg->mod_rate;
437 if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) {
438 if (clk_enable(clk) != 0) {
439 aprint_error(": couldn't enable CE clock\n");
440 return;
441 }
442 if (clk_set_rate(clk, mod_rate) != 0) {
443 aprint_error(": couldn't set CE clock to %d MHz\n",
444 mod_rate / (1000 * 1000));
445 return;
446 }
447 }
448
449 /* Get a reset handle if we need and try to deassert it. */
450 if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) {
451 if (fdtbus_reset_deassert(rst) != 0) {
452 aprint_error(": couldn't de-assert reset\n");
453 return;
454 }
455 }
456
457 aprint_naive("\n");
458 aprint_normal(": Crypto Engine\n");
459 aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk));
460
461 /*
462 * Disable and clear interrupts. Start in polling mode for
463 * synchronous self-tests and the first RNG draw.
464 */
465 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0);
466 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0);
467 sc->sc_polling = true;
468
469 /* Establish an interrupt handler. */
470 sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM,
471 FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self));
472 if (sc->sc_ih == NULL) {
473 aprint_error_dev(self, "failed to establish interrupt on %s\n",
474 intrstr);
475 return;
476 }
477 aprint_normal_dev(self, "interrupting on %s\n", intrstr);
478
479 /* Perform self-tests. If they fail, stop here. */
480 if (!sun8i_crypto_selftest(sc))
481 return;
482
483 /*
484 * Set up the RNG. This will try to synchronously draw the
485 * first sample by polling, so do this before we establish
486 * the interrupt handler.
487 */
488 sun8i_crypto_rng_attach(sc);
489
490 /*
491 * Self-test has passed and first RNG draw has finished. Use
492 * interrupts, not polling, for all subsequent tasks. Set this
493 * atomically in case the interrupt handler has fired -- can't
494 * be from us because we've kept ICR set to 0 to mask all
495 * interrupts, but in case the interrupt vector is shared.
496 */
497 atomic_store_relaxed(&sc->sc_polling, false);
498
499 /* Attach the sysctl. */
500 sun8i_crypto_sysctl_attach(sc);
501
502 /* Register opencrypto handlers. */
503 sun8i_crypto_register(sc);
504 }
505
506 static int
sun8i_crypto_task_ctor(void * cookie,void * vtask,int pflags)507 sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags)
508 {
509 struct sun8i_crypto_softc *sc = cookie;
510 struct sun8i_crypto_task *task = vtask;
511 int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT;
512 int error;
513
514 /* Create a DMA buffer for the task descriptor. */
515 error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc),
516 &task->ct_descbuf, dmaflags);
517 if (error)
518 goto fail0;
519 task->ct_desc = task->ct_descbuf.cb_kva;
520
521 /* Create DMA buffers for the IV and CTR. */
522 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES,
523 &task->ct_ivbuf, dmaflags);
524 if (error)
525 goto fail1;
526 task->ct_iv = task->ct_ivbuf.cb_kva;
527 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES,
528 &task->ct_ctrbuf, dmaflags);
529 if (error)
530 goto fail2;
531 task->ct_ctr = task->ct_ctrbuf.cb_kva;
532
533 /* Create a DMA map for the task descriptor and preload it. */
534 error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1,
535 sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap);
536 if (error)
537 goto fail3;
538 error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc,
539 sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK);
540 if (error)
541 goto fail4;
542
543 /* Create DMA maps for the key, IV, and CTR. */
544 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1,
545 SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap);
546 if (error)
547 goto fail5;
548 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1,
549 SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap);
550 if (error)
551 goto fail6;
552 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1,
553 SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap);
554 if (error)
555 goto fail7;
556
557 /* Create DMA maps for the src and dst scatter/gather vectors. */
558 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
559 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
560 &task->ct_srcmap);
561 if (error)
562 goto fail8;
563 error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
564 SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
565 &task->ct_dstmap);
566 if (error)
567 goto fail9;
568
569 /* Success! */
570 SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task);
571 return 0;
572
573 fail10: __unused
574 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
575 fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
576 fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
577 fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
578 fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
579 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
580 fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
581 fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
582 fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
583 fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
584 fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error);
585 return error;
586 }
587
588 static void
sun8i_crypto_task_dtor(void * cookie,void * vtask)589 sun8i_crypto_task_dtor(void *cookie, void *vtask)
590 {
591 struct sun8i_crypto_softc *sc = cookie;
592 struct sun8i_crypto_task *task = vtask;
593
594 SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task);
595
596 /* XXX Zero the bounce buffers if there are any. */
597
598 bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
599 bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
600 bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
601 bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
602 bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
603 bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
604 bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
605 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
606 sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
607 sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
608 }
609
610 /*
611 * sun8i_crypto_task_get(sc, callback, cookie, pflags)
612 *
613 * Allocate a task that will call callback(sc, task, cookie,
614 * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if
615 * PR_NOWAIT, may fail and return NULL. No further allocation is
616 * needed to submit the task if this succeeds (although task
617 * submission may still fail if all channels are busy).
618 */
619 static struct sun8i_crypto_task *
sun8i_crypto_task_get(struct sun8i_crypto_softc * sc,void (* callback)(struct sun8i_crypto_softc *,struct sun8i_crypto_task *,void *,int),void * cookie,int pflags)620 sun8i_crypto_task_get(struct sun8i_crypto_softc *sc,
621 void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *,
622 void *, int),
623 void *cookie, int pflags)
624 {
625 struct sun8i_crypto_task *task;
626
627 /* Allocate a task, or fail if we can't. */
628 task = pool_cache_get(sc->sc_taskpool, pflags);
629 if (task == NULL)
630 goto out;
631
632 /* Set up flags and the callback. */
633 task->ct_flags = 0;
634 task->ct_callback = callback;
635 task->ct_cookie = cookie;
636
637 out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task);
638 return task;
639 }
640
641 /*
642 * sun8i_crypto_task_invalid(sc, task, cookie, error)
643 *
644 * Callback for a task not currently in use, to detect errors.
645 */
646 static void
sun8i_crypto_task_invalid(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)647 sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc,
648 struct sun8i_crypto_task *task, void *cookie, int error)
649 {
650 void (*callback)(struct sun8i_crypto_softc *,
651 struct sun8i_crypto_task *, void *, int) = cookie;
652
653 panic("task for callback %p used after free", callback);
654 }
655
656 /*
657 * sun8i_crypto_task_put(sc, task)
658 *
659 * Free a task obtained with sun8i_crypto_task_get.
660 */
661 static void
sun8i_crypto_task_put(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task)662 sun8i_crypto_task_put(struct sun8i_crypto_softc *sc,
663 struct sun8i_crypto_task *task)
664 {
665
666 SDT_PROBE1(sdt, sun8i_crypto, task, put, task);
667
668 task->ct_cookie = task->ct_callback;
669 task->ct_callback = &sun8i_crypto_task_invalid;
670 pool_cache_put(sc->sc_taskpool, task);
671 }
672
673 /*
674 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa)
675 *
676 * Set up the task descriptor after the relevant DMA maps have
677 * been loaded for a transfer of nbytes. bus_dmamap_sync matches
678 * sun8i_crypto_chan_done. May fail if input is inadequately
679 * aligned.
680 *
681 * XXX Teach this to support task chains.
682 */
683 static int
sun8i_crypto_task_load(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes,uint32_t tdqc,uint32_t tdqs,uint32_t tdqa)684 sun8i_crypto_task_load(struct sun8i_crypto_softc *sc,
685 struct sun8i_crypto_task *task, uint32_t nbytes,
686 uint32_t tdqc, uint32_t tdqs, uint32_t tdqa)
687 {
688 struct sun8i_crypto_taskdesc *desc = task->ct_desc;
689 int error;
690
691 KASSERT(tdqs == 0 || tdqa == 0);
692 KASSERT(nbytes % 4 == 0);
693
694 memset(desc, 0, sizeof(*desc));
695
696 /* Always enable interrupt for the task. */
697 tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
698
699 desc->td_tdqc = htole32(tdqc);
700 desc->td_tdqs = htole32(tdqs);
701 desc->td_tdqa = htole32(tdqa);
702
703 if (task->ct_flags & TASK_KEY) {
704 bus_dmamap_t keymap = task->ct_keymap;
705 KASSERT(keymap->dm_nsegs == 1);
706 desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr);
707 bus_dmamap_sync(sc->sc_dmat, keymap, 0,
708 keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
709 }
710 if (task->ct_flags & TASK_IV) {
711 bus_dmamap_t ivmap = task->ct_ivmap;
712 KASSERT(ivmap->dm_nsegs == 1);
713 desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr);
714 bus_dmamap_sync(sc->sc_dmat, ivmap, 0,
715 ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
716 }
717 if (task->ct_flags & TASK_CTR) {
718 bus_dmamap_t ctrmap = task->ct_ctrmap;
719 KASSERT(ctrmap->dm_nsegs == 1);
720 desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr);
721 bus_dmamap_sync(sc->sc_dmat, ctrmap, 0,
722 ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD);
723 }
724
725 if (task->ct_flags & TASK_BYTES)
726 desc->td_datalen = htole32(nbytes);
727 else
728 desc->td_datalen = htole32(nbytes/4);
729
730 if (task->ct_flags & TASK_SRC) {
731 bus_dmamap_t srcmap = task->ct_srcmap;
732 KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize);
733 error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap,
734 nbytes);
735 if (error)
736 return error;
737 bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes,
738 BUS_DMASYNC_PREWRITE);
739 }
740
741 error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap,
742 nbytes);
743 if (error)
744 goto out;
745 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
746 BUS_DMASYNC_PREREAD);
747
748 task->ct_nbytes = nbytes;
749
750 /* Success! */
751 error = 0;
752
753 out: SDT_PROBE6(sdt, sun8i_crypto, task, load,
754 task, tdqc, tdqs, tdqa, desc, error);
755 return error;
756 }
757
758 /*
759 * sun8i_crypto_task_scatter(task, adrlen, map, nbytes)
760 *
761 * Set up a task's scatter/gather vector -- src or dst -- with the
762 * given DMA map for a transfer of nbytes. May fail if input is
763 * inadequately aligned.
764 */
765 static int
sun8i_crypto_task_scatter(struct sun8i_crypto_task * task,struct sun8i_crypto_adrlen * adrlen,bus_dmamap_t map,uint32_t nbytes __diagused)766 sun8i_crypto_task_scatter(struct sun8i_crypto_task *task,
767 struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
768 uint32_t nbytes __diagused)
769 {
770 uint32_t total __diagused = 0;
771 unsigned i;
772
773 /*
774 * Verify that the alignment is correct and initialize the
775 * scatter/gather vector.
776 */
777 KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS);
778 for (i = 0; i < map->dm_nsegs; i++) {
779 if ((map->dm_segs[i].ds_addr % 4) |
780 (map->dm_segs[i].ds_len % 4)) {
781 SDT_PROBE3(sdt, sun8i_crypto, task, misaligned,
782 task,
783 map->dm_segs[i].ds_addr,
784 map->dm_segs[i].ds_len);
785 return EINVAL;
786 }
787 KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX);
788 KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total);
789 adrlen[i].adr = htole32(map->dm_segs[i].ds_addr);
790 adrlen[i].len = htole32(map->dm_segs[i].ds_len/4);
791 total += map->dm_segs[i].ds_len;
792 }
793
794 /* Set the remainder to zero. */
795 for (; i < SUN8I_CRYPTO_MAXSEGS; i++) {
796 adrlen[i].adr = 0;
797 adrlen[i].len = 0;
798 }
799
800 /* Verify the total size matches the transfer length. */
801 KASSERT(total == nbytes);
802
803 /* Success! */
804 return 0;
805 }
806
807 /*
808 * sun8i_crypto_task_load_trng(task, nbytes)
809 *
810 * Set up the task descriptor for a transfer of nbytes from the
811 * TRNG.
812 */
813 static int
sun8i_crypto_task_load_trng(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes)814 sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc,
815 struct sun8i_crypto_task *task, uint32_t nbytes)
816 {
817 uint32_t tdqc = 0;
818
819 /* Caller must provide dst only. */
820 KASSERT((task->ct_flags & TASK_KEY) == 0);
821 KASSERT((task->ct_flags & TASK_IV) == 0);
822 KASSERT((task->ct_flags & TASK_CTR) == 0);
823 KASSERT((task->ct_flags & TASK_SRC) == 0);
824
825 /* Set up the task descriptor queue control words. */
826 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG,
827 SUN8I_CRYPTO_TDQC_METHOD);
828
829 /* Fill in the descriptor. */
830 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0);
831 }
832
833 static int
sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes,uint32_t keysize,uint32_t dir)834 sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc,
835 struct sun8i_crypto_task *task,
836 uint32_t nbytes, uint32_t keysize, uint32_t dir)
837 {
838 uint32_t tdqc = 0, tdqs = 0;
839
840 /* Caller must provide key, src, and dst only. */
841 KASSERT(task->ct_flags & TASK_KEY);
842 KASSERT((task->ct_flags & TASK_IV) == 0);
843 KASSERT((task->ct_flags & TASK_CTR) == 0);
844 KASSERT(task->ct_flags & TASK_SRC);
845
846 /* Set up the task descriptor queue control word. */
847 tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES,
848 SUN8I_CRYPTO_TDQC_METHOD);
849 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
850
851 #ifdef DIAGNOSTIC
852 switch (keysize) {
853 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128:
854 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16);
855 break;
856 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192:
857 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24);
858 break;
859 case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256:
860 KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32);
861 break;
862 }
863 #endif
864
865 /* Set up the symmetric control word. */
866 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
867 SUN8I_CRYPTO_TDQS_SKEY_SELECT);
868 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB,
869 SUN8I_CRYPTO_TDQS_OP_MODE);
870 tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
871
872 /* Fill in the descriptor. */
873 return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0);
874 }
875
876 /*
877 * sun8i_crypto_submit(sc, task)
878 *
879 * Submit a task to the crypto engine after it has been loaded
880 * with sun8i_crypto_task_load. On success, guarantees to
881 * eventually call the task's callback.
882 */
883 static int
sun8i_crypto_submit(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task)884 sun8i_crypto_submit(struct sun8i_crypto_softc *sc,
885 struct sun8i_crypto_task *task)
886 {
887 unsigned i, retries = 0;
888 uint32_t icr;
889 int error = 0;
890
891 /* One at a time at the device registers, please. */
892 mutex_enter(&sc->sc_lock);
893
894 /* Find a channel. */
895 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
896 if (sc->sc_chan[i].cc_task == NULL)
897 break;
898 }
899 if (i == SUN8I_CRYPTO_NCHAN) {
900 device_printf(sc->sc_dev, "no free channels\n");
901 error = ERESTART;
902 goto out;
903 }
904
905 /*
906 * Set the channel id. Caller is responsible for setting up
907 * all other parts of the descriptor.
908 */
909 task->ct_desc->td_cid = htole32(i);
910
911 /*
912 * Prepare to send the descriptor to the device by DMA.
913 * Matches POSTWRITE in sun8i_crypto_chan_done.
914 */
915 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
916 sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE);
917
918 /* Confirm we're ready to go. */
919 if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) {
920 device_printf(sc->sc_dev, "TLR not clear\n");
921 error = EIO;
922 goto out;
923 }
924
925 /*
926 * Enable interrupts for this channel, unless we're still
927 * polling.
928 */
929 if (!sc->sc_polling) {
930 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
931 icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
932 SUN8I_CRYPTO_ICR_INTR_EN);
933 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
934 }
935
936 /* Set the task descriptor queue address. */
937 sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ,
938 task->ct_descmap->dm_segs[0].ds_addr);
939
940 /* Notify the engine to load it, and wait for acknowledgement. */
941 sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD);
942 while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD)
943 {
944 /*
945 * XXX Timeout pulled from arse. Is it even important
946 * to wait here?
947 */
948 if (++retries == 1000) {
949 device_printf(sc->sc_dev, "TLR didn't clear: %08x\n",
950 sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR));
951 /*
952 * Hope it clears eventually; if not, we'll
953 * time out.
954 */
955 break;
956 }
957 DELAY(1);
958 }
959
960 /*
961 * Loaded up and ready to go. Start a timer ticking if it's
962 * not already and we're not polling.
963 */
964 sc->sc_chan[i].cc_task = task;
965 sc->sc_chan[i].cc_starttime = getticks();
966 if (!sc->sc_polling && !callout_pending(&sc->sc_timeout))
967 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
968
969 out: /* Done! */
970 if (error)
971 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure,
972 sc, task, error);
973 else
974 SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success,
975 sc, task, i);
976 mutex_exit(&sc->sc_lock);
977 return error;
978 }
979
980 /*
981 * sun8i_crypto_timeout(cookie)
982 *
983 * Timeout handler. Schedules work in a thread to cancel all
984 * pending tasks that were started long enough ago we're bored of
985 * waiting for them.
986 */
987 static void
sun8i_crypto_timeout(void * cookie)988 sun8i_crypto_timeout(void *cookie)
989 {
990 struct sun8i_crypto_softc *sc = cookie;
991
992 mutex_enter(&sc->sc_intr_lock);
993 sun8i_crypto_schedule_worker(sc);
994 mutex_exit(&sc->sc_intr_lock);
995 }
996
997 /*
998 * sun8i_crypto_intr(cookie)
999 *
1000 * Device interrupt handler. Find what channels have completed,
1001 * whether with success or with failure, and schedule work in
1002 * thread context to invoke the appropriate callbacks.
1003 */
1004 static int
sun8i_crypto_intr(void * cookie)1005 sun8i_crypto_intr(void *cookie)
1006 {
1007 struct sun8i_crypto_softc *sc = cookie;
1008 uint32_t done, esr;
1009
1010 if (atomic_load_relaxed(&sc->sc_polling) ||
1011 !sun8i_crypto_poll(sc, &done, &esr))
1012 return 0; /* not ours */
1013
1014 mutex_enter(&sc->sc_intr_lock);
1015 sun8i_crypto_schedule_worker(sc);
1016 sc->sc_done |= done;
1017 sc->sc_esr |= esr;
1018 mutex_exit(&sc->sc_intr_lock);
1019
1020 return 1;
1021 }
1022
1023 /*
1024 * sun8i_crypto_schedule_worker(sc)
1025 *
1026 * Ensure that crypto engine thread context work to invoke task
1027 * callbacks will run promptly. Idempotent.
1028 */
1029 static void
sun8i_crypto_schedule_worker(struct sun8i_crypto_softc * sc)1030 sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc)
1031 {
1032
1033 KASSERT(mutex_owned(&sc->sc_intr_lock));
1034
1035 /* Start the worker if necessary. */
1036 if (!sc->sc_work_pending) {
1037 workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL);
1038 sc->sc_work_pending = true;
1039 }
1040 }
1041
1042 /*
1043 * sun8i_crypto_worker(wk, cookie)
1044 *
1045 * Thread-context worker: Invoke all task callbacks for which the
1046 * device has notified us of completion or for which we gave up
1047 * waiting.
1048 */
1049 static void
sun8i_crypto_worker(struct work * wk,void * cookie)1050 sun8i_crypto_worker(struct work *wk, void *cookie)
1051 {
1052 struct sun8i_crypto_softc *sc = cookie;
1053 uint32_t done, esr;
1054
1055 /*
1056 * Under the interrupt lock, acknowledge our work and claim the
1057 * done mask and error status.
1058 */
1059 mutex_enter(&sc->sc_intr_lock);
1060 KASSERT(sc->sc_work_pending);
1061 sc->sc_work_pending = false;
1062 done = sc->sc_done;
1063 esr = sc->sc_esr;
1064 sc->sc_done = 0;
1065 sc->sc_esr = 0;
1066 mutex_exit(&sc->sc_intr_lock);
1067
1068 /*
1069 * If we cleared any channels, it is time to allow opencrypto
1070 * to issue new operations. Asymmetric operations (which we
1071 * don't support, at the moment, but we could) and symmetric
1072 * operations (which we do) use the same task channels, so we
1073 * unblock both kinds.
1074 */
1075 if (sun8i_crypto_done(sc, done, esr, getticks())) {
1076 crypto_unblock(sc->sc_opencrypto.co_driverid,
1077 CRYPTO_SYMQ|CRYPTO_ASYMQ);
1078 }
1079 }
1080
1081 /*
1082 * sun8i_crypto_poll(sc, &done, &esr)
1083 *
1084 * Poll for completion. Sets done and esr to the mask of done
1085 * channels and error channels. Returns true if anything was
1086 * done or failed.
1087 */
1088 static bool
sun8i_crypto_poll(struct sun8i_crypto_softc * sc,uint32_t * donep,uint32_t * esrp)1089 sun8i_crypto_poll(struct sun8i_crypto_softc *sc,
1090 uint32_t *donep, uint32_t *esrp)
1091 {
1092 uint32_t isr, esr;
1093
1094 /*
1095 * Get and acknowledge the interrupts and error status.
1096 *
1097 * XXX Data sheet says the error status register is read-only,
1098 * but then advises writing 1 to bit x1xx (keysram access error
1099 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do?
1100 */
1101 isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR);
1102 esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR);
1103 sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr);
1104 sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr);
1105
1106 SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr);
1107
1108 *donep = __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE);
1109 *esrp = esr;
1110
1111 return *donep || *esrp;
1112 }
1113
1114 /*
1115 * sun8i_crypto_done(sc, done, esr, now)
1116 *
1117 * Invoke all task callbacks for the channels in done or esr, or
1118 * for which we gave up waiting, according to the time `now'.
1119 * Returns true if any channels completed or timed out.
1120 */
1121 static bool
sun8i_crypto_done(struct sun8i_crypto_softc * sc,uint32_t done,uint32_t esr,unsigned now)1122 sun8i_crypto_done(struct sun8i_crypto_softc *sc, uint32_t done, uint32_t esr,
1123 unsigned now)
1124 {
1125 uint32_t esr_chan;
1126 unsigned i;
1127 bool anydone = false;
1128 bool schedtimeout = false;
1129 int error;
1130
1131 /* Under the lock, process the channels. */
1132 mutex_enter(&sc->sc_lock);
1133 for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
1134 /* Check whether the channel is done. */
1135 if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) {
1136 /* Nope. Do we have a task to time out? */
1137 if (sc->sc_chan[i].cc_task != NULL) {
1138 if (now - sc->sc_chan[i].cc_starttime >=
1139 SUN8I_CRYPTO_TIMEOUT) {
1140 anydone |= sun8i_crypto_chan_done(sc,
1141 i, ETIMEDOUT);
1142 } else {
1143 schedtimeout = true;
1144 }
1145 }
1146 continue;
1147 }
1148
1149 /* Channel is done. Interpret the error if any. */
1150 esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i));
1151 if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) {
1152 device_printf(sc->sc_dev, "channel %u:"
1153 " alg not supported\n", i);
1154 error = ENODEV;
1155 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) {
1156 device_printf(sc->sc_dev, "channel %u:"
1157 " data length error\n", i);
1158 error = EIO; /* XXX */
1159 } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) {
1160 device_printf(sc->sc_dev, "channel %u:"
1161 " key sram error\n", i);
1162 error = EIO; /* XXX */
1163 } else if (esr_chan != 0) {
1164 error = EIO; /* generic I/O error */
1165 } else {
1166 error = 0;
1167 }
1168
1169 /*
1170 * Notify the task of completion. May release the lock
1171 * to invoke a callback.
1172 */
1173 anydone |= sun8i_crypto_chan_done(sc, i, error);
1174 }
1175 mutex_exit(&sc->sc_lock);
1176
1177 /*
1178 * If there are tasks still pending, make sure there's a
1179 * timeout scheduled for them. If the callout is already
1180 * pending, it will take another pass through here to time some
1181 * things out and schedule a new timeout.
1182 */
1183 if (schedtimeout && !callout_pending(&sc->sc_timeout))
1184 callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
1185
1186 return anydone;
1187 }
1188
1189 /*
1190 * sun8i_crypto_chan_done(sc, i, error)
1191 *
1192 * Notify the callback for the task on channel i, if there is one,
1193 * of the specified error, or 0 for success.
1194 */
1195 static bool
sun8i_crypto_chan_done(struct sun8i_crypto_softc * sc,unsigned i,int error)1196 sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error)
1197 {
1198 struct sun8i_crypto_task *task;
1199 uint32_t nbytes;
1200 uint32_t icr;
1201
1202 KASSERT(mutex_owned(&sc->sc_lock));
1203
1204 SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error);
1205
1206 /* Claim the task if there is one; bail if not. */
1207 if ((task = sc->sc_chan[i].cc_task) == NULL) {
1208 device_printf(sc->sc_dev, "channel %u: no task but error=%d\n",
1209 i, error);
1210 /* We did not clear a channel. */
1211 return false;
1212 }
1213 sc->sc_chan[i].cc_task = NULL;
1214
1215 /* Disable interrupts on this channel. */
1216 icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
1217 icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
1218 SUN8I_CRYPTO_ICR_INTR_EN);
1219 sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
1220
1221 /*
1222 * Finished sending the descriptor to the device by DMA.
1223 * Matches PREWRITE in sun8i_crypto_task_submit.
1224 */
1225 bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
1226 sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE);
1227
1228 /*
1229 * Finished with all the other bits of DMA too. Matches
1230 * sun8i_crypto_task_load.
1231 */
1232 nbytes = task->ct_nbytes;
1233 bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
1234 BUS_DMASYNC_POSTREAD);
1235 if (task->ct_flags & TASK_SRC)
1236 bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes,
1237 BUS_DMASYNC_POSTWRITE);
1238 if (task->ct_flags & TASK_CTR)
1239 bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0,
1240 task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD);
1241 if (task->ct_flags & TASK_IV)
1242 bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0,
1243 task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
1244 if (task->ct_flags & TASK_KEY)
1245 /* XXX Can we zero the bounce buffer if there is one? */
1246 bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0,
1247 task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
1248
1249 /* Temporarily release the lock to invoke the callback. */
1250 mutex_exit(&sc->sc_lock);
1251 SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error);
1252 (*task->ct_callback)(sc, task, task->ct_cookie, error);
1253 mutex_enter(&sc->sc_lock);
1254
1255 /* We cleared a channel. */
1256 return true;
1257 }
1258
1259 /*
1260 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags)
1261 *
1262 * Allocate a single-segment DMA-safe buffer and map it into KVA.
1263 * May fail if dmaflags is BUS_DMA_NOWAIT.
1264 */
1265 static int
sun8i_crypto_allocbuf(struct sun8i_crypto_softc * sc,size_t size,struct sun8i_crypto_buf * buf,int dmaflags)1266 sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size,
1267 struct sun8i_crypto_buf *buf, int dmaflags)
1268 {
1269 int error;
1270
1271 /* Allocate a DMA-safe buffer. */
1272 error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0,
1273 buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
1274 if (error)
1275 goto fail0;
1276
1277 /* Map the buffer into kernel virtual address space. */
1278 error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs,
1279 size, &buf->cb_kva, dmaflags);
1280 if (error)
1281 goto fail1;
1282
1283 /* Success! */
1284 return 0;
1285
1286 fail2: __unused
1287 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
1288 fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
1289 fail0: return error;
1290 }
1291
1292 /*
1293 * sun8i_crypto_freebuf(sc, buf)
1294 *
1295 * Unmap buf and free it.
1296 */
1297 static void
sun8i_crypto_freebuf(struct sun8i_crypto_softc * sc,size_t size,struct sun8i_crypto_buf * buf)1298 sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size,
1299 struct sun8i_crypto_buf *buf)
1300 {
1301
1302 bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
1303 bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
1304 }
1305
1306 /*
1307 * sun8i_crypto_rng_attach(sc)
1308 *
1309 * Attach an rndsource for the crypto engine's TRNG.
1310 */
1311 static void
sun8i_crypto_rng_attach(struct sun8i_crypto_softc * sc)1312 sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc)
1313 {
1314 device_t self = sc->sc_dev;
1315 struct sun8i_crypto_rng *rng = &sc->sc_rng;
1316 struct sun8i_crypto_task *task;
1317 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
1318 uint32_t done, esr;
1319 int error;
1320
1321 /* Preallocate a buffer to reuse. */
1322 error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf,
1323 BUS_DMA_WAITOK);
1324 if (error) {
1325 aprint_error_dev(self, "failed to allocate RNG buffer: %d\n",
1326 error);
1327 goto fail0;
1328 }
1329
1330 /* Create a task to reuse. */
1331 task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done,
1332 rng, PR_WAITOK);
1333 if (rng->cr_task == NULL) {
1334 aprint_error_dev(self, "failed to allocate RNG task\n");
1335 error = ENOMEM;
1336 goto fail1;
1337 }
1338
1339 /* Preload the destination map. */
1340 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
1341 rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
1342 if (error) {
1343 aprint_error_dev(self, "failed to load RNG buffer: %d\n",
1344 error);
1345 goto fail2;
1346 }
1347
1348 /*
1349 * Attach the rndsource. This will trigger an initial call to
1350 * it since we have RND_FLAG_HASCB.
1351 */
1352 rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc);
1353 rnd_attach_source(&rng->cr_rndsource, device_xname(self),
1354 RND_TYPE_RNG,
1355 RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB);
1356
1357 /*
1358 * Poll for the first call to the RNG to complete. If not done
1359 * after the timeout, force a timeout.
1360 */
1361 for (; rng->cr_pending && timo --> 0; DELAY(1000)) {
1362 if (sun8i_crypto_poll(sc, &done, &esr))
1363 (void)sun8i_crypto_done(sc, done, esr, getticks());
1364 }
1365 if (rng->cr_pending) {
1366 (void)sun8i_crypto_done(sc, 0, 0,
1367 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
1368 KASSERT(!rng->cr_pending);
1369 }
1370
1371 return;
1372
1373 fail3: __unused
1374 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
1375 fail2: sun8i_crypto_task_put(sc, task);
1376 fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf);
1377 fail0: return;
1378 }
1379
1380 /*
1381 * sun8i_crypto_rng_get(nbytes, cookie)
1382 *
1383 * On-demand rndsource callback: try to gather nbytes of entropy
1384 * and enter them into the pool ASAP.
1385 */
1386 static void
sun8i_crypto_rng_get(size_t nbytes,void * cookie)1387 sun8i_crypto_rng_get(size_t nbytes, void *cookie)
1388 {
1389 struct sun8i_crypto_softc *sc = cookie;
1390 struct sun8i_crypto_rng *rng = &sc->sc_rng;
1391 struct sun8i_crypto_task *task = rng->cr_task;
1392 bool pending;
1393 int error;
1394
1395 /*
1396 * Test and set the RNG-pending flag. If it's already in
1397 * progress, nothing to do here.
1398 */
1399 mutex_enter(&sc->sc_lock);
1400 pending = rng->cr_pending;
1401 rng->cr_pending = true;
1402 mutex_exit(&sc->sc_lock);
1403 if (pending)
1404 return;
1405
1406 /* Load the task descriptor. */
1407 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
1408 if (error)
1409 goto fail;
1410
1411 /* Submit! */
1412 error = sun8i_crypto_submit(sc, task);
1413 if (error)
1414 goto fail;
1415
1416 /* All done! */
1417 return;
1418
1419 fail: mutex_enter(&sc->sc_lock);
1420 rng->cr_pending = false;
1421 mutex_exit(&sc->sc_lock);
1422 }
1423
1424 static void
sun8i_crypto_rng_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1425 sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc,
1426 struct sun8i_crypto_task *task, void *cookie, int error)
1427 {
1428 struct sun8i_crypto_rng *rng = cookie;
1429 uint8_t *buf = rng->cr_buf.cb_kva;
1430 uint32_t entropybits;
1431
1432 KASSERT(rng == &sc->sc_rng);
1433
1434 /* If anything went wrong, forget about it. */
1435 if (error)
1436 goto out;
1437
1438 /*
1439 * This TRNG has quite low entropy at best. But if it fails a
1440 * repeated output test, then assume it's busted.
1441 */
1442 CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY);
1443 entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY;
1444 if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2,
1445 SUN8I_CRYPTO_RNGBYTES/2)) {
1446 device_printf(sc->sc_dev, "failed repeated output test\n");
1447 entropybits = 0;
1448 }
1449
1450 /*
1451 * Actually we don't believe in any of the entropy until this
1452 * device has had more scrutiny.
1453 */
1454 entropybits = 0;
1455
1456 /* Success! Enter and erase the data. */
1457 rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES,
1458 entropybits);
1459 explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES);
1460
1461 out: /* Done -- clear the RNG-pending flag. */
1462 mutex_enter(&sc->sc_lock);
1463 rng->cr_pending = false;
1464 mutex_exit(&sc->sc_lock);
1465 }
1466
1467 /*
1468 * Self-test
1469 */
1470
1471 static const uint8_t selftest_input[16];
1472 static const uint8_t selftest_key[16];
1473 static const uint8_t selftest_output[16] = {
1474 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b,
1475 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e,
1476 };
1477
1478 static bool
sun8i_crypto_selftest(struct sun8i_crypto_softc * sc)1479 sun8i_crypto_selftest(struct sun8i_crypto_softc *sc)
1480 {
1481 const size_t keybytes = sizeof selftest_key;
1482 const size_t nbytes = sizeof selftest_input;
1483 struct sun8i_crypto_selftest *selftest = &sc->sc_selftest;
1484 struct sun8i_crypto_task *task;
1485 unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
1486 uint32_t done, esr;
1487 int error;
1488
1489 CTASSERT(sizeof selftest_input == sizeof selftest_output);
1490
1491 selftest->cs_pending = true;
1492
1493 /* Allocate an input buffer. */
1494 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in,
1495 BUS_DMA_WAITOK);
1496 if (error)
1497 goto fail0;
1498
1499 /* Allocate a key buffer. */
1500 error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key,
1501 BUS_DMA_WAITOK);
1502 if (error)
1503 goto fail1;
1504
1505 /* Allocate an output buffer. */
1506 error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out,
1507 BUS_DMA_WAITOK);
1508 if (error)
1509 goto fail2;
1510
1511 /* Allocate a task descriptor. */
1512 task = selftest->cs_task = sun8i_crypto_task_get(sc,
1513 sun8i_crypto_selftest_done, selftest, PR_WAITOK);
1514 if (selftest->cs_task == NULL) {
1515 error = ENOMEM;
1516 goto fail3;
1517 }
1518
1519 /* Copy the input and key into their buffers. */
1520 memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes);
1521 memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes);
1522
1523 /* Load the key, src, and dst for DMA transfers. */
1524 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
1525 selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK);
1526 if (error)
1527 goto fail4;
1528 task->ct_flags |= TASK_KEY;
1529
1530 error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap,
1531 selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
1532 if (error)
1533 goto fail5;
1534 task->ct_flags |= TASK_SRC;
1535
1536 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
1537 selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
1538 if (error)
1539 goto fail6;
1540
1541 /* Set up the task descriptor. */
1542 error = sun8i_crypto_task_load_aesecb(sc, task, nbytes,
1543 SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC);
1544 if (error)
1545 goto fail7;
1546
1547 /* Submit! */
1548 error = sun8i_crypto_submit(sc, task);
1549 if (error)
1550 goto fail7;
1551 device_printf(sc->sc_dev, "AES-128 self-test initiated\n");
1552
1553 /*
1554 * Poll for completion. If not done after the timeout, force a
1555 * timeout.
1556 */
1557 for (; selftest->cs_pending && timo --> 0; DELAY(1000)) {
1558 if (sun8i_crypto_poll(sc, &done, &esr))
1559 (void)sun8i_crypto_done(sc, done, esr, getticks());
1560 }
1561 if (selftest->cs_pending) {
1562 (void)sun8i_crypto_done(sc, 0, 0,
1563 (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
1564 KASSERT(!selftest->cs_pending);
1565 }
1566
1567 return selftest->cs_passed;
1568
1569 fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
1570 fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
1571 fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
1572 fail4: sun8i_crypto_task_put(sc, task);
1573 fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
1574 fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
1575 fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
1576 fail0: aprint_error_dev(sc->sc_dev, "failed to run self-test, error=%d\n",
1577 error);
1578 selftest->cs_pending = false;
1579 return false;
1580 }
1581
1582 static bool
sun8i_crypto_selftest_check(struct sun8i_crypto_softc * sc,const char * title,size_t n,const void * expected,const void * actual)1583 sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title,
1584 size_t n, const void *expected, const void *actual)
1585 {
1586 const uint8_t *e = expected;
1587 const uint8_t *a = actual;
1588 size_t i;
1589
1590 if (memcmp(e, a, n) == 0)
1591 return true;
1592
1593 device_printf(sc->sc_dev, "self-test: %s\n", title);
1594 printf("expected: ");
1595 for (i = 0; i < n; i++)
1596 printf("%02hhx", e[i]);
1597 printf("\n");
1598 printf("actual: ");
1599 for (i = 0; i < n; i++)
1600 printf("%02hhx", a[i]);
1601 printf("\n");
1602 return false;
1603 }
1604
1605 static void
sun8i_crypto_selftest_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1606 sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc,
1607 struct sun8i_crypto_task *task, void *cookie, int error)
1608 {
1609 const size_t keybytes = sizeof selftest_key;
1610 const size_t nbytes = sizeof selftest_input;
1611 struct sun8i_crypto_selftest *selftest = cookie;
1612 bool ok = true;
1613
1614 KASSERT(selftest == &sc->sc_selftest);
1615
1616 /* If anything went wrong, fail now. */
1617 if (error) {
1618 device_printf(sc->sc_dev, "self-test error=%d\n", error);
1619 goto out;
1620 }
1621
1622 /*
1623 * Verify the input and key weren't clobbered, and verify the
1624 * output matches what we expect.
1625 */
1626 ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes,
1627 selftest_input, selftest->cs_in.cb_kva);
1628 ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes,
1629 selftest_key, selftest->cs_key.cb_kva);
1630 ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes,
1631 selftest_output, selftest->cs_out.cb_kva);
1632
1633 selftest->cs_passed = ok;
1634 if (ok)
1635 device_printf(sc->sc_dev, "AES-128 self-test passed\n");
1636
1637 out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
1638 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
1639 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
1640 sun8i_crypto_task_put(sc, task);
1641 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
1642 sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
1643 sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
1644 selftest->cs_pending = false;
1645 }
1646
1647 /*
1648 * Sysctl for testing
1649 */
1650
1651 struct sun8i_crypto_userreq {
1652 kmutex_t cu_lock;
1653 kcondvar_t cu_cv;
1654 size_t cu_size;
1655 struct sun8i_crypto_buf cu_buf;
1656 struct sun8i_crypto_task *cu_task;
1657 int cu_error;
1658 bool cu_done;
1659 bool cu_cancel;
1660 };
1661
1662 static void
sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc * sc)1663 sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc)
1664 {
1665 struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl;
1666 int error;
1667
1668 /* hw.sun8icryptoN (node) */
1669 error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node,
1670 CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev),
1671 SYSCTL_DESCR("sun8i crypto engine knobs"),
1672 NULL, 0, NULL, 0,
1673 CTL_HW, CTL_CREATE, CTL_EOL);
1674 if (error) {
1675 aprint_error_dev(sc->sc_dev,
1676 "failed to set up sysctl hw.%s: %d\n",
1677 device_xname(sc->sc_dev), error);
1678 return;
1679 }
1680
1681 /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */
1682 sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node,
1683 CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT,
1684 "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"),
1685 &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL);
1686 if (error) {
1687 aprint_error_dev(sc->sc_dev,
1688 "failed to set up sysctl hw.%s.rng: %d\n",
1689 device_xname(sc->sc_dev), error);
1690 return;
1691 }
1692 }
1693
1694 static int
sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)1695 sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)
1696 {
1697 struct sysctlnode node = *rnode;
1698 struct sun8i_crypto_softc *sc = node.sysctl_data;
1699 struct sun8i_crypto_userreq *req;
1700 struct sun8i_crypto_task *task;
1701 size_t size;
1702 int error;
1703
1704 /* If oldp == NULL, the caller wants to learn the size. */
1705 if (oldp == NULL) {
1706 *oldlenp = 4096;
1707 return 0;
1708 }
1709
1710 /* Truncate to 4096 bytes. */
1711 size = MIN(4096, *oldlenp);
1712 if (size == 0)
1713 return 0; /* nothing to do */
1714
1715 /* Allocate a request context. */
1716 req = kmem_alloc(sizeof(*req), KM_NOSLEEP);
1717 if (req == NULL)
1718 return ENOMEM;
1719
1720 /* Initialize the request context. */
1721 mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE);
1722 cv_init(&req->cu_cv, "sun8isy");
1723 req->cu_size = size;
1724 req->cu_error = EIO;
1725 req->cu_done = false;
1726 req->cu_cancel = false;
1727
1728 /* Allocate a buffer for the RNG output. */
1729 error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT);
1730 if (error)
1731 goto out0;
1732
1733 /* Allocate a task. */
1734 task = req->cu_task = sun8i_crypto_task_get(sc,
1735 sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT);
1736 if (task == NULL) {
1737 error = ENOMEM;
1738 goto out1;
1739 }
1740
1741 /* Set the task up for TRNG to our buffer. */
1742 error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
1743 req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
1744 if (error)
1745 goto out2;
1746 error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
1747 if (error)
1748 goto out3;
1749
1750 /* Submit! */
1751 error = sun8i_crypto_submit(sc, task);
1752 if (error) {
1753 /* Make sure we don't restart the syscall -- just fail. */
1754 if (error == ERESTART)
1755 error = EBUSY;
1756 goto out3;
1757 }
1758
1759 /* Wait for the request to complete. */
1760 mutex_enter(&req->cu_lock);
1761 while (!req->cu_done) {
1762 error = cv_wait_sig(&req->cu_cv, &req->cu_lock);
1763 if (error) {
1764 /*
1765 * If we finished while waiting to acquire the
1766 * lock, ignore the error and just return now.
1767 * Otherwise, notify the callback that it has
1768 * to clean up after us.
1769 */
1770 if (req->cu_done)
1771 error = 0;
1772 else
1773 req->cu_cancel = true;
1774 break;
1775 }
1776 }
1777 mutex_exit(&req->cu_lock);
1778
1779 /*
1780 * Return early on error from cv_wait_sig, which means
1781 * interruption; the callback will clean up instead.
1782 */
1783 if (error)
1784 return error;
1785
1786 /* Check for error from the device. */
1787 error = req->cu_error;
1788 if (error)
1789 goto out3;
1790
1791 /* Copy out the data. */
1792 node.sysctl_data = req->cu_buf.cb_kva;
1793 node.sysctl_size = size;
1794 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1795
1796 /* Clear the buffer. */
1797 explicit_memset(req->cu_buf.cb_kva, 0, size);
1798
1799 /* Clean up. */
1800 out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
1801 out2: sun8i_crypto_task_put(sc, task);
1802 out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
1803 out0: cv_destroy(&req->cu_cv);
1804 mutex_destroy(&req->cu_lock);
1805 kmem_free(req, sizeof(*req));
1806 return error;
1807 }
1808
1809 static void
sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1810 sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc,
1811 struct sun8i_crypto_task *task, void *cookie, int error)
1812 {
1813 struct sun8i_crypto_userreq *req = cookie;
1814 bool cancel;
1815
1816 /*
1817 * Notify the waiting thread of the error, and find out whether
1818 * that thread cancelled.
1819 */
1820 mutex_enter(&req->cu_lock);
1821 cancel = req->cu_cancel;
1822 req->cu_error = error;
1823 req->cu_done = true;
1824 cv_broadcast(&req->cu_cv);
1825 mutex_exit(&req->cu_lock);
1826
1827 /*
1828 * If it wasn't cancelled, we're done -- the main thread will
1829 * clean up after itself.
1830 */
1831 if (!cancel)
1832 return;
1833
1834 /* Clean up after the main thread cancelled. */
1835 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
1836 sun8i_crypto_task_put(sc, task);
1837 sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
1838 cv_destroy(&req->cu_cv);
1839 mutex_destroy(&req->cu_lock);
1840 kmem_free(req, sizeof(*req));
1841 }
1842
1843 /*
1844 * sun8i_crypto_register(sc)
1845 *
1846 * Register opencrypto algorithms supported by the crypto engine.
1847 */
1848 static void
sun8i_crypto_register(struct sun8i_crypto_softc * sc)1849 sun8i_crypto_register(struct sun8i_crypto_softc *sc)
1850 {
1851 struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto;
1852
1853 co->co_driverid = crypto_get_driverid(0);
1854 if (co->co_driverid == (uint32_t)-1) {
1855 aprint_error_dev(sc->sc_dev,
1856 "failed to register crypto driver\n");
1857 return;
1858 }
1859
1860 sun8i_crypto_register1(sc, CRYPTO_AES_CBC);
1861 sun8i_crypto_register1(sc, CRYPTO_AES_CTR);
1862 #ifdef CRYPTO_AES_ECB
1863 sun8i_crypto_register1(sc, CRYPTO_AES_ECB);
1864 #endif
1865 #ifdef CRYPTO_AES_XTS
1866 sun8i_crypto_register1(sc, CRYPTO_AES_XTS);
1867 #endif
1868 #ifdef CRYPTO_DES_CBC
1869 sun8i_crypto_register1(sc, CRYPTO_DES_CBC);
1870 #endif
1871 #ifdef CRYPTO_DES_ECB
1872 sun8i_crypto_register1(sc, CRYPTO_DES_ECB);
1873 #endif
1874 sun8i_crypto_register1(sc, CRYPTO_3DES_CBC);
1875 #ifdef CRYPTO_3DES_ECB
1876 sun8i_crypto_register1(sc, CRYPTO_3DES_ECB);
1877 #endif
1878
1879 sun8i_crypto_register1(sc, CRYPTO_MD5);
1880 sun8i_crypto_register1(sc, CRYPTO_SHA1);
1881 #ifdef CRYPTO_SHA224
1882 sun8i_crypto_register1(sc, CRYPTO_SHA224);
1883 #endif
1884 #ifdef CRYPTO_SHA256
1885 sun8i_crypto_register1(sc, CRYPTO_SHA256);
1886 #endif
1887
1888 sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC);
1889 sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC);
1890
1891 //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */
1892 }
1893
1894 /*
1895 * sun8i_crypto_register1(sc, alg)
1896 *
1897 * Register support for one algorithm alg using
1898 * sun8i_crypto_newsession/freesession/process.
1899 */
1900 static void
sun8i_crypto_register1(struct sun8i_crypto_softc * sc,uint32_t alg)1901 sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg)
1902 {
1903
1904 crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0,
1905 sun8i_crypto_newsession,
1906 sun8i_crypto_freesession,
1907 sun8i_crypto_process,
1908 sc);
1909 }
1910
1911 /*
1912 * sun8i_crypto_newsession(cookie, sidp, cri)
1913 *
1914 * Called by opencrypto to allocate a new session. We don't keep
1915 * track of sessions, since there are no persistent keys in the
1916 * hardware that we take advantage of, so this only validates the
1917 * crypto operations and returns a dummy session id of 1.
1918 */
1919 static int
sun8i_crypto_newsession(void * cookie,uint32_t * sidp,struct cryptoini * cri)1920 sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri)
1921 {
1922
1923 /* No composition of operations is supported here. */
1924 if (cri->cri_next)
1925 return EINVAL;
1926
1927 /*
1928 * No variation of rounds is supported here. (XXX Unused and
1929 * unimplemented in opencrypto(9) altogether?)
1930 */
1931 if (cri->cri_rnd)
1932 return EINVAL;
1933
1934 /*
1935 * Validate per-algorithm key length.
1936 *
1937 * XXX Does opencrypto(9) do this internally?
1938 */
1939 switch (cri->cri_alg) {
1940 case CRYPTO_MD5:
1941 case CRYPTO_SHA1:
1942 #ifdef CRYPTO_SHA224
1943 case CRYPTO_SHA224:
1944 #endif
1945 #ifdef CRYPTO_SHA256
1946 case CRYPTO_SHA256:
1947 #endif
1948 if (cri->cri_klen)
1949 return EINVAL;
1950 break;
1951 case CRYPTO_AES_CBC:
1952 #ifdef CRYPTO_AES_ECB
1953 case CRYPTO_AES_ECB:
1954 #endif
1955 switch (cri->cri_klen) {
1956 case 128:
1957 case 192:
1958 case 256:
1959 break;
1960 default:
1961 return EINVAL;
1962 }
1963 break;
1964 case CRYPTO_AES_CTR:
1965 /*
1966 * opencrypto `AES-CTR' takes four bytes of the input
1967 * block as the last four bytes of the key, for reasons
1968 * that are not entirely clear.
1969 */
1970 switch (cri->cri_klen) {
1971 case 128 + 32:
1972 case 192 + 32:
1973 case 256 + 32:
1974 break;
1975 default:
1976 return EINVAL;
1977 }
1978 break;
1979 #ifdef CRYPTO_AES_XTS
1980 case CRYPTO_AES_XTS:
1981 switch (cri->cri_klen) {
1982 case 256:
1983 case 384:
1984 case 512:
1985 break;
1986 default:
1987 return EINVAL;
1988 }
1989 break;
1990 #endif
1991 case CRYPTO_DES_CBC:
1992 #ifdef CRYPTO_DES_ECB
1993 case CRYPTO_DES_ECB:
1994 #endif
1995 switch (cri->cri_klen) {
1996 case 64:
1997 break;
1998 default:
1999 return EINVAL;
2000 }
2001 break;
2002 case CRYPTO_3DES_CBC:
2003 #ifdef CRYPTO_3DES_ECB
2004 case CRYPTO_3DES_ECB:
2005 #endif
2006 switch (cri->cri_klen) {
2007 case 192:
2008 break;
2009 default:
2010 return EINVAL;
2011 }
2012 break;
2013 case CRYPTO_SHA1_HMAC:
2014 /*
2015 * XXX Unclear what the length limit is, but since HMAC
2016 * behaves qualitatively different for a key of at
2017 * least the full block size -- and is generally best
2018 * to use with half the block size -- let's limit it to
2019 * one block.
2020 */
2021 if (cri->cri_klen % 8)
2022 return EINVAL;
2023 if (cri->cri_klen > 512)
2024 return EINVAL;
2025 break;
2026 case CRYPTO_SHA2_256_HMAC:
2027 if (cri->cri_klen % 8)
2028 return EINVAL;
2029 if (cri->cri_klen > 512)
2030 return EINVAL;
2031 break;
2032 default:
2033 panic("unsupported algorithm %d", cri->cri_alg);
2034 }
2035
2036 KASSERT(cri->cri_klen % 8 == 0);
2037
2038 /* Success! */
2039 *sidp = 1;
2040 return 0;
2041 }
2042
2043 /*
2044 * sun8i_crypto_freesession(cookie, dsid)
2045 *
2046 * Called by opencrypto to free a session. We don't keep track of
2047 * sessions, since there are no persistent keys in the hardware
2048 * that we take advantage of, so this is a no-op.
2049 *
2050 * Note: dsid is actually a 64-bit quantity containing both the
2051 * driver id in the high half and the session id in the low half.
2052 */
2053 static void
sun8i_crypto_freesession(void * cookie,uint64_t dsid)2054 sun8i_crypto_freesession(void *cookie, uint64_t dsid)
2055 {
2056
2057 KASSERT((dsid & 0xffffffff) == 1);
2058 }
2059
2060 /*
2061 * sun8i_crypto_ivlen(crd)
2062 *
2063 * Return the crypto engine's notion of `IV length', in bytes, for
2064 * an opencrypto operation.
2065 */
2066 static u_int
sun8i_crypto_ivlen(const struct cryptodesc * crd)2067 sun8i_crypto_ivlen(const struct cryptodesc *crd)
2068 {
2069
2070 switch (crd->crd_alg) {
2071 case CRYPTO_AES_CBC:
2072 return 16;
2073 #ifdef CRYPTO_AES_XTS
2074 case CRYPTO_AES_XTS:
2075 return 16;
2076 #endif
2077 case CRYPTO_AES_CTR: /* XXX opencrypto quirk */
2078 return 8;
2079 #ifdef CRYPTO_DES_CBC
2080 case CRYPTO_DES_CBC:
2081 return 8;
2082 #endif
2083 case CRYPTO_3DES_CBC:
2084 return 8;
2085 case CRYPTO_MD5:
2086 return 16;
2087 #ifdef CRYPTO_SHA224
2088 case CRYPTO_SHA224:
2089 return 32;
2090 #endif
2091 #ifdef CRYPTO_SHA256
2092 case CRYPTO_SHA256:
2093 return 32;
2094 #endif
2095 case CRYPTO_SHA1_HMAC:
2096 return 20;
2097 case CRYPTO_SHA2_256_HMAC:
2098 return 32;
2099 default:
2100 return 0;
2101 }
2102 }
2103
2104 /*
2105 * sun8i_crypto_process(cookie, crp, hint)
2106 *
2107 * Main opencrypto processing dispatch.
2108 */
2109 static int
sun8i_crypto_process(void * cookie,struct cryptop * crp,int hint)2110 sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint)
2111 {
2112 struct sun8i_crypto_softc *sc = cookie;
2113 struct sun8i_crypto_task *task;
2114 struct cryptodesc *crd = crp->crp_desc;
2115 unsigned klen, ivlen;
2116 uint32_t tdqc = 0, tdqs = 0;
2117 uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0;
2118 const uint32_t tdqa = 0;
2119 int error;
2120
2121 SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint);
2122
2123 /* Reject compositions -- we do not handle them. */
2124 if (crd->crd_next != NULL) {
2125 error = EOPNOTSUPP;
2126 goto fail0;
2127 }
2128
2129 /* Reject transfers with nonsense skip. */
2130 if (crd->crd_skip < 0) {
2131 error = EINVAL;
2132 goto fail0;
2133 }
2134
2135 /*
2136 * Actually just reject any nonzero skip, because it requires
2137 * DMA segment bookkeeping that we don't do yet.
2138 */
2139 if (crd->crd_skip) {
2140 error = EOPNOTSUPP;
2141 goto fail0;
2142 }
2143
2144 /* Reject large transfers. */
2145 if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) {
2146 error = EFBIG;
2147 goto fail0;
2148 }
2149
2150 /* Reject nonsense, unaligned, or mismatched lengths. */
2151 if (crd->crd_len < 0 ||
2152 crd->crd_len % 4 ||
2153 crd->crd_len != crp->crp_ilen) {
2154 error = EINVAL;
2155 goto fail0;
2156 }
2157
2158 /* Reject mismatched buffer lengths. */
2159 /* XXX Handle crd_skip. */
2160 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2161 struct mbuf *m = crp->crp_buf;
2162 uint32_t nbytes = 0;
2163 while (m != NULL) {
2164 KASSERT(m->m_len >= 0);
2165 if (m->m_len > crd->crd_len ||
2166 nbytes > crd->crd_len - m->m_len) {
2167 error = EINVAL;
2168 goto fail0;
2169 }
2170 nbytes += m->m_len;
2171 m = m->m_next;
2172 }
2173 if (nbytes != crd->crd_len) {
2174 error = EINVAL;
2175 goto fail0;
2176 }
2177 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2178 struct uio *uio = crp->crp_buf;
2179 if (uio->uio_resid != crd->crd_len) {
2180 error = EINVAL;
2181 goto fail0;
2182 }
2183 }
2184
2185 /* Get a task, or fail with ERESTART if we can't. */
2186 task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp,
2187 PR_NOWAIT);
2188 if (task == NULL) {
2189 /*
2190 * Don't invoke crypto_done -- we are asking the
2191 * opencrypto(9) machinery to queue the request and get
2192 * back to us.
2193 */
2194 SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint);
2195 return ERESTART;
2196 }
2197
2198 /* Load key in, if relevant. */
2199 klen = crd->crd_klen;
2200 if (klen) {
2201 if (crd->crd_alg == CRYPTO_AES_CTR)
2202 /* AES-CTR is special -- see IV processing below. */
2203 klen -= 32;
2204 error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
2205 crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT);
2206 if (error)
2207 goto fail1;
2208 task->ct_flags |= TASK_KEY;
2209 }
2210
2211 /* Handle the IV, if relevant. */
2212 ivlen = sun8i_crypto_ivlen(crd);
2213 if (ivlen) {
2214 void *iv;
2215
2216 /*
2217 * If there's an explicit IV, use it; otherwise
2218 * randomly generate one.
2219 */
2220 if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
2221 iv = crd->crd_iv;
2222 } else {
2223 cprng_fast(task->ct_iv, ivlen);
2224 iv = task->ct_iv;
2225 }
2226
2227 /*
2228 * If the IV is not already present in the user's
2229 * buffer, copy it over.
2230 */
2231 if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
2232 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2233 m_copyback(crp->crp_buf, crd->crd_inject,
2234 ivlen, iv);
2235 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2236 cuio_copyback(crp->crp_buf, crd->crd_inject,
2237 ivlen, iv);
2238 } else {
2239 panic("invalid buffer type %x",
2240 crp->crp_flags);
2241 }
2242 }
2243
2244 /*
2245 * opencrypto's idea of `AES-CTR' is special.
2246 *
2247 * - The low 4 bytes of the input block are drawn from
2248 * an extra 4 bytes at the end of the key.
2249 *
2250 * - The next 8 bytes of the input block are drawn from
2251 * the opencrypto iv.
2252 *
2253 * - The high 4 bytes are the big-endian block counter,
2254 * which starts at 1 because why not.
2255 */
2256 if (crd->crd_alg == CRYPTO_AES_CTR) {
2257 uint8_t block[16];
2258 uint32_t blkno = 1;
2259
2260 /* Format the initial input block. */
2261 memcpy(block, crd->crd_key + klen/8, 4);
2262 memcpy(block + 4, iv, 8);
2263 be32enc(block + 12, blkno);
2264
2265 /* Copy it into the DMA buffer. */
2266 memcpy(task->ct_iv, block, 16);
2267 iv = task->ct_iv;
2268 ivlen = 16;
2269 }
2270
2271 /* Load the IV. */
2272 error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen,
2273 NULL, BUS_DMA_NOWAIT);
2274 if (error)
2275 goto fail1;
2276 task->ct_flags |= TASK_IV;
2277 }
2278
2279 /* Load the src and dst. */
2280 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2281 struct mbuf *m = crp->crp_buf;
2282
2283 /* XXX Handle crd_skip. */
2284 KASSERT(crd->crd_skip == 0);
2285 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m,
2286 BUS_DMA_NOWAIT);
2287 if (error)
2288 goto fail1;
2289 task->ct_flags |= TASK_SRC;
2290
2291 /* XXX Handle crd_skip. */
2292 KASSERT(crd->crd_skip == 0);
2293 error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m,
2294 BUS_DMA_NOWAIT);
2295 if (error)
2296 goto fail1;
2297 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2298 struct uio *uio = crp->crp_buf;
2299
2300 /* XXX Handle crd_skip. */
2301 KASSERT(crd->crd_skip == 0);
2302 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio,
2303 BUS_DMA_NOWAIT);
2304 if (error)
2305 goto fail1;
2306 task->ct_flags |= TASK_SRC;
2307
2308 /* XXX Handle crd_skip. */
2309 KASSERT(crd->crd_skip == 0);
2310 error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio,
2311 BUS_DMA_NOWAIT);
2312 if (error)
2313 goto fail1;
2314 } else {
2315 panic("invalid buffer type %x", crp->crp_flags);
2316 }
2317
2318 /* Set the encryption direction. */
2319 if (crd->crd_flags & CRD_F_ENCRYPT)
2320 dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC;
2321 else
2322 dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC;
2323 tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
2324
2325 /* Set the method. */
2326 switch (crd->crd_alg) {
2327 case CRYPTO_AES_CBC:
2328 case CRYPTO_AES_CTR:
2329 #ifdef CRYPTO_AES_ECB
2330 case CRYPTO_AES_ECB:
2331 #endif
2332 method = SUN8I_CRYPTO_TDQC_METHOD_AES;
2333 break;
2334 #ifdef CRYPTO_AES_XTS
2335 case CRYPTO_AES_XTS:
2336 method = SUN8I_CRYPTO_TDQC_METHOD_AES;
2337 break;
2338 #endif
2339 case CRYPTO_DES_CBC:
2340 #ifdef CRYPTO_DES_ECB
2341 case CRYPTO_DES_ECB:
2342 #endif
2343 method = SUN8I_CRYPTO_TDQC_METHOD_DES;
2344 break;
2345 case CRYPTO_3DES_CBC:
2346 #ifdef CRYPTO_3DES_ECB
2347 case CRYPTO_3DES_ECB:
2348 #endif
2349 method = SUN8I_CRYPTO_TDQC_METHOD_3DES;
2350 break;
2351 case CRYPTO_MD5:
2352 method = SUN8I_CRYPTO_TDQC_METHOD_MD5;
2353 break;
2354 case CRYPTO_SHA1:
2355 method = SUN8I_CRYPTO_TDQC_METHOD_SHA1;
2356 break;
2357 #ifdef CRYPTO_SHA224
2358 case CRYPTO_SHA224:
2359 method = SUN8I_CRYPTO_TDQC_METHOD_SHA224;
2360 break;
2361 #endif
2362 #ifdef CRYPTO_SHA256
2363 case CRYPTO_SHA256:
2364 method = SUN8I_CRYPTO_TDQC_METHOD_SHA256;
2365 break;
2366 #endif
2367 case CRYPTO_SHA1_HMAC:
2368 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1;
2369 break;
2370 case CRYPTO_SHA2_256_HMAC:
2371 method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256;
2372 break;
2373 default:
2374 panic("unknown algorithm %d", crd->crd_alg);
2375 }
2376 tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD);
2377
2378 /* Set the key selector. No idea how to use the internal keys. */
2379 tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
2380 SUN8I_CRYPTO_TDQS_SKEY_SELECT);
2381
2382 /* XXX Deal with AES_CTS_Last_Block_Flag. */
2383
2384 /* Set the mode. */
2385 switch (crd->crd_alg) {
2386 #ifdef CRYPTO_AES_ECB
2387 case CRYPTO_AES_ECB:
2388 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
2389 break;
2390 #endif
2391 #ifdef CRYPTO_DES_ECB
2392 case CRYPTO_DES_ECB:
2393 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
2394 break;
2395 #endif
2396 #ifdef CRYPTO_3DES_ECB
2397 case CRYPTO_3DES_ECB:
2398 mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
2399 break;
2400 #endif
2401 case CRYPTO_AES_CBC:
2402 case CRYPTO_DES_CBC:
2403 case CRYPTO_3DES_CBC:
2404 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC;
2405 break;
2406 case CRYPTO_AES_CTR:
2407 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR;
2408 break;
2409 #ifdef CRYPTO_AES_XTS
2410 case CRYPTO_AES_XTS:
2411 mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS;
2412 break;
2413 #endif
2414 default:
2415 panic("unknown algorithm %d", crd->crd_alg);
2416 }
2417 tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE);
2418
2419 /* Set the CTR width. */
2420 switch (crd->crd_alg) {
2421 case CRYPTO_AES_CTR:
2422 ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32;
2423 break;
2424 }
2425 tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH);
2426
2427 /* Set the AES key size. */
2428 switch (crd->crd_alg) {
2429 case CRYPTO_AES_CBC:
2430 #ifdef CRYPTO_AES_ECB
2431 case CRYPTO_AES_ECB:
2432 #endif
2433 switch (crd->crd_klen) {
2434 case 128:
2435 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
2436 break;
2437 case 192:
2438 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
2439 break;
2440 case 256:
2441 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
2442 break;
2443 default:
2444 panic("invalid AES key size in bits: %u",
2445 crd->crd_klen);
2446 }
2447 break;
2448 case CRYPTO_AES_CTR:
2449 switch (crd->crd_klen) {
2450 case 128 + 32:
2451 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
2452 break;
2453 case 192 + 32:
2454 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
2455 break;
2456 case 256 + 32:
2457 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
2458 break;
2459 default:
2460 panic("invalid `AES-CTR' ` ``key'' size' in bits: %u",
2461 crd->crd_klen);
2462 }
2463 break;
2464 #ifdef CRYPTO_AES_XTS
2465 case CRYPTO_AES_XTS:
2466 switch (crd->crd_klen) {
2467 case 256:
2468 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
2469 break;
2470 case 384:
2471 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
2472 break;
2473 case 512:
2474 aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
2475 break;
2476 default:
2477 panic("invalid AES-XTS key size in bits: %u",
2478 crd->crd_klen);
2479 }
2480 break;
2481 #endif
2482 }
2483 tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
2484
2485 /* Set up the task descriptor. */
2486 error = sun8i_crypto_task_load(sc, task, crd->crd_len,
2487 tdqc, tdqs, tdqa);
2488 if (error)
2489 goto fail2;
2490
2491 /* Submit! */
2492 error = sun8i_crypto_submit(sc, task);
2493 if (error)
2494 goto fail2;
2495
2496 /* Success! */
2497 SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task);
2498 return 0;
2499
2500 fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
2501 fail1: if (task->ct_flags & TASK_SRC)
2502 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
2503 if (task->ct_flags & TASK_CTR)
2504 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
2505 if (task->ct_flags & TASK_IV)
2506 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
2507 if (task->ct_flags & TASK_KEY)
2508 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
2509 sun8i_crypto_task_put(sc, task);
2510 fail0: KASSERT(error);
2511 KASSERT(error != ERESTART);
2512 crp->crp_etype = error;
2513 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
2514 crypto_done(crp);
2515 return 0;
2516 }
2517
2518 /*
2519 * sun8i_crypto_callback(sc, task, cookie, error)
2520 *
2521 * Completion callback for a task submitted via opencrypto.
2522 * Release the task and pass the error on to opencrypto with
2523 * crypto_done.
2524 */
2525 static void
sun8i_crypto_callback(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)2526 sun8i_crypto_callback(struct sun8i_crypto_softc *sc,
2527 struct sun8i_crypto_task *task, void *cookie, int error)
2528 {
2529 struct cryptop *crp = cookie;
2530 struct cryptodesc *crd __diagused = crp->crp_desc;
2531
2532 KASSERT(error != ERESTART);
2533 KASSERT(crd != NULL);
2534 KASSERT(crd->crd_next == NULL);
2535
2536 /* Return the number of bytes processed. */
2537 crp->crp_olen = error ? 0 : crp->crp_ilen;
2538
2539 bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
2540 bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
2541 if (task->ct_flags & TASK_CTR)
2542 bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
2543 if (task->ct_flags & TASK_IV)
2544 bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
2545 if (task->ct_flags & TASK_KEY)
2546 bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
2547 sun8i_crypto_task_put(sc, task);
2548 KASSERT(error != ERESTART);
2549 crp->crp_etype = error;
2550 SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
2551 crypto_done(crp);
2552 }
2553