1*fdf161e4Sriastradh /* $NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $ */
2defaca02Sriastradh
3defaca02Sriastradh /*-
4defaca02Sriastradh * Copyright (c) 2019 The NetBSD Foundation, Inc.
5defaca02Sriastradh * All rights reserved.
6defaca02Sriastradh *
7defaca02Sriastradh * This code is derived from software contributed to The NetBSD Foundation
8defaca02Sriastradh * by Taylor R. Campbell.
9defaca02Sriastradh *
10defaca02Sriastradh * Redistribution and use in source and binary forms, with or without
11defaca02Sriastradh * modification, are permitted provided that the following conditions
12defaca02Sriastradh * are met:
13defaca02Sriastradh * 1. Redistributions of source code must retain the above copyright
14defaca02Sriastradh * notice, this list of conditions and the following disclaimer.
15defaca02Sriastradh * 2. Redistributions in binary form must reproduce the above copyright
16defaca02Sriastradh * notice, this list of conditions and the following disclaimer in the
17defaca02Sriastradh * documentation and/or other materials provided with the distribution.
18defaca02Sriastradh *
19defaca02Sriastradh * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20defaca02Sriastradh * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21defaca02Sriastradh * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22defaca02Sriastradh * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23defaca02Sriastradh * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24defaca02Sriastradh * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25defaca02Sriastradh * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26defaca02Sriastradh * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27defaca02Sriastradh * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28defaca02Sriastradh * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29defaca02Sriastradh * POSSIBILITY OF SUCH DAMAGE.
30defaca02Sriastradh */
31defaca02Sriastradh
32defaca02Sriastradh /*
33defaca02Sriastradh * sun8i_crypto -- Allwinner Crypto Engine driver
34defaca02Sriastradh *
35defaca02Sriastradh * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64
36defaca02Sriastradh * User Manual v1.1, on pp. 230--241. We only use it for the TRNG at
37defaca02Sriastradh * the moment, but in principle it could be wired up with opencrypto(9)
38defaca02Sriastradh * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1,
39defaca02Sriastradh * HMAC-HA256, RSA, and an undocumented PRNG. It also seems to support
40defaca02Sriastradh * AES keys in SRAM (for some kind of HDMI HDCP stuff?).
41defaca02Sriastradh *
42defaca02Sriastradh * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf
43defaca02Sriastradh */
44defaca02Sriastradh
45defaca02Sriastradh #include <sys/cdefs.h>
46*fdf161e4Sriastradh __KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.32 2022/05/22 11:39:26 riastradh Exp $");
47defaca02Sriastradh
48defaca02Sriastradh #include <sys/types.h>
49defaca02Sriastradh #include <sys/param.h>
50defaca02Sriastradh #include <sys/atomic.h>
51defaca02Sriastradh #include <sys/bus.h>
52defaca02Sriastradh #include <sys/callout.h>
53defaca02Sriastradh #include <sys/conf.h>
5486953edcSriastradh #include <sys/cprng.h>
55defaca02Sriastradh #include <sys/device.h>
56defaca02Sriastradh #include <sys/kernel.h>
57defaca02Sriastradh #include <sys/kmem.h>
5886953edcSriastradh #include <sys/mbuf.h>
59defaca02Sriastradh #include <sys/mutex.h>
60defaca02Sriastradh #include <sys/rndsource.h>
6186953edcSriastradh #include <sys/sdt.h>
62defaca02Sriastradh #include <sys/sysctl.h>
63defaca02Sriastradh #include <sys/workqueue.h>
64defaca02Sriastradh
65defaca02Sriastradh #include <dev/fdt/fdtvar.h>
66defaca02Sriastradh
6786953edcSriastradh #include <opencrypto/cryptodev.h>
6886953edcSriastradh
69defaca02Sriastradh #include <arm/sunxi/sun8i_crypto.h>
70defaca02Sriastradh
71defaca02Sriastradh #define SUN8I_CRYPTO_TIMEOUT hz
72f9ae7d17Sriastradh #define SUN8I_CRYPTO_RNGENTROPY 100 /* estimated bits per bit of entropy */
73f9ae7d17Sriastradh #define SUN8I_CRYPTO_RNGBYTES PAGE_SIZE
74defaca02Sriastradh
751b26e856Sbad struct sun8i_crypto_config {
761b26e856Sbad u_int mod_rate; /* module clock rate */
771b26e856Sbad };
781b26e856Sbad
791b26e856Sbad /*
801b26e856Sbad * The module clock is set to 50 MHz on H3, 300 MHz otherwise.
811b26e856Sbad * From Linux drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c:
821b26e856Sbad * Module clock is lower on H3 than other SoC due to some DMA
831b26e856Sbad * timeout occurring with high value.
841b26e856Sbad */
851b26e856Sbad static const struct sun8i_crypto_config sun50i_a64_crypto_config = {
861b26e856Sbad .mod_rate = 300*1000*1000,
871b26e856Sbad };
881b26e856Sbad
891b26e856Sbad static const struct sun8i_crypto_config sun50i_h5_crypto_config = {
901b26e856Sbad .mod_rate = 300*1000*1000,
911b26e856Sbad };
921b26e856Sbad
931b26e856Sbad static const struct sun8i_crypto_config sun8i_h3_crypto_config = {
941b26e856Sbad .mod_rate = 50*1000*1000,
951b26e856Sbad };
961b26e856Sbad
97defaca02Sriastradh struct sun8i_crypto_task;
98defaca02Sriastradh
99defaca02Sriastradh struct sun8i_crypto_buf {
100defaca02Sriastradh bus_dma_segment_t cb_seg[1];
101defaca02Sriastradh int cb_nsegs;
102defaca02Sriastradh void *cb_kva;
103defaca02Sriastradh };
104defaca02Sriastradh
105defaca02Sriastradh struct sun8i_crypto_softc {
106defaca02Sriastradh device_t sc_dev;
107defaca02Sriastradh bus_space_tag_t sc_bst;
108defaca02Sriastradh bus_space_handle_t sc_bsh;
109defaca02Sriastradh bus_dma_tag_t sc_dmat;
11042356a74Sriastradh struct pool_cache *sc_taskpool;
1111b26e856Sbad
1121b26e856Sbad const struct sun8i_crypto_config *sc_cfg;
1131b26e856Sbad
1143c959c1cSriastradh struct workqueue *sc_wq;
1153c959c1cSriastradh void *sc_ih;
116c92613e9Sriastradh bool sc_polling;
1173c959c1cSriastradh
118defaca02Sriastradh kmutex_t sc_lock;
119defaca02Sriastradh struct sun8i_crypto_chan {
120defaca02Sriastradh struct sun8i_crypto_task *cc_task;
121defaca02Sriastradh unsigned cc_starttime;
122defaca02Sriastradh } sc_chan[SUN8I_CRYPTO_NCHAN];
123defaca02Sriastradh struct callout sc_timeout;
1243c959c1cSriastradh
1253c959c1cSriastradh kmutex_t sc_intr_lock;
126defaca02Sriastradh uint32_t sc_done;
127defaca02Sriastradh uint32_t sc_esr;
1283c959c1cSriastradh struct work sc_work;
129defaca02Sriastradh bool sc_work_pending;
1303c959c1cSriastradh
131defaca02Sriastradh struct sun8i_crypto_rng {
132defaca02Sriastradh struct sun8i_crypto_buf cr_buf;
133defaca02Sriastradh struct sun8i_crypto_task *cr_task;
134defaca02Sriastradh struct krndsource cr_rndsource;
135defaca02Sriastradh bool cr_pending;
136defaca02Sriastradh } sc_rng;
137defaca02Sriastradh struct sun8i_crypto_selftest {
138defaca02Sriastradh struct sun8i_crypto_buf cs_in;
139defaca02Sriastradh struct sun8i_crypto_buf cs_key;
140defaca02Sriastradh struct sun8i_crypto_buf cs_out;
141defaca02Sriastradh struct sun8i_crypto_task *cs_task;
142c92613e9Sriastradh bool cs_pending;
143c92613e9Sriastradh bool cs_passed;
144defaca02Sriastradh } sc_selftest;
145defaca02Sriastradh struct sun8i_crypto_sysctl {
146defaca02Sriastradh struct sysctllog *cy_log;
147defaca02Sriastradh const struct sysctlnode *cy_root_node;
148defaca02Sriastradh const struct sysctlnode *cy_trng_node;
149defaca02Sriastradh } sc_sysctl;
15086953edcSriastradh struct sun8i_crypto_opencrypto {
15186953edcSriastradh uint32_t co_driverid;
15286953edcSriastradh } sc_opencrypto;
153defaca02Sriastradh };
154defaca02Sriastradh
155defaca02Sriastradh struct sun8i_crypto_task {
15642356a74Sriastradh struct sun8i_crypto_buf ct_descbuf;
157defaca02Sriastradh struct sun8i_crypto_taskdesc *ct_desc;
15886953edcSriastradh struct sun8i_crypto_buf ct_ivbuf;
15986953edcSriastradh void *ct_iv;
16086953edcSriastradh struct sun8i_crypto_buf ct_ctrbuf;
16186953edcSriastradh void *ct_ctr;
16242356a74Sriastradh bus_dmamap_t ct_descmap;
16342356a74Sriastradh bus_dmamap_t ct_keymap;
16486953edcSriastradh bus_dmamap_t ct_ivmap; /* IV input */
16586953edcSriastradh bus_dmamap_t ct_ctrmap; /* updated IV output */
16642356a74Sriastradh bus_dmamap_t ct_srcmap;
16742356a74Sriastradh bus_dmamap_t ct_dstmap;
16842356a74Sriastradh uint32_t ct_nbytes;
16942356a74Sriastradh int ct_flags;
17042356a74Sriastradh #define TASK_KEY __BIT(0)
17142356a74Sriastradh #define TASK_IV __BIT(1)
17242356a74Sriastradh #define TASK_CTR __BIT(2)
17342356a74Sriastradh #define TASK_SRC __BIT(3)
17442356a74Sriastradh #define TASK_BYTES __BIT(4) /* datalen is in bytes, not words */
175defaca02Sriastradh void (*ct_callback)(struct sun8i_crypto_softc *,
176defaca02Sriastradh struct sun8i_crypto_task *, void *, int);
177defaca02Sriastradh void *ct_cookie;
178defaca02Sriastradh };
179defaca02Sriastradh
18042356a74Sriastradh #define SUN8I_CRYPTO_MAXDMASIZE PAGE_SIZE
18142356a74Sriastradh #define SUN8I_CRYPTO_MAXDMASEGSIZE PAGE_SIZE
18242356a74Sriastradh
18342356a74Sriastradh CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN);
18442356a74Sriastradh CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN);
18542356a74Sriastradh
186defaca02Sriastradh /*
187defaca02Sriastradh * Forward declarations
188defaca02Sriastradh */
189defaca02Sriastradh
190defaca02Sriastradh static int sun8i_crypto_match(device_t, cfdata_t, void *);
191defaca02Sriastradh static void sun8i_crypto_attach(device_t, device_t, void *);
192defaca02Sriastradh
19342356a74Sriastradh static int sun8i_crypto_task_ctor(void *, void *, int);
19442356a74Sriastradh static void sun8i_crypto_task_dtor(void *, void *);
195defaca02Sriastradh static struct sun8i_crypto_task *
196defaca02Sriastradh sun8i_crypto_task_get(struct sun8i_crypto_softc *,
197defaca02Sriastradh void (*)(struct sun8i_crypto_softc *,
198defaca02Sriastradh struct sun8i_crypto_task *, void *, int),
19942356a74Sriastradh void *, int);
200defaca02Sriastradh static void sun8i_crypto_task_put(struct sun8i_crypto_softc *,
201defaca02Sriastradh struct sun8i_crypto_task *);
202defaca02Sriastradh
20342356a74Sriastradh static int sun8i_crypto_task_load(struct sun8i_crypto_softc *,
20442356a74Sriastradh struct sun8i_crypto_task *, uint32_t,
20542356a74Sriastradh uint32_t, uint32_t, uint32_t);
20686953edcSriastradh static int sun8i_crypto_task_scatter(struct sun8i_crypto_task *,
20786953edcSriastradh struct sun8i_crypto_adrlen *, bus_dmamap_t, uint32_t);
208defaca02Sriastradh
20942356a74Sriastradh static int sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *,
210defaca02Sriastradh struct sun8i_crypto_task *, uint32_t);
21142356a74Sriastradh static int sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *,
212defaca02Sriastradh struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t);
21342356a74Sriastradh
214defaca02Sriastradh static int sun8i_crypto_submit(struct sun8i_crypto_softc *,
215defaca02Sriastradh struct sun8i_crypto_task *);
216defaca02Sriastradh
217defaca02Sriastradh static void sun8i_crypto_timeout(void *);
218defaca02Sriastradh static int sun8i_crypto_intr(void *);
219defaca02Sriastradh static void sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *);
220defaca02Sriastradh static void sun8i_crypto_worker(struct work *, void *);
221c92613e9Sriastradh
222c92613e9Sriastradh static bool sun8i_crypto_poll(struct sun8i_crypto_softc *, uint32_t *,
223c92613e9Sriastradh uint32_t *);
224c92613e9Sriastradh static bool sun8i_crypto_done(struct sun8i_crypto_softc *, uint32_t,
225c92613e9Sriastradh uint32_t, unsigned);
226f521c7d9Sriastradh static bool sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned,
227defaca02Sriastradh int);
228defaca02Sriastradh
229defaca02Sriastradh static int sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t,
23042356a74Sriastradh struct sun8i_crypto_buf *, int);
231defaca02Sriastradh static void sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t,
232defaca02Sriastradh struct sun8i_crypto_buf *);
233defaca02Sriastradh
234defaca02Sriastradh static void sun8i_crypto_rng_attach(struct sun8i_crypto_softc *);
235defaca02Sriastradh static void sun8i_crypto_rng_get(size_t, void *);
236defaca02Sriastradh static void sun8i_crypto_rng_done(struct sun8i_crypto_softc *,
237defaca02Sriastradh struct sun8i_crypto_task *, void *, int);
238defaca02Sriastradh
239c92613e9Sriastradh static bool sun8i_crypto_selftest(struct sun8i_crypto_softc *);
240defaca02Sriastradh static void sun8i_crypto_selftest_done(struct sun8i_crypto_softc *,
241defaca02Sriastradh struct sun8i_crypto_task *, void *, int);
242defaca02Sriastradh
243defaca02Sriastradh static void sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *);
244defaca02Sriastradh static int sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS);
245defaca02Sriastradh static void sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *,
246defaca02Sriastradh struct sun8i_crypto_task *, void *, int);
247defaca02Sriastradh
24886953edcSriastradh static void sun8i_crypto_register(struct sun8i_crypto_softc *);
24986953edcSriastradh static void sun8i_crypto_register1(struct sun8i_crypto_softc *, uint32_t);
25086953edcSriastradh static int sun8i_crypto_newsession(void *, uint32_t *,
25186953edcSriastradh struct cryptoini *);
252*fdf161e4Sriastradh static void sun8i_crypto_freesession(void *, uint64_t);
25386953edcSriastradh static u_int sun8i_crypto_ivlen(const struct cryptodesc *);
25486953edcSriastradh static int sun8i_crypto_process(void *, struct cryptop *, int);
25586953edcSriastradh static void sun8i_crypto_callback(struct sun8i_crypto_softc *,
25686953edcSriastradh struct sun8i_crypto_task *, void *, int);
25786953edcSriastradh
25886953edcSriastradh /*
25986953edcSriastradh * Probes
26086953edcSriastradh */
26186953edcSriastradh
26286953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, read,
26386953edcSriastradh "bus_size_t"/*reg*/,
26486953edcSriastradh "uint32_t"/*value*/);
26586953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, register, write,
26686953edcSriastradh "bus_size_t"/*reg*/,
26786953edcSriastradh "uint32_t"/*write*/);
26886953edcSriastradh
26986953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__success,
27086953edcSriastradh "struct sun8i_crypto_task *"/*task*/);
27186953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, ctor__failure,
27286953edcSriastradh "int"/*error*/);
27386953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, dtor,
27486953edcSriastradh "struct sun8i_crypto_task *"/*task*/);
27586953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, get,
27686953edcSriastradh "struct sun8i_crypto_task *"/*task*/);
27786953edcSriastradh SDT_PROBE_DEFINE1(sdt, sun8i_crypto, task, put,
27886953edcSriastradh "struct sun8i_crypto_task *"/*task*/);
27986953edcSriastradh
28086953edcSriastradh SDT_PROBE_DEFINE6(sdt, sun8i_crypto, task, load,
28186953edcSriastradh "struct sun8i_crypto_task *"/*task*/,
28286953edcSriastradh "uint32_t"/*tdqc*/,
28386953edcSriastradh "uint32_t"/*tdqs*/,
28486953edcSriastradh "uint32_t"/*tdqa*/,
28586953edcSriastradh "struct sun8i_crypto_taskdesc *"/*desc*/,
28686953edcSriastradh "int"/*error*/);
28786953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, task, misaligned,
28886953edcSriastradh "struct sun8i_crypto_task *"/*task*/,
28986953edcSriastradh "bus_addr_t"/*ds_addr*/,
29086953edcSriastradh "bus_size_t"/*ds_len*/);
29186953edcSriastradh SDT_PROBE_DEFINE2(sdt, sun8i_crypto, task, done,
29286953edcSriastradh "struct sun8i_crypto_task *"/*task*/,
29386953edcSriastradh "int"/*error*/);
29486953edcSriastradh
29586953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__failure,
29686953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
29786953edcSriastradh "struct sun8i_crypto_task *"/*task*/,
29886953edcSriastradh "int"/*error*/);
29986953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, submit__success,
30086953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
30186953edcSriastradh "struct sun8i_crypto_task *"/*task*/,
30286953edcSriastradh "unsigned"/*chan*/);
30386953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, intr,
30486953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
30586953edcSriastradh "uint32_t"/*isr*/,
30686953edcSriastradh "uint32_t"/*esr*/);
30786953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, engine, done,
30886953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
30986953edcSriastradh "unsigned"/*chan*/,
31086953edcSriastradh "int"/*error*/);
31186953edcSriastradh
31286953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, entry,
31386953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
31486953edcSriastradh "struct cryptop *"/*crp*/,
31586953edcSriastradh "int"/*hint*/);
31686953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, busy,
31786953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
31886953edcSriastradh "struct cryptop *"/*crp*/,
31986953edcSriastradh "int"/*hint*/);
32086953edcSriastradh SDT_PROBE_DEFINE4(sdt, sun8i_crypto, process, queued,
32186953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
32286953edcSriastradh "struct cryptop *"/*crp*/,
32386953edcSriastradh "int"/*hint*/,
32486953edcSriastradh "struct sun8i_crypto_task *"/*task*/);
32586953edcSriastradh SDT_PROBE_DEFINE3(sdt, sun8i_crypto, process, done,
32686953edcSriastradh "struct sun8i_crypto_softc *"/*sc*/,
32786953edcSriastradh "struct cryptop *"/*crp*/,
32886953edcSriastradh "int"/*error*/);
32986953edcSriastradh
330defaca02Sriastradh /*
331defaca02Sriastradh * Register access
332defaca02Sriastradh */
333defaca02Sriastradh
334defaca02Sriastradh static uint32_t
sun8i_crypto_read(struct sun8i_crypto_softc * sc,bus_size_t reg)33586953edcSriastradh sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_size_t reg)
336defaca02Sriastradh {
33786953edcSriastradh uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
33886953edcSriastradh
33986953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, register, read, reg, v);
34086953edcSriastradh return v;
341defaca02Sriastradh }
342defaca02Sriastradh
343defaca02Sriastradh static void
sun8i_crypto_write(struct sun8i_crypto_softc * sc,bus_size_t reg,uint32_t v)34486953edcSriastradh sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_size_t reg, uint32_t v)
345defaca02Sriastradh {
34686953edcSriastradh
34786953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, register, write, reg, v);
348defaca02Sriastradh bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v);
349defaca02Sriastradh }
350defaca02Sriastradh
351defaca02Sriastradh /*
352defaca02Sriastradh * Autoconf goo
353defaca02Sriastradh */
354defaca02Sriastradh
355defaca02Sriastradh CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc),
356defaca02Sriastradh sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL);
357defaca02Sriastradh
3580717bee5Sthorpej static const struct device_compatible_entry compat_data[] = {
3591b26e856Sbad { .compat = "allwinner,sun50i-a64-crypto",
3601b26e856Sbad .data = &sun50i_a64_crypto_config },
3611b26e856Sbad { .compat = "allwinner,sun50i-h5-crypto",
3621b26e856Sbad .data = &sun50i_h5_crypto_config },
3631b26e856Sbad { .compat = "allwinner,sun8i-h3-crypto",
3641b26e856Sbad .data = &sun8i_h3_crypto_config },
365f18cbf47Sthorpej DEVICE_COMPAT_EOL
366defaca02Sriastradh };
367defaca02Sriastradh
368defaca02Sriastradh static int
sun8i_crypto_match(device_t parent,cfdata_t cf,void * aux)369defaca02Sriastradh sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux)
370defaca02Sriastradh {
371defaca02Sriastradh const struct fdt_attach_args *const faa = aux;
372defaca02Sriastradh
3738e90f9edSthorpej return of_compatible_match(faa->faa_phandle, compat_data);
374defaca02Sriastradh }
375defaca02Sriastradh
376defaca02Sriastradh static void
sun8i_crypto_attach(device_t parent,device_t self,void * aux)377defaca02Sriastradh sun8i_crypto_attach(device_t parent, device_t self, void *aux)
378defaca02Sriastradh {
379defaca02Sriastradh struct sun8i_crypto_softc *const sc = device_private(self);
380defaca02Sriastradh const struct fdt_attach_args *const faa = aux;
381defaca02Sriastradh bus_addr_t addr;
382defaca02Sriastradh bus_size_t size;
383defaca02Sriastradh const int phandle = faa->faa_phandle;
384defaca02Sriastradh char intrstr[128];
385defaca02Sriastradh struct clk *clk;
386defaca02Sriastradh struct fdtbus_reset *rst;
3871b26e856Sbad u_int mod_rate;
388defaca02Sriastradh
389defaca02Sriastradh sc->sc_dev = self;
390defaca02Sriastradh sc->sc_dmat = faa->faa_dmat;
391defaca02Sriastradh sc->sc_bst = faa->faa_bst;
39242356a74Sriastradh sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task),
39347575044Sriastradh 0, 0, 0, "sun8icry", NULL, IPL_SOFTSERIAL,
39442356a74Sriastradh &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc);
3951b26e856Sbad sc->sc_cfg = of_compatible_lookup(phandle, compat_data)->data;
39647575044Sriastradh mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
3973c959c1cSriastradh mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_VM);
398defaca02Sriastradh callout_init(&sc->sc_timeout, CALLOUT_MPSAFE);
399defaca02Sriastradh callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc);
400defaca02Sriastradh if (workqueue_create(&sc->sc_wq, device_xname(self),
401defaca02Sriastradh &sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) {
402defaca02Sriastradh aprint_error(": couldn't create workqueue\n");
403defaca02Sriastradh return;
404defaca02Sriastradh }
405defaca02Sriastradh
40686953edcSriastradh /*
40786953edcSriastradh * Prime the pool with enough tasks that each channel can be
40886953edcSriastradh * busy with a task as we prepare another task for when it's
40986953edcSriastradh * done.
41086953edcSriastradh */
41186953edcSriastradh pool_cache_prime(sc->sc_taskpool, 2*SUN8I_CRYPTO_NCHAN);
41286953edcSriastradh
413defaca02Sriastradh /* Get and map device registers. */
414defaca02Sriastradh if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
415defaca02Sriastradh aprint_error(": couldn't get registers\n");
416defaca02Sriastradh return;
417defaca02Sriastradh }
418defaca02Sriastradh if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
419defaca02Sriastradh aprint_error(": couldn't map registers\n");
420defaca02Sriastradh return;
421defaca02Sriastradh }
422defaca02Sriastradh
423defaca02Sriastradh /* Get an interrupt handle. */
424defaca02Sriastradh if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
425defaca02Sriastradh aprint_error(": failed to decode interrupt\n");
426defaca02Sriastradh return;
427defaca02Sriastradh }
428defaca02Sriastradh
429defaca02Sriastradh /* Enable the bus clock. */
430defaca02Sriastradh if (fdtbus_clock_enable(phandle, "bus", true) != 0) {
431defaca02Sriastradh aprint_error(": couldn't enable bus clock\n");
432defaca02Sriastradh return;
433defaca02Sriastradh }
434defaca02Sriastradh
4351b26e856Sbad /* Get the module clock and set it. */
4361b26e856Sbad mod_rate = sc->sc_cfg->mod_rate;
437defaca02Sriastradh if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) {
438defaca02Sriastradh if (clk_enable(clk) != 0) {
439defaca02Sriastradh aprint_error(": couldn't enable CE clock\n");
440defaca02Sriastradh return;
441defaca02Sriastradh }
4421b26e856Sbad if (clk_set_rate(clk, mod_rate) != 0) {
4431b26e856Sbad aprint_error(": couldn't set CE clock to %d MHz\n",
4441b26e856Sbad mod_rate / (1000 * 1000));
445defaca02Sriastradh return;
446defaca02Sriastradh }
447defaca02Sriastradh }
448defaca02Sriastradh
449defaca02Sriastradh /* Get a reset handle if we need and try to deassert it. */
450defaca02Sriastradh if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) {
451defaca02Sriastradh if (fdtbus_reset_deassert(rst) != 0) {
452defaca02Sriastradh aprint_error(": couldn't de-assert reset\n");
453defaca02Sriastradh return;
454defaca02Sriastradh }
455defaca02Sriastradh }
456defaca02Sriastradh
457defaca02Sriastradh aprint_naive("\n");
458defaca02Sriastradh aprint_normal(": Crypto Engine\n");
459defaca02Sriastradh aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk));
460defaca02Sriastradh
461c92613e9Sriastradh /*
462c92613e9Sriastradh * Disable and clear interrupts. Start in polling mode for
463c92613e9Sriastradh * synchronous self-tests and the first RNG draw.
464c92613e9Sriastradh */
465defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0);
466defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0);
467c92613e9Sriastradh sc->sc_polling = true;
468defaca02Sriastradh
469defaca02Sriastradh /* Establish an interrupt handler. */
47074536a76Sjmcneill sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_VM,
47174536a76Sjmcneill FDT_INTR_MPSAFE, &sun8i_crypto_intr, sc, device_xname(self));
472defaca02Sriastradh if (sc->sc_ih == NULL) {
473defaca02Sriastradh aprint_error_dev(self, "failed to establish interrupt on %s\n",
474defaca02Sriastradh intrstr);
475defaca02Sriastradh return;
476defaca02Sriastradh }
477defaca02Sriastradh aprint_normal_dev(self, "interrupting on %s\n", intrstr);
478defaca02Sriastradh
479c92613e9Sriastradh /* Perform self-tests. If they fail, stop here. */
480c92613e9Sriastradh if (!sun8i_crypto_selftest(sc))
481c92613e9Sriastradh return;
482c92613e9Sriastradh
483c92613e9Sriastradh /*
484c92613e9Sriastradh * Set up the RNG. This will try to synchronously draw the
485c92613e9Sriastradh * first sample by polling, so do this before we establish
486c92613e9Sriastradh * the interrupt handler.
487c92613e9Sriastradh */
488defaca02Sriastradh sun8i_crypto_rng_attach(sc);
489defaca02Sriastradh
490c92613e9Sriastradh /*
491c92613e9Sriastradh * Self-test has passed and first RNG draw has finished. Use
492c92613e9Sriastradh * interrupts, not polling, for all subsequent tasks. Set this
493c92613e9Sriastradh * atomically in case the interrupt handler has fired -- can't
494c92613e9Sriastradh * be from us because we've kept ICR set to 0 to mask all
495c92613e9Sriastradh * interrupts, but in case the interrupt vector is shared.
496c92613e9Sriastradh */
49777c36046Sriastradh atomic_store_relaxed(&sc->sc_polling, false);
498c92613e9Sriastradh
499defaca02Sriastradh /* Attach the sysctl. */
500defaca02Sriastradh sun8i_crypto_sysctl_attach(sc);
501defaca02Sriastradh
50286953edcSriastradh /* Register opencrypto handlers. */
50386953edcSriastradh sun8i_crypto_register(sc);
504defaca02Sriastradh }
505defaca02Sriastradh
50642356a74Sriastradh static int
sun8i_crypto_task_ctor(void * cookie,void * vtask,int pflags)50742356a74Sriastradh sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags)
50842356a74Sriastradh {
50942356a74Sriastradh struct sun8i_crypto_softc *sc = cookie;
51042356a74Sriastradh struct sun8i_crypto_task *task = vtask;
51142356a74Sriastradh int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT;
51242356a74Sriastradh int error;
513defaca02Sriastradh
51442356a74Sriastradh /* Create a DMA buffer for the task descriptor. */
51542356a74Sriastradh error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc),
51642356a74Sriastradh &task->ct_descbuf, dmaflags);
51742356a74Sriastradh if (error)
51842356a74Sriastradh goto fail0;
51942356a74Sriastradh task->ct_desc = task->ct_descbuf.cb_kva;
52042356a74Sriastradh
52186953edcSriastradh /* Create DMA buffers for the IV and CTR. */
52286953edcSriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXIVBYTES,
52386953edcSriastradh &task->ct_ivbuf, dmaflags);
52486953edcSriastradh if (error)
52586953edcSriastradh goto fail1;
52686953edcSriastradh task->ct_iv = task->ct_ivbuf.cb_kva;
52786953edcSriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_MAXCTRBYTES,
52886953edcSriastradh &task->ct_ctrbuf, dmaflags);
52986953edcSriastradh if (error)
53086953edcSriastradh goto fail2;
53186953edcSriastradh task->ct_ctr = task->ct_ctrbuf.cb_kva;
53286953edcSriastradh
53342356a74Sriastradh /* Create a DMA map for the task descriptor and preload it. */
53442356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1,
53542356a74Sriastradh sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap);
53642356a74Sriastradh if (error)
53786953edcSriastradh goto fail3;
53842356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc,
53942356a74Sriastradh sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK);
54042356a74Sriastradh if (error)
54186953edcSriastradh goto fail4;
54242356a74Sriastradh
54342356a74Sriastradh /* Create DMA maps for the key, IV, and CTR. */
54442356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1,
54542356a74Sriastradh SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap);
54642356a74Sriastradh if (error)
54786953edcSriastradh goto fail5;
54842356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1,
54942356a74Sriastradh SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap);
55042356a74Sriastradh if (error)
55186953edcSriastradh goto fail6;
55242356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1,
55342356a74Sriastradh SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap);
55442356a74Sriastradh if (error)
55586953edcSriastradh goto fail7;
55642356a74Sriastradh
55742356a74Sriastradh /* Create DMA maps for the src and dst scatter/gather vectors. */
55842356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
55942356a74Sriastradh SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
56042356a74Sriastradh &task->ct_srcmap);
56142356a74Sriastradh if (error)
56286953edcSriastradh goto fail8;
56342356a74Sriastradh error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
56442356a74Sriastradh SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
56542356a74Sriastradh &task->ct_dstmap);
56642356a74Sriastradh if (error)
56786953edcSriastradh goto fail9;
56842356a74Sriastradh
56942356a74Sriastradh /* Success! */
57086953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, ctor__success, task);
57142356a74Sriastradh return 0;
57242356a74Sriastradh
57386953edcSriastradh fail10: __unused
57442356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
57586953edcSriastradh fail9: bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
57686953edcSriastradh fail8: bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
57786953edcSriastradh fail7: bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
57886953edcSriastradh fail6: bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
57986953edcSriastradh fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
58086953edcSriastradh fail4: bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
58186953edcSriastradh fail3: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
58286953edcSriastradh fail2: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
58342356a74Sriastradh fail1: sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
58486953edcSriastradh fail0: SDT_PROBE1(sdt, sun8i_crypto, task, ctor__failure, error);
58586953edcSriastradh return error;
58642356a74Sriastradh }
58742356a74Sriastradh
58842356a74Sriastradh static void
sun8i_crypto_task_dtor(void * cookie,void * vtask)58942356a74Sriastradh sun8i_crypto_task_dtor(void *cookie, void *vtask)
59042356a74Sriastradh {
59142356a74Sriastradh struct sun8i_crypto_softc *sc = cookie;
59242356a74Sriastradh struct sun8i_crypto_task *task = vtask;
59342356a74Sriastradh
59486953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, dtor, task);
59586953edcSriastradh
59642356a74Sriastradh /* XXX Zero the bounce buffers if there are any. */
59742356a74Sriastradh
59842356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
59942356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
60042356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
60142356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
60242356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
60342356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
60442356a74Sriastradh bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
60586953edcSriastradh sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXIVBYTES, &task->ct_ivbuf);
60686953edcSriastradh sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_MAXCTRBYTES, &task->ct_ctrbuf);
60742356a74Sriastradh sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
60842356a74Sriastradh }
60942356a74Sriastradh
61042356a74Sriastradh /*
61142356a74Sriastradh * sun8i_crypto_task_get(sc, callback, cookie, pflags)
61242356a74Sriastradh *
61342356a74Sriastradh * Allocate a task that will call callback(sc, task, cookie,
61442356a74Sriastradh * error) when done. pflags is PR_WAITOK or PR_NOWAIT; if
61542356a74Sriastradh * PR_NOWAIT, may fail and return NULL. No further allocation is
61642356a74Sriastradh * needed to submit the task if this succeeds (although task
61742356a74Sriastradh * submission may still fail if all channels are busy).
61842356a74Sriastradh */
619defaca02Sriastradh static struct sun8i_crypto_task *
sun8i_crypto_task_get(struct sun8i_crypto_softc * sc,void (* callback)(struct sun8i_crypto_softc *,struct sun8i_crypto_task *,void *,int),void * cookie,int pflags)620defaca02Sriastradh sun8i_crypto_task_get(struct sun8i_crypto_softc *sc,
621defaca02Sriastradh void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *,
622defaca02Sriastradh void *, int),
62342356a74Sriastradh void *cookie, int pflags)
624defaca02Sriastradh {
625defaca02Sriastradh struct sun8i_crypto_task *task;
626defaca02Sriastradh
62742356a74Sriastradh /* Allocate a task, or fail if we can't. */
62842356a74Sriastradh task = pool_cache_get(sc->sc_taskpool, pflags);
62942356a74Sriastradh if (task == NULL)
63086953edcSriastradh goto out;
631defaca02Sriastradh
63242356a74Sriastradh /* Set up flags and the callback. */
63342356a74Sriastradh task->ct_flags = 0;
634defaca02Sriastradh task->ct_callback = callback;
635defaca02Sriastradh task->ct_cookie = cookie;
63686953edcSriastradh
63786953edcSriastradh out: SDT_PROBE1(sdt, sun8i_crypto, task, get, task);
638defaca02Sriastradh return task;
639defaca02Sriastradh }
640defaca02Sriastradh
64142356a74Sriastradh /*
64242356a74Sriastradh * sun8i_crypto_task_invalid(sc, task, cookie, error)
64342356a74Sriastradh *
64442356a74Sriastradh * Callback for a task not currently in use, to detect errors.
64542356a74Sriastradh */
64642356a74Sriastradh static void
sun8i_crypto_task_invalid(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)64742356a74Sriastradh sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc,
64842356a74Sriastradh struct sun8i_crypto_task *task, void *cookie, int error)
64942356a74Sriastradh {
65042356a74Sriastradh void (*callback)(struct sun8i_crypto_softc *,
65142356a74Sriastradh struct sun8i_crypto_task *, void *, int) = cookie;
65242356a74Sriastradh
65342356a74Sriastradh panic("task for callback %p used after free", callback);
65442356a74Sriastradh }
65542356a74Sriastradh
65642356a74Sriastradh /*
65742356a74Sriastradh * sun8i_crypto_task_put(sc, task)
65842356a74Sriastradh *
65942356a74Sriastradh * Free a task obtained with sun8i_crypto_task_get.
66042356a74Sriastradh */
661defaca02Sriastradh static void
sun8i_crypto_task_put(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task)662defaca02Sriastradh sun8i_crypto_task_put(struct sun8i_crypto_softc *sc,
663defaca02Sriastradh struct sun8i_crypto_task *task)
664defaca02Sriastradh {
665defaca02Sriastradh
66686953edcSriastradh SDT_PROBE1(sdt, sun8i_crypto, task, put, task);
66786953edcSriastradh
66842356a74Sriastradh task->ct_cookie = task->ct_callback;
66942356a74Sriastradh task->ct_callback = &sun8i_crypto_task_invalid;
67042356a74Sriastradh pool_cache_put(sc->sc_taskpool, task);
671defaca02Sriastradh }
672defaca02Sriastradh
673defaca02Sriastradh /*
67442356a74Sriastradh * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa)
675defaca02Sriastradh *
67642356a74Sriastradh * Set up the task descriptor after the relevant DMA maps have
67742356a74Sriastradh * been loaded for a transfer of nbytes. bus_dmamap_sync matches
67842356a74Sriastradh * sun8i_crypto_chan_done. May fail if input is inadequately
67942356a74Sriastradh * aligned.
68042356a74Sriastradh *
68142356a74Sriastradh * XXX Teach this to support task chains.
682defaca02Sriastradh */
68342356a74Sriastradh static int
sun8i_crypto_task_load(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes,uint32_t tdqc,uint32_t tdqs,uint32_t tdqa)68442356a74Sriastradh sun8i_crypto_task_load(struct sun8i_crypto_softc *sc,
68542356a74Sriastradh struct sun8i_crypto_task *task, uint32_t nbytes,
68642356a74Sriastradh uint32_t tdqc, uint32_t tdqs, uint32_t tdqa)
687defaca02Sriastradh {
68842356a74Sriastradh struct sun8i_crypto_taskdesc *desc = task->ct_desc;
68942356a74Sriastradh int error;
690defaca02Sriastradh
69142356a74Sriastradh KASSERT(tdqs == 0 || tdqa == 0);
69242356a74Sriastradh KASSERT(nbytes % 4 == 0);
69342356a74Sriastradh
69442356a74Sriastradh memset(desc, 0, sizeof(*desc));
69542356a74Sriastradh
69686953edcSriastradh /* Always enable interrupt for the task. */
69786953edcSriastradh tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
69886953edcSriastradh
69942356a74Sriastradh desc->td_tdqc = htole32(tdqc);
70042356a74Sriastradh desc->td_tdqs = htole32(tdqs);
70142356a74Sriastradh desc->td_tdqa = htole32(tdqa);
70242356a74Sriastradh
70342356a74Sriastradh if (task->ct_flags & TASK_KEY) {
70442356a74Sriastradh bus_dmamap_t keymap = task->ct_keymap;
70542356a74Sriastradh KASSERT(keymap->dm_nsegs == 1);
70642356a74Sriastradh desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr);
70742356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, keymap, 0,
70842356a74Sriastradh keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
70942356a74Sriastradh }
71042356a74Sriastradh if (task->ct_flags & TASK_IV) {
71142356a74Sriastradh bus_dmamap_t ivmap = task->ct_ivmap;
71242356a74Sriastradh KASSERT(ivmap->dm_nsegs == 1);
71342356a74Sriastradh desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr);
71442356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, ivmap, 0,
71542356a74Sriastradh ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
71642356a74Sriastradh }
71742356a74Sriastradh if (task->ct_flags & TASK_CTR) {
71842356a74Sriastradh bus_dmamap_t ctrmap = task->ct_ctrmap;
71942356a74Sriastradh KASSERT(ctrmap->dm_nsegs == 1);
72042356a74Sriastradh desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr);
72142356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, ctrmap, 0,
72286953edcSriastradh ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREREAD);
723defaca02Sriastradh }
724defaca02Sriastradh
72542356a74Sriastradh if (task->ct_flags & TASK_BYTES)
72642356a74Sriastradh desc->td_datalen = htole32(nbytes);
72742356a74Sriastradh else
72842356a74Sriastradh desc->td_datalen = htole32(nbytes/4);
729defaca02Sriastradh
73042356a74Sriastradh if (task->ct_flags & TASK_SRC) {
73142356a74Sriastradh bus_dmamap_t srcmap = task->ct_srcmap;
73242356a74Sriastradh KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize);
73386953edcSriastradh error = sun8i_crypto_task_scatter(task, desc->td_src, srcmap,
73442356a74Sriastradh nbytes);
73542356a74Sriastradh if (error)
73642356a74Sriastradh return error;
73742356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes,
73842356a74Sriastradh BUS_DMASYNC_PREWRITE);
739defaca02Sriastradh }
740defaca02Sriastradh
74186953edcSriastradh error = sun8i_crypto_task_scatter(task, desc->td_dst, task->ct_dstmap,
74242356a74Sriastradh nbytes);
74342356a74Sriastradh if (error)
74486953edcSriastradh goto out;
74542356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
74642356a74Sriastradh BUS_DMASYNC_PREREAD);
747defaca02Sriastradh
74842356a74Sriastradh task->ct_nbytes = nbytes;
74942356a74Sriastradh
75042356a74Sriastradh /* Success! */
75186953edcSriastradh error = 0;
75286953edcSriastradh
75386953edcSriastradh out: SDT_PROBE6(sdt, sun8i_crypto, task, load,
75486953edcSriastradh task, tdqc, tdqs, tdqa, desc, error);
75586953edcSriastradh return error;
756defaca02Sriastradh }
757defaca02Sriastradh
75842356a74Sriastradh /*
75986953edcSriastradh * sun8i_crypto_task_scatter(task, adrlen, map, nbytes)
76042356a74Sriastradh *
76142356a74Sriastradh * Set up a task's scatter/gather vector -- src or dst -- with the
76242356a74Sriastradh * given DMA map for a transfer of nbytes. May fail if input is
76342356a74Sriastradh * inadequately aligned.
76442356a74Sriastradh */
76542356a74Sriastradh static int
sun8i_crypto_task_scatter(struct sun8i_crypto_task * task,struct sun8i_crypto_adrlen * adrlen,bus_dmamap_t map,uint32_t nbytes __diagused)76686953edcSriastradh sun8i_crypto_task_scatter(struct sun8i_crypto_task *task,
76786953edcSriastradh struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
76842356a74Sriastradh uint32_t nbytes __diagused)
769defaca02Sriastradh {
770defaca02Sriastradh uint32_t total __diagused = 0;
771defaca02Sriastradh unsigned i;
772defaca02Sriastradh
77386953edcSriastradh /*
77486953edcSriastradh * Verify that the alignment is correct and initialize the
77586953edcSriastradh * scatter/gather vector.
77686953edcSriastradh */
777defaca02Sriastradh KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS);
778defaca02Sriastradh for (i = 0; i < map->dm_nsegs; i++) {
77986953edcSriastradh if ((map->dm_segs[i].ds_addr % 4) |
78086953edcSriastradh (map->dm_segs[i].ds_len % 4)) {
78186953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, task, misaligned,
78286953edcSriastradh task,
78386953edcSriastradh map->dm_segs[i].ds_addr,
78486953edcSriastradh map->dm_segs[i].ds_len);
78586953edcSriastradh return EINVAL;
78686953edcSriastradh }
787defaca02Sriastradh KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX);
788defaca02Sriastradh KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total);
789defaca02Sriastradh adrlen[i].adr = htole32(map->dm_segs[i].ds_addr);
790defaca02Sriastradh adrlen[i].len = htole32(map->dm_segs[i].ds_len/4);
791defaca02Sriastradh total += map->dm_segs[i].ds_len;
792defaca02Sriastradh }
793defaca02Sriastradh
79486953edcSriastradh /* Set the remainder to zero. */
795defaca02Sriastradh for (; i < SUN8I_CRYPTO_MAXSEGS; i++) {
79686953edcSriastradh adrlen[i].adr = 0;
79786953edcSriastradh adrlen[i].len = 0;
798defaca02Sriastradh }
799defaca02Sriastradh
80042356a74Sriastradh /* Verify the total size matches the transfer length. */
80142356a74Sriastradh KASSERT(total == nbytes);
80242356a74Sriastradh
80342356a74Sriastradh /* Success! */
80442356a74Sriastradh return 0;
805defaca02Sriastradh }
806defaca02Sriastradh
807defaca02Sriastradh /*
80842356a74Sriastradh * sun8i_crypto_task_load_trng(task, nbytes)
809defaca02Sriastradh *
81042356a74Sriastradh * Set up the task descriptor for a transfer of nbytes from the
81142356a74Sriastradh * TRNG.
812defaca02Sriastradh */
813defaca02Sriastradh static int
sun8i_crypto_task_load_trng(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes)81442356a74Sriastradh sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc,
81542356a74Sriastradh struct sun8i_crypto_task *task, uint32_t nbytes)
816defaca02Sriastradh {
817defaca02Sriastradh uint32_t tdqc = 0;
818defaca02Sriastradh
81942356a74Sriastradh /* Caller must provide dst only. */
82042356a74Sriastradh KASSERT((task->ct_flags & TASK_KEY) == 0);
82142356a74Sriastradh KASSERT((task->ct_flags & TASK_IV) == 0);
82242356a74Sriastradh KASSERT((task->ct_flags & TASK_CTR) == 0);
82342356a74Sriastradh KASSERT((task->ct_flags & TASK_SRC) == 0);
824defaca02Sriastradh
825defaca02Sriastradh /* Set up the task descriptor queue control words. */
826defaca02Sriastradh tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG,
827defaca02Sriastradh SUN8I_CRYPTO_TDQC_METHOD);
828defaca02Sriastradh
82942356a74Sriastradh /* Fill in the descriptor. */
83042356a74Sriastradh return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0);
831defaca02Sriastradh }
832defaca02Sriastradh
833defaca02Sriastradh static int
sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,uint32_t nbytes,uint32_t keysize,uint32_t dir)83442356a74Sriastradh sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc,
835defaca02Sriastradh struct sun8i_crypto_task *task,
83642356a74Sriastradh uint32_t nbytes, uint32_t keysize, uint32_t dir)
837defaca02Sriastradh {
838defaca02Sriastradh uint32_t tdqc = 0, tdqs = 0;
839defaca02Sriastradh
84042356a74Sriastradh /* Caller must provide key, src, and dst only. */
84142356a74Sriastradh KASSERT(task->ct_flags & TASK_KEY);
84242356a74Sriastradh KASSERT((task->ct_flags & TASK_IV) == 0);
84342356a74Sriastradh KASSERT((task->ct_flags & TASK_CTR) == 0);
84442356a74Sriastradh KASSERT(task->ct_flags & TASK_SRC);
845defaca02Sriastradh
846defaca02Sriastradh /* Set up the task descriptor queue control word. */
847defaca02Sriastradh tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES,
848defaca02Sriastradh SUN8I_CRYPTO_TDQC_METHOD);
84986953edcSriastradh tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
85042356a74Sriastradh
85142356a74Sriastradh #ifdef DIAGNOSTIC
85242356a74Sriastradh switch (keysize) {
85342356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128:
85442356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16);
85542356a74Sriastradh break;
85642356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192:
85742356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24);
85842356a74Sriastradh break;
85942356a74Sriastradh case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256:
86042356a74Sriastradh KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32);
86142356a74Sriastradh break;
86242356a74Sriastradh }
86342356a74Sriastradh #endif
864defaca02Sriastradh
865defaca02Sriastradh /* Set up the symmetric control word. */
866defaca02Sriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
867defaca02Sriastradh SUN8I_CRYPTO_TDQS_SKEY_SELECT);
868defaca02Sriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB,
869defaca02Sriastradh SUN8I_CRYPTO_TDQS_OP_MODE);
87042356a74Sriastradh tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
871defaca02Sriastradh
87242356a74Sriastradh /* Fill in the descriptor. */
87342356a74Sriastradh return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0);
874defaca02Sriastradh }
875defaca02Sriastradh
87642356a74Sriastradh /*
87742356a74Sriastradh * sun8i_crypto_submit(sc, task)
87842356a74Sriastradh *
87942356a74Sriastradh * Submit a task to the crypto engine after it has been loaded
88042356a74Sriastradh * with sun8i_crypto_task_load. On success, guarantees to
88142356a74Sriastradh * eventually call the task's callback.
88242356a74Sriastradh */
883defaca02Sriastradh static int
sun8i_crypto_submit(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task)884defaca02Sriastradh sun8i_crypto_submit(struct sun8i_crypto_softc *sc,
885defaca02Sriastradh struct sun8i_crypto_task *task)
886defaca02Sriastradh {
887defaca02Sriastradh unsigned i, retries = 0;
888defaca02Sriastradh uint32_t icr;
889defaca02Sriastradh int error = 0;
890defaca02Sriastradh
891defaca02Sriastradh /* One at a time at the device registers, please. */
892defaca02Sriastradh mutex_enter(&sc->sc_lock);
893defaca02Sriastradh
894defaca02Sriastradh /* Find a channel. */
895defaca02Sriastradh for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
896defaca02Sriastradh if (sc->sc_chan[i].cc_task == NULL)
897defaca02Sriastradh break;
898defaca02Sriastradh }
899defaca02Sriastradh if (i == SUN8I_CRYPTO_NCHAN) {
900defaca02Sriastradh device_printf(sc->sc_dev, "no free channels\n");
901defaca02Sriastradh error = ERESTART;
902defaca02Sriastradh goto out;
903defaca02Sriastradh }
904defaca02Sriastradh
905defaca02Sriastradh /*
906defaca02Sriastradh * Set the channel id. Caller is responsible for setting up
907defaca02Sriastradh * all other parts of the descriptor.
908defaca02Sriastradh */
909defaca02Sriastradh task->ct_desc->td_cid = htole32(i);
910defaca02Sriastradh
91142356a74Sriastradh /*
91242356a74Sriastradh * Prepare to send the descriptor to the device by DMA.
91342356a74Sriastradh * Matches POSTWRITE in sun8i_crypto_chan_done.
91442356a74Sriastradh */
91542356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
916defaca02Sriastradh sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE);
917defaca02Sriastradh
918defaca02Sriastradh /* Confirm we're ready to go. */
919defaca02Sriastradh if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) {
920defaca02Sriastradh device_printf(sc->sc_dev, "TLR not clear\n");
921defaca02Sriastradh error = EIO;
922defaca02Sriastradh goto out;
923defaca02Sriastradh }
924defaca02Sriastradh
925c92613e9Sriastradh /*
926c92613e9Sriastradh * Enable interrupts for this channel, unless we're still
927c92613e9Sriastradh * polling.
928c92613e9Sriastradh */
929c92613e9Sriastradh if (!sc->sc_polling) {
930defaca02Sriastradh icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
931defaca02Sriastradh icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
932defaca02Sriastradh SUN8I_CRYPTO_ICR_INTR_EN);
933defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
934c92613e9Sriastradh }
935defaca02Sriastradh
936defaca02Sriastradh /* Set the task descriptor queue address. */
937defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ,
93842356a74Sriastradh task->ct_descmap->dm_segs[0].ds_addr);
939defaca02Sriastradh
940defaca02Sriastradh /* Notify the engine to load it, and wait for acknowledgement. */
941defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD);
942defaca02Sriastradh while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD)
943defaca02Sriastradh {
944defaca02Sriastradh /*
945defaca02Sriastradh * XXX Timeout pulled from arse. Is it even important
946defaca02Sriastradh * to wait here?
947defaca02Sriastradh */
948defaca02Sriastradh if (++retries == 1000) {
949defaca02Sriastradh device_printf(sc->sc_dev, "TLR didn't clear: %08x\n",
950defaca02Sriastradh sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR));
951defaca02Sriastradh /*
952defaca02Sriastradh * Hope it clears eventually; if not, we'll
953defaca02Sriastradh * time out.
954defaca02Sriastradh */
955defaca02Sriastradh break;
956defaca02Sriastradh }
957defaca02Sriastradh DELAY(1);
958defaca02Sriastradh }
959defaca02Sriastradh
96042356a74Sriastradh /*
96142356a74Sriastradh * Loaded up and ready to go. Start a timer ticking if it's
962c92613e9Sriastradh * not already and we're not polling.
96342356a74Sriastradh */
964defaca02Sriastradh sc->sc_chan[i].cc_task = task;
9652d8f3966Smaxv sc->sc_chan[i].cc_starttime = getticks();
966c92613e9Sriastradh if (!sc->sc_polling && !callout_pending(&sc->sc_timeout))
967defaca02Sriastradh callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
968defaca02Sriastradh
969defaca02Sriastradh out: /* Done! */
97086953edcSriastradh if (error)
97186953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, submit__failure,
97286953edcSriastradh sc, task, error);
97386953edcSriastradh else
97486953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, submit__success,
97586953edcSriastradh sc, task, i);
976defaca02Sriastradh mutex_exit(&sc->sc_lock);
977defaca02Sriastradh return error;
978defaca02Sriastradh }
979defaca02Sriastradh
98042356a74Sriastradh /*
98142356a74Sriastradh * sun8i_crypto_timeout(cookie)
98242356a74Sriastradh *
98342356a74Sriastradh * Timeout handler. Schedules work in a thread to cancel all
98442356a74Sriastradh * pending tasks that were started long enough ago we're bored of
9853c959c1cSriastradh * waiting for them.
98642356a74Sriastradh */
987defaca02Sriastradh static void
sun8i_crypto_timeout(void * cookie)988defaca02Sriastradh sun8i_crypto_timeout(void *cookie)
989defaca02Sriastradh {
990defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie;
991defaca02Sriastradh
9923c959c1cSriastradh mutex_enter(&sc->sc_intr_lock);
993defaca02Sriastradh sun8i_crypto_schedule_worker(sc);
9943c959c1cSriastradh mutex_exit(&sc->sc_intr_lock);
995defaca02Sriastradh }
996defaca02Sriastradh
99742356a74Sriastradh /*
99842356a74Sriastradh * sun8i_crypto_intr(cookie)
99942356a74Sriastradh *
100042356a74Sriastradh * Device interrupt handler. Find what channels have completed,
100142356a74Sriastradh * whether with success or with failure, and schedule work in
100242356a74Sriastradh * thread context to invoke the appropriate callbacks.
100342356a74Sriastradh */
1004defaca02Sriastradh static int
sun8i_crypto_intr(void * cookie)1005defaca02Sriastradh sun8i_crypto_intr(void *cookie)
1006defaca02Sriastradh {
1007defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie;
1008c92613e9Sriastradh uint32_t done, esr;
1009c92613e9Sriastradh
1010c92613e9Sriastradh if (atomic_load_relaxed(&sc->sc_polling) ||
1011c92613e9Sriastradh !sun8i_crypto_poll(sc, &done, &esr))
1012c92613e9Sriastradh return 0; /* not ours */
1013defaca02Sriastradh
10143c959c1cSriastradh mutex_enter(&sc->sc_intr_lock);
1015defaca02Sriastradh sun8i_crypto_schedule_worker(sc);
1016c92613e9Sriastradh sc->sc_done |= done;
1017defaca02Sriastradh sc->sc_esr |= esr;
10183c959c1cSriastradh mutex_exit(&sc->sc_intr_lock);
1019defaca02Sriastradh
1020c92613e9Sriastradh return 1;
1021defaca02Sriastradh }
1022defaca02Sriastradh
102342356a74Sriastradh /*
102442356a74Sriastradh * sun8i_crypto_schedule_worker(sc)
102542356a74Sriastradh *
102642356a74Sriastradh * Ensure that crypto engine thread context work to invoke task
102742356a74Sriastradh * callbacks will run promptly. Idempotent.
102842356a74Sriastradh */
1029defaca02Sriastradh static void
sun8i_crypto_schedule_worker(struct sun8i_crypto_softc * sc)1030defaca02Sriastradh sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc)
1031defaca02Sriastradh {
1032defaca02Sriastradh
10333c959c1cSriastradh KASSERT(mutex_owned(&sc->sc_intr_lock));
1034defaca02Sriastradh
1035defaca02Sriastradh /* Start the worker if necessary. */
1036defaca02Sriastradh if (!sc->sc_work_pending) {
1037defaca02Sriastradh workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL);
1038defaca02Sriastradh sc->sc_work_pending = true;
1039defaca02Sriastradh }
1040defaca02Sriastradh }
1041defaca02Sriastradh
104242356a74Sriastradh /*
104342356a74Sriastradh * sun8i_crypto_worker(wk, cookie)
104442356a74Sriastradh *
104542356a74Sriastradh * Thread-context worker: Invoke all task callbacks for which the
104642356a74Sriastradh * device has notified us of completion or for which we gave up
104742356a74Sriastradh * waiting.
104842356a74Sriastradh */
1049defaca02Sriastradh static void
sun8i_crypto_worker(struct work * wk,void * cookie)1050defaca02Sriastradh sun8i_crypto_worker(struct work *wk, void *cookie)
1051defaca02Sriastradh {
1052defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie;
1053c92613e9Sriastradh uint32_t done, esr;
1054defaca02Sriastradh
1055defaca02Sriastradh /*
10563c959c1cSriastradh * Under the interrupt lock, acknowledge our work and claim the
10573c959c1cSriastradh * done mask and error status.
1058defaca02Sriastradh */
10593c959c1cSriastradh mutex_enter(&sc->sc_intr_lock);
1060defaca02Sriastradh KASSERT(sc->sc_work_pending);
1061defaca02Sriastradh sc->sc_work_pending = false;
1062defaca02Sriastradh done = sc->sc_done;
1063defaca02Sriastradh esr = sc->sc_esr;
1064defaca02Sriastradh sc->sc_done = 0;
1065defaca02Sriastradh sc->sc_esr = 0;
10663c959c1cSriastradh mutex_exit(&sc->sc_intr_lock);
1067defaca02Sriastradh
1068c92613e9Sriastradh /*
1069c92613e9Sriastradh * If we cleared any channels, it is time to allow opencrypto
1070c92613e9Sriastradh * to issue new operations. Asymmetric operations (which we
1071c92613e9Sriastradh * don't support, at the moment, but we could) and symmetric
1072c92613e9Sriastradh * operations (which we do) use the same task channels, so we
1073c92613e9Sriastradh * unblock both kinds.
1074c92613e9Sriastradh */
1075c92613e9Sriastradh if (sun8i_crypto_done(sc, done, esr, getticks())) {
1076c92613e9Sriastradh crypto_unblock(sc->sc_opencrypto.co_driverid,
1077c92613e9Sriastradh CRYPTO_SYMQ|CRYPTO_ASYMQ);
1078c92613e9Sriastradh }
1079c92613e9Sriastradh }
1080c92613e9Sriastradh
1081c92613e9Sriastradh /*
1082c92613e9Sriastradh * sun8i_crypto_poll(sc, &done, &esr)
1083c92613e9Sriastradh *
1084c92613e9Sriastradh * Poll for completion. Sets done and esr to the mask of done
1085c92613e9Sriastradh * channels and error channels. Returns true if anything was
1086c92613e9Sriastradh * done or failed.
1087c92613e9Sriastradh */
1088c92613e9Sriastradh static bool
sun8i_crypto_poll(struct sun8i_crypto_softc * sc,uint32_t * donep,uint32_t * esrp)1089c92613e9Sriastradh sun8i_crypto_poll(struct sun8i_crypto_softc *sc,
1090c92613e9Sriastradh uint32_t *donep, uint32_t *esrp)
1091c92613e9Sriastradh {
1092c92613e9Sriastradh uint32_t isr, esr;
1093c92613e9Sriastradh
1094c92613e9Sriastradh /*
1095c92613e9Sriastradh * Get and acknowledge the interrupts and error status.
1096c92613e9Sriastradh *
1097c92613e9Sriastradh * XXX Data sheet says the error status register is read-only,
1098c92613e9Sriastradh * but then advises writing 1 to bit x1xx (keysram access error
1099c92613e9Sriastradh * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it. What do?
1100c92613e9Sriastradh */
1101c92613e9Sriastradh isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR);
1102c92613e9Sriastradh esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR);
1103c92613e9Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr);
1104c92613e9Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr);
1105c92613e9Sriastradh
1106c92613e9Sriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, intr, sc, isr, esr);
1107c92613e9Sriastradh
1108c92613e9Sriastradh *donep = __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE);
1109c92613e9Sriastradh *esrp = esr;
1110c92613e9Sriastradh
1111c92613e9Sriastradh return *donep || *esrp;
1112c92613e9Sriastradh }
1113c92613e9Sriastradh
1114c92613e9Sriastradh /*
1115c92613e9Sriastradh * sun8i_crypto_done(sc, done, esr, now)
1116c92613e9Sriastradh *
1117c92613e9Sriastradh * Invoke all task callbacks for the channels in done or esr, or
1118c92613e9Sriastradh * for which we gave up waiting, according to the time `now'.
1119c92613e9Sriastradh * Returns true if any channels completed or timed out.
1120c92613e9Sriastradh */
1121c92613e9Sriastradh static bool
sun8i_crypto_done(struct sun8i_crypto_softc * sc,uint32_t done,uint32_t esr,unsigned now)1122c92613e9Sriastradh sun8i_crypto_done(struct sun8i_crypto_softc *sc, uint32_t done, uint32_t esr,
1123c92613e9Sriastradh unsigned now)
1124c92613e9Sriastradh {
1125c92613e9Sriastradh uint32_t esr_chan;
1126c92613e9Sriastradh unsigned i;
1127c92613e9Sriastradh bool anydone = false;
1128c92613e9Sriastradh bool schedtimeout = false;
1129c92613e9Sriastradh int error;
1130defaca02Sriastradh
11313c959c1cSriastradh /* Under the lock, process the channels. */
11323c959c1cSriastradh mutex_enter(&sc->sc_lock);
1133defaca02Sriastradh for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
113444c3f65aSriastradh /* Check whether the channel is done. */
1135defaca02Sriastradh if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) {
113644c3f65aSriastradh /* Nope. Do we have a task to time out? */
11373c959c1cSriastradh if (sc->sc_chan[i].cc_task != NULL) {
11383c959c1cSriastradh if (now - sc->sc_chan[i].cc_starttime >=
11393c959c1cSriastradh SUN8I_CRYPTO_TIMEOUT) {
1140c92613e9Sriastradh anydone |= sun8i_crypto_chan_done(sc,
11413c959c1cSriastradh i, ETIMEDOUT);
11423c959c1cSriastradh } else {
11433c959c1cSriastradh schedtimeout = true;
11443c959c1cSriastradh }
11453c959c1cSriastradh }
1146defaca02Sriastradh continue;
1147defaca02Sriastradh }
114844c3f65aSriastradh
114944c3f65aSriastradh /* Channel is done. Interpret the error if any. */
1150defaca02Sriastradh esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i));
1151defaca02Sriastradh if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) {
1152defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:"
1153defaca02Sriastradh " alg not supported\n", i);
1154defaca02Sriastradh error = ENODEV;
1155defaca02Sriastradh } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) {
1156defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:"
1157defaca02Sriastradh " data length error\n", i);
1158defaca02Sriastradh error = EIO; /* XXX */
1159defaca02Sriastradh } else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) {
1160defaca02Sriastradh device_printf(sc->sc_dev, "channel %u:"
1161defaca02Sriastradh " key sram error\n", i);
1162defaca02Sriastradh error = EIO; /* XXX */
1163defaca02Sriastradh } else if (esr_chan != 0) {
1164defaca02Sriastradh error = EIO; /* generic I/O error */
1165defaca02Sriastradh } else {
1166defaca02Sriastradh error = 0;
1167defaca02Sriastradh }
1168defaca02Sriastradh
116944c3f65aSriastradh /*
117044c3f65aSriastradh * Notify the task of completion. May release the lock
117144c3f65aSriastradh * to invoke a callback.
117244c3f65aSriastradh */
1173c92613e9Sriastradh anydone |= sun8i_crypto_chan_done(sc, i, error);
1174defaca02Sriastradh }
1175defaca02Sriastradh mutex_exit(&sc->sc_lock);
1176f521c7d9Sriastradh
1177f521c7d9Sriastradh /*
11783c959c1cSriastradh * If there are tasks still pending, make sure there's a
11793c959c1cSriastradh * timeout scheduled for them. If the callout is already
11803c959c1cSriastradh * pending, it will take another pass through here to time some
11813c959c1cSriastradh * things out and schedule a new timeout.
11823c959c1cSriastradh */
11833c959c1cSriastradh if (schedtimeout && !callout_pending(&sc->sc_timeout))
11843c959c1cSriastradh callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);
11853c959c1cSriastradh
1186c92613e9Sriastradh return anydone;
1187defaca02Sriastradh }
1188defaca02Sriastradh
118942356a74Sriastradh /*
119042356a74Sriastradh * sun8i_crypto_chan_done(sc, i, error)
119142356a74Sriastradh *
119242356a74Sriastradh * Notify the callback for the task on channel i, if there is one,
119342356a74Sriastradh * of the specified error, or 0 for success.
119442356a74Sriastradh */
1195f521c7d9Sriastradh static bool
sun8i_crypto_chan_done(struct sun8i_crypto_softc * sc,unsigned i,int error)1196defaca02Sriastradh sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error)
1197defaca02Sriastradh {
1198defaca02Sriastradh struct sun8i_crypto_task *task;
119942356a74Sriastradh uint32_t nbytes;
1200defaca02Sriastradh uint32_t icr;
1201defaca02Sriastradh
1202defaca02Sriastradh KASSERT(mutex_owned(&sc->sc_lock));
1203defaca02Sriastradh
120486953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, engine, done, sc, i, error);
120586953edcSriastradh
1206defaca02Sriastradh /* Claim the task if there is one; bail if not. */
1207defaca02Sriastradh if ((task = sc->sc_chan[i].cc_task) == NULL) {
1208defaca02Sriastradh device_printf(sc->sc_dev, "channel %u: no task but error=%d\n",
1209defaca02Sriastradh i, error);
1210f521c7d9Sriastradh /* We did not clear a channel. */
1211f521c7d9Sriastradh return false;
1212defaca02Sriastradh }
1213defaca02Sriastradh sc->sc_chan[i].cc_task = NULL;
1214defaca02Sriastradh
1215defaca02Sriastradh /* Disable interrupts on this channel. */
1216defaca02Sriastradh icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
1217defaca02Sriastradh icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
1218defaca02Sriastradh SUN8I_CRYPTO_ICR_INTR_EN);
1219defaca02Sriastradh sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);
1220defaca02Sriastradh
122142356a74Sriastradh /*
122242356a74Sriastradh * Finished sending the descriptor to the device by DMA.
122342356a74Sriastradh * Matches PREWRITE in sun8i_crypto_task_submit.
122442356a74Sriastradh */
122542356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
1226defaca02Sriastradh sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE);
1227defaca02Sriastradh
122842356a74Sriastradh /*
122942356a74Sriastradh * Finished with all the other bits of DMA too. Matches
123042356a74Sriastradh * sun8i_crypto_task_load.
123142356a74Sriastradh */
123242356a74Sriastradh nbytes = task->ct_nbytes;
123342356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
123442356a74Sriastradh BUS_DMASYNC_POSTREAD);
123542356a74Sriastradh if (task->ct_flags & TASK_SRC)
123642356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes,
123742356a74Sriastradh BUS_DMASYNC_POSTWRITE);
123842356a74Sriastradh if (task->ct_flags & TASK_CTR)
123942356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0,
124086953edcSriastradh task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTREAD);
124142356a74Sriastradh if (task->ct_flags & TASK_IV)
124242356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0,
124342356a74Sriastradh task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
124442356a74Sriastradh if (task->ct_flags & TASK_KEY)
124542356a74Sriastradh /* XXX Can we zero the bounce buffer if there is one? */
124642356a74Sriastradh bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0,
124742356a74Sriastradh task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
124842356a74Sriastradh
1249defaca02Sriastradh /* Temporarily release the lock to invoke the callback. */
1250defaca02Sriastradh mutex_exit(&sc->sc_lock);
125186953edcSriastradh SDT_PROBE2(sdt, sun8i_crypto, task, done, task, error);
1252defaca02Sriastradh (*task->ct_callback)(sc, task, task->ct_cookie, error);
1253defaca02Sriastradh mutex_enter(&sc->sc_lock);
1254f521c7d9Sriastradh
1255f521c7d9Sriastradh /* We cleared a channel. */
1256f521c7d9Sriastradh return true;
1257defaca02Sriastradh }
1258defaca02Sriastradh
1259defaca02Sriastradh /*
126042356a74Sriastradh * sun8i_crypto_allocbuf(sc, size, buf, dmaflags)
126142356a74Sriastradh *
126242356a74Sriastradh * Allocate a single-segment DMA-safe buffer and map it into KVA.
126342356a74Sriastradh * May fail if dmaflags is BUS_DMA_NOWAIT.
1264defaca02Sriastradh */
1265defaca02Sriastradh static int
sun8i_crypto_allocbuf(struct sun8i_crypto_softc * sc,size_t size,struct sun8i_crypto_buf * buf,int dmaflags)1266defaca02Sriastradh sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size,
126742356a74Sriastradh struct sun8i_crypto_buf *buf, int dmaflags)
1268defaca02Sriastradh {
1269defaca02Sriastradh int error;
1270defaca02Sriastradh
1271defaca02Sriastradh /* Allocate a DMA-safe buffer. */
127286953edcSriastradh error = bus_dmamem_alloc(sc->sc_dmat, size, sizeof(uint32_t), 0,
127386953edcSriastradh buf->cb_seg, __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
1274defaca02Sriastradh if (error)
1275defaca02Sriastradh goto fail0;
1276defaca02Sriastradh
1277defaca02Sriastradh /* Map the buffer into kernel virtual address space. */
1278defaca02Sriastradh error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs,
127942356a74Sriastradh size, &buf->cb_kva, dmaflags);
1280defaca02Sriastradh if (error)
1281defaca02Sriastradh goto fail1;
1282defaca02Sriastradh
1283defaca02Sriastradh /* Success! */
1284defaca02Sriastradh return 0;
1285defaca02Sriastradh
128642356a74Sriastradh fail2: __unused
128742356a74Sriastradh bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
1288defaca02Sriastradh fail1: bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
1289defaca02Sriastradh fail0: return error;
1290defaca02Sriastradh }
1291defaca02Sriastradh
129242356a74Sriastradh /*
129342356a74Sriastradh * sun8i_crypto_freebuf(sc, buf)
129442356a74Sriastradh *
129542356a74Sriastradh * Unmap buf and free it.
129642356a74Sriastradh */
1297defaca02Sriastradh static void
sun8i_crypto_freebuf(struct sun8i_crypto_softc * sc,size_t size,struct sun8i_crypto_buf * buf)1298defaca02Sriastradh sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size,
1299defaca02Sriastradh struct sun8i_crypto_buf *buf)
1300defaca02Sriastradh {
1301defaca02Sriastradh
1302defaca02Sriastradh bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
1303defaca02Sriastradh bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
1304defaca02Sriastradh }
1305defaca02Sriastradh
1306defaca02Sriastradh /*
130742356a74Sriastradh * sun8i_crypto_rng_attach(sc)
130842356a74Sriastradh *
130942356a74Sriastradh * Attach an rndsource for the crypto engine's TRNG.
1310defaca02Sriastradh */
1311defaca02Sriastradh static void
sun8i_crypto_rng_attach(struct sun8i_crypto_softc * sc)1312defaca02Sriastradh sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc)
1313defaca02Sriastradh {
1314defaca02Sriastradh device_t self = sc->sc_dev;
1315defaca02Sriastradh struct sun8i_crypto_rng *rng = &sc->sc_rng;
131642356a74Sriastradh struct sun8i_crypto_task *task;
1317c92613e9Sriastradh unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
1318c92613e9Sriastradh uint32_t done, esr;
1319defaca02Sriastradh int error;
1320defaca02Sriastradh
1321defaca02Sriastradh /* Preallocate a buffer to reuse. */
132242356a74Sriastradh error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf,
132342356a74Sriastradh BUS_DMA_WAITOK);
132486953edcSriastradh if (error) {
132586953edcSriastradh aprint_error_dev(self, "failed to allocate RNG buffer: %d\n",
132686953edcSriastradh error);
1327defaca02Sriastradh goto fail0;
132886953edcSriastradh }
1329defaca02Sriastradh
1330defaca02Sriastradh /* Create a task to reuse. */
133142356a74Sriastradh task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done,
133242356a74Sriastradh rng, PR_WAITOK);
133342356a74Sriastradh if (rng->cr_task == NULL) {
133486953edcSriastradh aprint_error_dev(self, "failed to allocate RNG task\n");
133542356a74Sriastradh error = ENOMEM;
1336defaca02Sriastradh goto fail1;
133742356a74Sriastradh }
133842356a74Sriastradh
133942356a74Sriastradh /* Preload the destination map. */
134042356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
134142356a74Sriastradh rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
134286953edcSriastradh if (error) {
134386953edcSriastradh aprint_error_dev(self, "failed to load RNG buffer: %d\n",
134486953edcSriastradh error);
134542356a74Sriastradh goto fail2;
134686953edcSriastradh }
1347defaca02Sriastradh
1348defaca02Sriastradh /*
134904604289Sriastradh * Attach the rndsource. This will trigger an initial call to
135004604289Sriastradh * it since we have RND_FLAG_HASCB.
1351defaca02Sriastradh */
1352defaca02Sriastradh rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc);
1353defaca02Sriastradh rnd_attach_source(&rng->cr_rndsource, device_xname(self),
135404604289Sriastradh RND_TYPE_RNG,
1355defaca02Sriastradh RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB);
1356defaca02Sriastradh
1357c92613e9Sriastradh /*
1358c92613e9Sriastradh * Poll for the first call to the RNG to complete. If not done
1359c92613e9Sriastradh * after the timeout, force a timeout.
1360c92613e9Sriastradh */
1361c92613e9Sriastradh for (; rng->cr_pending && timo --> 0; DELAY(1000)) {
1362c92613e9Sriastradh if (sun8i_crypto_poll(sc, &done, &esr))
1363c92613e9Sriastradh (void)sun8i_crypto_done(sc, done, esr, getticks());
1364c92613e9Sriastradh }
1365c92613e9Sriastradh if (rng->cr_pending) {
1366c92613e9Sriastradh (void)sun8i_crypto_done(sc, 0, 0,
1367c92613e9Sriastradh (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
1368c92613e9Sriastradh KASSERT(!rng->cr_pending);
1369c92613e9Sriastradh }
1370c92613e9Sriastradh
1371defaca02Sriastradh return;
1372defaca02Sriastradh
137342356a74Sriastradh fail3: __unused
137442356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
137542356a74Sriastradh fail2: sun8i_crypto_task_put(sc, task);
1376defaca02Sriastradh fail1: sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf);
137786953edcSriastradh fail0: return;
1378defaca02Sriastradh }
1379defaca02Sriastradh
138042356a74Sriastradh /*
138142356a74Sriastradh * sun8i_crypto_rng_get(nbytes, cookie)
138242356a74Sriastradh *
138342356a74Sriastradh * On-demand rndsource callback: try to gather nbytes of entropy
138442356a74Sriastradh * and enter them into the pool ASAP.
138542356a74Sriastradh */
1386defaca02Sriastradh static void
sun8i_crypto_rng_get(size_t nbytes,void * cookie)1387defaca02Sriastradh sun8i_crypto_rng_get(size_t nbytes, void *cookie)
1388defaca02Sriastradh {
1389defaca02Sriastradh struct sun8i_crypto_softc *sc = cookie;
1390defaca02Sriastradh struct sun8i_crypto_rng *rng = &sc->sc_rng;
139142356a74Sriastradh struct sun8i_crypto_task *task = rng->cr_task;
1392defaca02Sriastradh bool pending;
1393defaca02Sriastradh int error;
1394defaca02Sriastradh
1395defaca02Sriastradh /*
1396defaca02Sriastradh * Test and set the RNG-pending flag. If it's already in
1397defaca02Sriastradh * progress, nothing to do here.
1398defaca02Sriastradh */
1399defaca02Sriastradh mutex_enter(&sc->sc_lock);
1400defaca02Sriastradh pending = rng->cr_pending;
1401defaca02Sriastradh rng->cr_pending = true;
1402defaca02Sriastradh mutex_exit(&sc->sc_lock);
1403defaca02Sriastradh if (pending)
1404defaca02Sriastradh return;
1405defaca02Sriastradh
140642356a74Sriastradh /* Load the task descriptor. */
140742356a74Sriastradh error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
140842356a74Sriastradh if (error)
140942356a74Sriastradh goto fail;
1410defaca02Sriastradh
141142356a74Sriastradh /* Submit! */
141242356a74Sriastradh error = sun8i_crypto_submit(sc, task);
1413defaca02Sriastradh if (error)
1414defaca02Sriastradh goto fail;
1415defaca02Sriastradh
1416defaca02Sriastradh /* All done! */
1417defaca02Sriastradh return;
1418defaca02Sriastradh
1419defaca02Sriastradh fail: mutex_enter(&sc->sc_lock);
1420defaca02Sriastradh rng->cr_pending = false;
1421defaca02Sriastradh mutex_exit(&sc->sc_lock);
1422defaca02Sriastradh }
1423defaca02Sriastradh
1424defaca02Sriastradh static void
sun8i_crypto_rng_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1425defaca02Sriastradh sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc,
1426defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error)
1427defaca02Sriastradh {
1428defaca02Sriastradh struct sun8i_crypto_rng *rng = cookie;
1429defaca02Sriastradh uint8_t *buf = rng->cr_buf.cb_kva;
1430defaca02Sriastradh uint32_t entropybits;
1431defaca02Sriastradh
1432defaca02Sriastradh KASSERT(rng == &sc->sc_rng);
1433defaca02Sriastradh
1434defaca02Sriastradh /* If anything went wrong, forget about it. */
1435defaca02Sriastradh if (error)
1436defaca02Sriastradh goto out;
1437defaca02Sriastradh
1438defaca02Sriastradh /*
1439defaca02Sriastradh * This TRNG has quite low entropy at best. But if it fails a
1440defaca02Sriastradh * repeated output test, then assume it's busted.
1441defaca02Sriastradh */
1442f9ae7d17Sriastradh CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY);
1443f9ae7d17Sriastradh entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY;
1444defaca02Sriastradh if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2,
1445defaca02Sriastradh SUN8I_CRYPTO_RNGBYTES/2)) {
1446defaca02Sriastradh device_printf(sc->sc_dev, "failed repeated output test\n");
1447defaca02Sriastradh entropybits = 0;
1448defaca02Sriastradh }
1449defaca02Sriastradh
1450560536a1Sriastradh /*
1451560536a1Sriastradh * Actually we don't believe in any of the entropy until this
1452560536a1Sriastradh * device has had more scrutiny.
1453560536a1Sriastradh */
1454560536a1Sriastradh entropybits = 0;
1455560536a1Sriastradh
1456defaca02Sriastradh /* Success! Enter and erase the data. */
1457defaca02Sriastradh rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES,
1458defaca02Sriastradh entropybits);
1459defaca02Sriastradh explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES);
1460defaca02Sriastradh
1461defaca02Sriastradh out: /* Done -- clear the RNG-pending flag. */
1462defaca02Sriastradh mutex_enter(&sc->sc_lock);
1463defaca02Sriastradh rng->cr_pending = false;
1464defaca02Sriastradh mutex_exit(&sc->sc_lock);
1465defaca02Sriastradh }
1466defaca02Sriastradh
1467defaca02Sriastradh /*
1468defaca02Sriastradh * Self-test
1469defaca02Sriastradh */
1470defaca02Sriastradh
1471dc50386fSriastradh static const uint8_t selftest_input[16];
1472dc50386fSriastradh static const uint8_t selftest_key[16];
1473dc50386fSriastradh static const uint8_t selftest_output[16] = {
1474defaca02Sriastradh 0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b,
1475defaca02Sriastradh 0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e,
1476defaca02Sriastradh };
1477defaca02Sriastradh
1478c92613e9Sriastradh static bool
sun8i_crypto_selftest(struct sun8i_crypto_softc * sc)1479c92613e9Sriastradh sun8i_crypto_selftest(struct sun8i_crypto_softc *sc)
1480defaca02Sriastradh {
148142356a74Sriastradh const size_t keybytes = sizeof selftest_key;
148242356a74Sriastradh const size_t nbytes = sizeof selftest_input;
1483defaca02Sriastradh struct sun8i_crypto_selftest *selftest = &sc->sc_selftest;
148442356a74Sriastradh struct sun8i_crypto_task *task;
1485c92613e9Sriastradh unsigned timo = hztoms(SUN8I_CRYPTO_TIMEOUT);
1486c92613e9Sriastradh uint32_t done, esr;
1487defaca02Sriastradh int error;
1488defaca02Sriastradh
1489defaca02Sriastradh CTASSERT(sizeof selftest_input == sizeof selftest_output);
1490defaca02Sriastradh
1491c92613e9Sriastradh selftest->cs_pending = true;
1492c92613e9Sriastradh
1493defaca02Sriastradh /* Allocate an input buffer. */
149442356a74Sriastradh error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in,
149542356a74Sriastradh BUS_DMA_WAITOK);
1496defaca02Sriastradh if (error)
1497defaca02Sriastradh goto fail0;
1498defaca02Sriastradh
1499defaca02Sriastradh /* Allocate a key buffer. */
150042356a74Sriastradh error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key,
150142356a74Sriastradh BUS_DMA_WAITOK);
1502defaca02Sriastradh if (error)
1503defaca02Sriastradh goto fail1;
1504defaca02Sriastradh
1505defaca02Sriastradh /* Allocate an output buffer. */
150642356a74Sriastradh error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out,
150742356a74Sriastradh BUS_DMA_WAITOK);
1508defaca02Sriastradh if (error)
1509defaca02Sriastradh goto fail2;
1510defaca02Sriastradh
1511defaca02Sriastradh /* Allocate a task descriptor. */
151242356a74Sriastradh task = selftest->cs_task = sun8i_crypto_task_get(sc,
151342356a74Sriastradh sun8i_crypto_selftest_done, selftest, PR_WAITOK);
151442356a74Sriastradh if (selftest->cs_task == NULL) {
151542356a74Sriastradh error = ENOMEM;
1516defaca02Sriastradh goto fail3;
151742356a74Sriastradh }
1518defaca02Sriastradh
1519defaca02Sriastradh /* Copy the input and key into their buffers. */
152042356a74Sriastradh memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes);
152142356a74Sriastradh memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes);
1522defaca02Sriastradh
152342356a74Sriastradh /* Load the key, src, and dst for DMA transfers. */
152442356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
152542356a74Sriastradh selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK);
1526defaca02Sriastradh if (error)
1527defaca02Sriastradh goto fail4;
152842356a74Sriastradh task->ct_flags |= TASK_KEY;
152942356a74Sriastradh
153042356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap,
153142356a74Sriastradh selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
153242356a74Sriastradh if (error)
153342356a74Sriastradh goto fail5;
153442356a74Sriastradh task->ct_flags |= TASK_SRC;
153542356a74Sriastradh
153642356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
153742356a74Sriastradh selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
153842356a74Sriastradh if (error)
153942356a74Sriastradh goto fail6;
154042356a74Sriastradh
154142356a74Sriastradh /* Set up the task descriptor. */
154286953edcSriastradh error = sun8i_crypto_task_load_aesecb(sc, task, nbytes,
154342356a74Sriastradh SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC);
154486953edcSriastradh if (error)
154586953edcSriastradh goto fail7;
154642356a74Sriastradh
154742356a74Sriastradh /* Submit! */
154842356a74Sriastradh error = sun8i_crypto_submit(sc, task);
154942356a74Sriastradh if (error)
155042356a74Sriastradh goto fail7;
1551defaca02Sriastradh device_printf(sc->sc_dev, "AES-128 self-test initiated\n");
1552defaca02Sriastradh
1553c92613e9Sriastradh /*
1554c92613e9Sriastradh * Poll for completion. If not done after the timeout, force a
1555c92613e9Sriastradh * timeout.
1556c92613e9Sriastradh */
1557c92613e9Sriastradh for (; selftest->cs_pending && timo --> 0; DELAY(1000)) {
1558c92613e9Sriastradh if (sun8i_crypto_poll(sc, &done, &esr))
1559c92613e9Sriastradh (void)sun8i_crypto_done(sc, done, esr, getticks());
1560c92613e9Sriastradh }
1561c92613e9Sriastradh if (selftest->cs_pending) {
1562c92613e9Sriastradh (void)sun8i_crypto_done(sc, 0, 0,
1563c92613e9Sriastradh (unsigned)getticks() + SUN8I_CRYPTO_TIMEOUT);
1564c92613e9Sriastradh KASSERT(!selftest->cs_pending);
1565c92613e9Sriastradh }
1566c92613e9Sriastradh
1567c92613e9Sriastradh return selftest->cs_passed;
1568defaca02Sriastradh
156942356a74Sriastradh fail7: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
157042356a74Sriastradh fail6: bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
157142356a74Sriastradh fail5: bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
157242356a74Sriastradh fail4: sun8i_crypto_task_put(sc, task);
157342356a74Sriastradh fail3: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
157442356a74Sriastradh fail2: sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
157542356a74Sriastradh fail1: sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
1576c92613e9Sriastradh fail0: aprint_error_dev(sc->sc_dev, "failed to run self-test, error=%d\n",
1577c92613e9Sriastradh error);
1578c92613e9Sriastradh selftest->cs_pending = false;
1579c92613e9Sriastradh return false;
1580defaca02Sriastradh }
1581defaca02Sriastradh
1582dc50386fSriastradh static bool
sun8i_crypto_selftest_check(struct sun8i_crypto_softc * sc,const char * title,size_t n,const void * expected,const void * actual)1583dc50386fSriastradh sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title,
1584dc50386fSriastradh size_t n, const void *expected, const void *actual)
1585dc50386fSriastradh {
1586dc50386fSriastradh const uint8_t *e = expected;
1587dc50386fSriastradh const uint8_t *a = actual;
1588dc50386fSriastradh size_t i;
1589dc50386fSriastradh
1590dc50386fSriastradh if (memcmp(e, a, n) == 0)
1591dc50386fSriastradh return true;
1592dc50386fSriastradh
1593dc50386fSriastradh device_printf(sc->sc_dev, "self-test: %s\n", title);
1594dc50386fSriastradh printf("expected: ");
1595dc50386fSriastradh for (i = 0; i < n; i++)
1596dc50386fSriastradh printf("%02hhx", e[i]);
1597dc50386fSriastradh printf("\n");
1598dc50386fSriastradh printf("actual: ");
1599dc50386fSriastradh for (i = 0; i < n; i++)
1600dc50386fSriastradh printf("%02hhx", a[i]);
1601dc50386fSriastradh printf("\n");
1602dc50386fSriastradh return false;
1603dc50386fSriastradh }
1604dc50386fSriastradh
1605defaca02Sriastradh static void
sun8i_crypto_selftest_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1606defaca02Sriastradh sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc,
1607defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error)
1608defaca02Sriastradh {
160942356a74Sriastradh const size_t keybytes = sizeof selftest_key;
161042356a74Sriastradh const size_t nbytes = sizeof selftest_input;
1611defaca02Sriastradh struct sun8i_crypto_selftest *selftest = cookie;
1612defaca02Sriastradh bool ok = true;
1613defaca02Sriastradh
1614defaca02Sriastradh KASSERT(selftest == &sc->sc_selftest);
1615defaca02Sriastradh
1616dc50386fSriastradh /* If anything went wrong, fail now. */
1617defaca02Sriastradh if (error) {
1618defaca02Sriastradh device_printf(sc->sc_dev, "self-test error=%d\n", error);
1619defaca02Sriastradh goto out;
1620defaca02Sriastradh }
1621defaca02Sriastradh
1622dc50386fSriastradh /*
1623dc50386fSriastradh * Verify the input and key weren't clobbered, and verify the
1624dc50386fSriastradh * output matches what we expect.
1625dc50386fSriastradh */
162642356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes,
162742356a74Sriastradh selftest_input, selftest->cs_in.cb_kva);
162842356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes,
162942356a74Sriastradh selftest_key, selftest->cs_key.cb_kva);
163042356a74Sriastradh ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes,
163142356a74Sriastradh selftest_output, selftest->cs_out.cb_kva);
1632defaca02Sriastradh
1633c92613e9Sriastradh selftest->cs_passed = ok;
1634defaca02Sriastradh if (ok)
1635defaca02Sriastradh device_printf(sc->sc_dev, "AES-128 self-test passed\n");
1636defaca02Sriastradh
163742356a74Sriastradh out: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
163842356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
163942356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
164042356a74Sriastradh sun8i_crypto_task_put(sc, task);
164142356a74Sriastradh sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
164242356a74Sriastradh sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
164342356a74Sriastradh sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
1644c92613e9Sriastradh selftest->cs_pending = false;
1645defaca02Sriastradh }
1646defaca02Sriastradh
1647defaca02Sriastradh /*
1648defaca02Sriastradh * Sysctl for testing
1649defaca02Sriastradh */
1650defaca02Sriastradh
1651defaca02Sriastradh struct sun8i_crypto_userreq {
1652defaca02Sriastradh kmutex_t cu_lock;
1653defaca02Sriastradh kcondvar_t cu_cv;
1654defaca02Sriastradh size_t cu_size;
1655defaca02Sriastradh struct sun8i_crypto_buf cu_buf;
1656defaca02Sriastradh struct sun8i_crypto_task *cu_task;
1657defaca02Sriastradh int cu_error;
1658defaca02Sriastradh bool cu_done;
1659defaca02Sriastradh bool cu_cancel;
1660defaca02Sriastradh };
1661defaca02Sriastradh
1662defaca02Sriastradh static void
sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc * sc)1663defaca02Sriastradh sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc)
1664defaca02Sriastradh {
1665defaca02Sriastradh struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl;
1666defaca02Sriastradh int error;
1667defaca02Sriastradh
166844c3f65aSriastradh /* hw.sun8icryptoN (node) */
1669defaca02Sriastradh error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node,
1670defaca02Sriastradh CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev),
1671defaca02Sriastradh SYSCTL_DESCR("sun8i crypto engine knobs"),
1672defaca02Sriastradh NULL, 0, NULL, 0,
1673defaca02Sriastradh CTL_HW, CTL_CREATE, CTL_EOL);
1674defaca02Sriastradh if (error) {
1675defaca02Sriastradh aprint_error_dev(sc->sc_dev,
1676defaca02Sriastradh "failed to set up sysctl hw.%s: %d\n",
1677defaca02Sriastradh device_xname(sc->sc_dev), error);
1678defaca02Sriastradh return;
1679defaca02Sriastradh }
1680defaca02Sriastradh
1681f9ae7d17Sriastradh /* hw.sun8icryptoN.rng (`struct', 4096-byte array) */
1682827c081cSriastradh sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node,
1683defaca02Sriastradh CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT,
1684f9ae7d17Sriastradh "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"),
1685defaca02Sriastradh &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL);
1686defaca02Sriastradh if (error) {
1687defaca02Sriastradh aprint_error_dev(sc->sc_dev,
1688defaca02Sriastradh "failed to set up sysctl hw.%s.rng: %d\n",
1689defaca02Sriastradh device_xname(sc->sc_dev), error);
1690defaca02Sriastradh return;
1691defaca02Sriastradh }
1692defaca02Sriastradh }
1693defaca02Sriastradh
1694defaca02Sriastradh static int
sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)1695defaca02Sriastradh sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)
1696defaca02Sriastradh {
1697defaca02Sriastradh struct sysctlnode node = *rnode;
1698defaca02Sriastradh struct sun8i_crypto_softc *sc = node.sysctl_data;
1699defaca02Sriastradh struct sun8i_crypto_userreq *req;
170042356a74Sriastradh struct sun8i_crypto_task *task;
1701defaca02Sriastradh size_t size;
1702defaca02Sriastradh int error;
1703defaca02Sriastradh
1704defaca02Sriastradh /* If oldp == NULL, the caller wants to learn the size. */
1705defaca02Sriastradh if (oldp == NULL) {
1706f9ae7d17Sriastradh *oldlenp = 4096;
1707defaca02Sriastradh return 0;
1708defaca02Sriastradh }
1709defaca02Sriastradh
1710843b783fSriastradh /* Truncate to 4096 bytes. */
1711843b783fSriastradh size = MIN(4096, *oldlenp);
1712defaca02Sriastradh if (size == 0)
1713defaca02Sriastradh return 0; /* nothing to do */
1714defaca02Sriastradh
1715defaca02Sriastradh /* Allocate a request context. */
1716defaca02Sriastradh req = kmem_alloc(sizeof(*req), KM_NOSLEEP);
1717defaca02Sriastradh if (req == NULL)
1718defaca02Sriastradh return ENOMEM;
1719defaca02Sriastradh
1720defaca02Sriastradh /* Initialize the request context. */
1721defaca02Sriastradh mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE);
1722defaca02Sriastradh cv_init(&req->cu_cv, "sun8isy");
1723defaca02Sriastradh req->cu_size = size;
1724defaca02Sriastradh req->cu_error = EIO;
1725defaca02Sriastradh req->cu_done = false;
1726defaca02Sriastradh req->cu_cancel = false;
1727defaca02Sriastradh
1728defaca02Sriastradh /* Allocate a buffer for the RNG output. */
172942356a74Sriastradh error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT);
1730defaca02Sriastradh if (error)
1731defaca02Sriastradh goto out0;
1732defaca02Sriastradh
1733defaca02Sriastradh /* Allocate a task. */
173442356a74Sriastradh task = req->cu_task = sun8i_crypto_task_get(sc,
173542356a74Sriastradh sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT);
173642356a74Sriastradh if (task == NULL) {
173742356a74Sriastradh error = ENOMEM;
1738defaca02Sriastradh goto out1;
173942356a74Sriastradh }
1740defaca02Sriastradh
1741defaca02Sriastradh /* Set the task up for TRNG to our buffer. */
174242356a74Sriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
174342356a74Sriastradh req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
174442356a74Sriastradh if (error)
174542356a74Sriastradh goto out2;
174642356a74Sriastradh error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
174742356a74Sriastradh if (error)
174842356a74Sriastradh goto out3;
1749defaca02Sriastradh
175042356a74Sriastradh /* Submit! */
175142356a74Sriastradh error = sun8i_crypto_submit(sc, task);
1752443fde03Sriastradh if (error) {
175342356a74Sriastradh /* Make sure we don't restart the syscall -- just fail. */
1754443fde03Sriastradh if (error == ERESTART)
1755443fde03Sriastradh error = EBUSY;
175642356a74Sriastradh goto out3;
1757443fde03Sriastradh }
1758defaca02Sriastradh
1759defaca02Sriastradh /* Wait for the request to complete. */
1760defaca02Sriastradh mutex_enter(&req->cu_lock);
1761defaca02Sriastradh while (!req->cu_done) {
1762defaca02Sriastradh error = cv_wait_sig(&req->cu_cv, &req->cu_lock);
1763defaca02Sriastradh if (error) {
1764defaca02Sriastradh /*
17654a726746Sriastradh * If we finished while waiting to acquire the
17664a726746Sriastradh * lock, ignore the error and just return now.
17674a726746Sriastradh * Otherwise, notify the callback that it has
17684a726746Sriastradh * to clean up after us.
1769defaca02Sriastradh */
17704a726746Sriastradh if (req->cu_done)
17714a726746Sriastradh error = 0;
17724a726746Sriastradh else
1773defaca02Sriastradh req->cu_cancel = true;
1774defaca02Sriastradh break;
1775defaca02Sriastradh }
1776defaca02Sriastradh }
1777defaca02Sriastradh mutex_exit(&req->cu_lock);
1778defaca02Sriastradh
1779defaca02Sriastradh /*
1780defaca02Sriastradh * Return early on error from cv_wait_sig, which means
1781defaca02Sriastradh * interruption; the callback will clean up instead.
1782defaca02Sriastradh */
1783defaca02Sriastradh if (error)
1784defaca02Sriastradh return error;
1785defaca02Sriastradh
1786defaca02Sriastradh /* Check for error from the device. */
1787defaca02Sriastradh error = req->cu_error;
1788defaca02Sriastradh if (error)
178942356a74Sriastradh goto out3;
1790defaca02Sriastradh
1791defaca02Sriastradh /* Copy out the data. */
1792defaca02Sriastradh node.sysctl_data = req->cu_buf.cb_kva;
1793defaca02Sriastradh node.sysctl_size = size;
1794defaca02Sriastradh error = sysctl_lookup(SYSCTLFN_CALL(&node));
1795defaca02Sriastradh
1796defaca02Sriastradh /* Clear the buffer. */
1797defaca02Sriastradh explicit_memset(req->cu_buf.cb_kva, 0, size);
1798defaca02Sriastradh
17994a726746Sriastradh /* Clean up. */
180042356a74Sriastradh out3: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
180142356a74Sriastradh out2: sun8i_crypto_task_put(sc, task);
1802defaca02Sriastradh out1: sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
1803defaca02Sriastradh out0: cv_destroy(&req->cu_cv);
1804defaca02Sriastradh mutex_destroy(&req->cu_lock);
1805845298d6Sriastradh kmem_free(req, sizeof(*req));
1806defaca02Sriastradh return error;
1807defaca02Sriastradh }
1808defaca02Sriastradh
1809defaca02Sriastradh static void
sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)1810defaca02Sriastradh sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc,
1811defaca02Sriastradh struct sun8i_crypto_task *task, void *cookie, int error)
1812defaca02Sriastradh {
1813defaca02Sriastradh struct sun8i_crypto_userreq *req = cookie;
1814defaca02Sriastradh bool cancel;
1815defaca02Sriastradh
1816defaca02Sriastradh /*
1817defaca02Sriastradh * Notify the waiting thread of the error, and find out whether
1818defaca02Sriastradh * that thread cancelled.
1819defaca02Sriastradh */
1820defaca02Sriastradh mutex_enter(&req->cu_lock);
1821defaca02Sriastradh cancel = req->cu_cancel;
1822defaca02Sriastradh req->cu_error = error;
1823defaca02Sriastradh req->cu_done = true;
1824defaca02Sriastradh cv_broadcast(&req->cu_cv);
1825defaca02Sriastradh mutex_exit(&req->cu_lock);
1826defaca02Sriastradh
1827defaca02Sriastradh /*
1828defaca02Sriastradh * If it wasn't cancelled, we're done -- the main thread will
1829defaca02Sriastradh * clean up after itself.
1830defaca02Sriastradh */
1831defaca02Sriastradh if (!cancel)
1832defaca02Sriastradh return;
1833defaca02Sriastradh
18344a726746Sriastradh /* Clean up after the main thread cancelled. */
183542356a74Sriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
183642356a74Sriastradh sun8i_crypto_task_put(sc, task);
1837defaca02Sriastradh sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
1838defaca02Sriastradh cv_destroy(&req->cu_cv);
1839defaca02Sriastradh mutex_destroy(&req->cu_lock);
1840845298d6Sriastradh kmem_free(req, sizeof(*req));
1841defaca02Sriastradh }
184286953edcSriastradh
184386953edcSriastradh /*
184486953edcSriastradh * sun8i_crypto_register(sc)
184586953edcSriastradh *
184686953edcSriastradh * Register opencrypto algorithms supported by the crypto engine.
184786953edcSriastradh */
184886953edcSriastradh static void
sun8i_crypto_register(struct sun8i_crypto_softc * sc)184986953edcSriastradh sun8i_crypto_register(struct sun8i_crypto_softc *sc)
185086953edcSriastradh {
185186953edcSriastradh struct sun8i_crypto_opencrypto *co = &sc->sc_opencrypto;
185286953edcSriastradh
185386953edcSriastradh co->co_driverid = crypto_get_driverid(0);
185486953edcSriastradh if (co->co_driverid == (uint32_t)-1) {
185586953edcSriastradh aprint_error_dev(sc->sc_dev,
185686953edcSriastradh "failed to register crypto driver\n");
185786953edcSriastradh return;
185886953edcSriastradh }
185986953edcSriastradh
186086953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_CBC);
186186953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_CTR);
186286953edcSriastradh #ifdef CRYPTO_AES_ECB
186386953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_ECB);
186486953edcSriastradh #endif
186586953edcSriastradh #ifdef CRYPTO_AES_XTS
186686953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_AES_XTS);
186786953edcSriastradh #endif
186886953edcSriastradh #ifdef CRYPTO_DES_CBC
186986953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_DES_CBC);
187086953edcSriastradh #endif
187186953edcSriastradh #ifdef CRYPTO_DES_ECB
187286953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_DES_ECB);
187386953edcSriastradh #endif
187486953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_3DES_CBC);
187586953edcSriastradh #ifdef CRYPTO_3DES_ECB
187686953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_3DES_ECB);
187786953edcSriastradh #endif
187886953edcSriastradh
187986953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_MD5);
188086953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA1);
188186953edcSriastradh #ifdef CRYPTO_SHA224
188286953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA224);
188386953edcSriastradh #endif
188486953edcSriastradh #ifdef CRYPTO_SHA256
188586953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA256);
188686953edcSriastradh #endif
188786953edcSriastradh
188886953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA1_HMAC);
188986953edcSriastradh sun8i_crypto_register1(sc, CRYPTO_SHA2_256_HMAC);
189086953edcSriastradh
189186953edcSriastradh //sun8i_crypto_kregister(sc, CRK_MOD_EXP); /* XXX unclear */
189286953edcSriastradh }
189386953edcSriastradh
189486953edcSriastradh /*
189586953edcSriastradh * sun8i_crypto_register1(sc, alg)
189686953edcSriastradh *
189786953edcSriastradh * Register support for one algorithm alg using
189886953edcSriastradh * sun8i_crypto_newsession/freesession/process.
189986953edcSriastradh */
190086953edcSriastradh static void
sun8i_crypto_register1(struct sun8i_crypto_softc * sc,uint32_t alg)190186953edcSriastradh sun8i_crypto_register1(struct sun8i_crypto_softc *sc, uint32_t alg)
190286953edcSriastradh {
190386953edcSriastradh
190486953edcSriastradh crypto_register(sc->sc_opencrypto.co_driverid, alg, 0, 0,
190586953edcSriastradh sun8i_crypto_newsession,
190686953edcSriastradh sun8i_crypto_freesession,
190786953edcSriastradh sun8i_crypto_process,
190886953edcSriastradh sc);
190986953edcSriastradh }
191086953edcSriastradh
191186953edcSriastradh /*
191286953edcSriastradh * sun8i_crypto_newsession(cookie, sidp, cri)
191386953edcSriastradh *
191486953edcSriastradh * Called by opencrypto to allocate a new session. We don't keep
191586953edcSriastradh * track of sessions, since there are no persistent keys in the
191686953edcSriastradh * hardware that we take advantage of, so this only validates the
1917f521c7d9Sriastradh * crypto operations and returns a dummy session id of 1.
191886953edcSriastradh */
191986953edcSriastradh static int
sun8i_crypto_newsession(void * cookie,uint32_t * sidp,struct cryptoini * cri)192086953edcSriastradh sun8i_crypto_newsession(void *cookie, uint32_t *sidp, struct cryptoini *cri)
192186953edcSriastradh {
192286953edcSriastradh
192386953edcSriastradh /* No composition of operations is supported here. */
192486953edcSriastradh if (cri->cri_next)
192586953edcSriastradh return EINVAL;
192686953edcSriastradh
192786953edcSriastradh /*
192886953edcSriastradh * No variation of rounds is supported here. (XXX Unused and
1929f521c7d9Sriastradh * unimplemented in opencrypto(9) altogether?)
193086953edcSriastradh */
193186953edcSriastradh if (cri->cri_rnd)
193286953edcSriastradh return EINVAL;
193386953edcSriastradh
193486953edcSriastradh /*
193586953edcSriastradh * Validate per-algorithm key length.
193686953edcSriastradh *
193786953edcSriastradh * XXX Does opencrypto(9) do this internally?
193886953edcSriastradh */
193986953edcSriastradh switch (cri->cri_alg) {
194086953edcSriastradh case CRYPTO_MD5:
194186953edcSriastradh case CRYPTO_SHA1:
194286953edcSriastradh #ifdef CRYPTO_SHA224
194386953edcSriastradh case CRYPTO_SHA224:
194486953edcSriastradh #endif
194586953edcSriastradh #ifdef CRYPTO_SHA256
194686953edcSriastradh case CRYPTO_SHA256:
194786953edcSriastradh #endif
194886953edcSriastradh if (cri->cri_klen)
194986953edcSriastradh return EINVAL;
195086953edcSriastradh break;
195186953edcSriastradh case CRYPTO_AES_CBC:
195286953edcSriastradh #ifdef CRYPTO_AES_ECB
195386953edcSriastradh case CRYPTO_AES_ECB:
195486953edcSriastradh #endif
195586953edcSriastradh switch (cri->cri_klen) {
195686953edcSriastradh case 128:
195786953edcSriastradh case 192:
195886953edcSriastradh case 256:
195986953edcSriastradh break;
196086953edcSriastradh default:
196186953edcSriastradh return EINVAL;
196286953edcSriastradh }
196386953edcSriastradh break;
196486953edcSriastradh case CRYPTO_AES_CTR:
196586953edcSriastradh /*
196686953edcSriastradh * opencrypto `AES-CTR' takes four bytes of the input
196786953edcSriastradh * block as the last four bytes of the key, for reasons
196886953edcSriastradh * that are not entirely clear.
196986953edcSriastradh */
197086953edcSriastradh switch (cri->cri_klen) {
197186953edcSriastradh case 128 + 32:
197286953edcSriastradh case 192 + 32:
197386953edcSriastradh case 256 + 32:
197486953edcSriastradh break;
197586953edcSriastradh default:
197686953edcSriastradh return EINVAL;
197786953edcSriastradh }
197886953edcSriastradh break;
197986953edcSriastradh #ifdef CRYPTO_AES_XTS
198086953edcSriastradh case CRYPTO_AES_XTS:
198186953edcSriastradh switch (cri->cri_klen) {
198286953edcSriastradh case 256:
198386953edcSriastradh case 384:
198486953edcSriastradh case 512:
198586953edcSriastradh break;
198686953edcSriastradh default:
198786953edcSriastradh return EINVAL;
198886953edcSriastradh }
198986953edcSriastradh break;
199086953edcSriastradh #endif
199186953edcSriastradh case CRYPTO_DES_CBC:
199286953edcSriastradh #ifdef CRYPTO_DES_ECB
199386953edcSriastradh case CRYPTO_DES_ECB:
199486953edcSriastradh #endif
199586953edcSriastradh switch (cri->cri_klen) {
199686953edcSriastradh case 64:
199786953edcSriastradh break;
199886953edcSriastradh default:
199986953edcSriastradh return EINVAL;
200086953edcSriastradh }
200186953edcSriastradh break;
200286953edcSriastradh case CRYPTO_3DES_CBC:
200386953edcSriastradh #ifdef CRYPTO_3DES_ECB
200486953edcSriastradh case CRYPTO_3DES_ECB:
200586953edcSriastradh #endif
200686953edcSriastradh switch (cri->cri_klen) {
200786953edcSriastradh case 192:
200886953edcSriastradh break;
200986953edcSriastradh default:
201086953edcSriastradh return EINVAL;
201186953edcSriastradh }
201286953edcSriastradh break;
201386953edcSriastradh case CRYPTO_SHA1_HMAC:
201486953edcSriastradh /*
201586953edcSriastradh * XXX Unclear what the length limit is, but since HMAC
201686953edcSriastradh * behaves qualitatively different for a key of at
201786953edcSriastradh * least the full block size -- and is generally best
201886953edcSriastradh * to use with half the block size -- let's limit it to
201986953edcSriastradh * one block.
202086953edcSriastradh */
202186953edcSriastradh if (cri->cri_klen % 8)
202286953edcSriastradh return EINVAL;
202386953edcSriastradh if (cri->cri_klen > 512)
202486953edcSriastradh return EINVAL;
202586953edcSriastradh break;
202686953edcSriastradh case CRYPTO_SHA2_256_HMAC:
202786953edcSriastradh if (cri->cri_klen % 8)
202886953edcSriastradh return EINVAL;
202986953edcSriastradh if (cri->cri_klen > 512)
203086953edcSriastradh return EINVAL;
203186953edcSriastradh break;
203286953edcSriastradh default:
203386953edcSriastradh panic("unsupported algorithm %d", cri->cri_alg);
203486953edcSriastradh }
203586953edcSriastradh
203686953edcSriastradh KASSERT(cri->cri_klen % 8 == 0);
203786953edcSriastradh
203886953edcSriastradh /* Success! */
203986953edcSriastradh *sidp = 1;
204086953edcSriastradh return 0;
204186953edcSriastradh }
204286953edcSriastradh
204386953edcSriastradh /*
204486953edcSriastradh * sun8i_crypto_freesession(cookie, dsid)
204586953edcSriastradh *
204686953edcSriastradh * Called by opencrypto to free a session. We don't keep track of
204786953edcSriastradh * sessions, since there are no persistent keys in the hardware
204886953edcSriastradh * that we take advantage of, so this is a no-op.
204986953edcSriastradh *
205086953edcSriastradh * Note: dsid is actually a 64-bit quantity containing both the
205186953edcSriastradh * driver id in the high half and the session id in the low half.
205286953edcSriastradh */
2053*fdf161e4Sriastradh static void
sun8i_crypto_freesession(void * cookie,uint64_t dsid)205486953edcSriastradh sun8i_crypto_freesession(void *cookie, uint64_t dsid)
205586953edcSriastradh {
205686953edcSriastradh
205786953edcSriastradh KASSERT((dsid & 0xffffffff) == 1);
205886953edcSriastradh }
205986953edcSriastradh
206086953edcSriastradh /*
206186953edcSriastradh * sun8i_crypto_ivlen(crd)
206286953edcSriastradh *
206386953edcSriastradh * Return the crypto engine's notion of `IV length', in bytes, for
206486953edcSriastradh * an opencrypto operation.
206586953edcSriastradh */
206686953edcSriastradh static u_int
sun8i_crypto_ivlen(const struct cryptodesc * crd)206786953edcSriastradh sun8i_crypto_ivlen(const struct cryptodesc *crd)
206886953edcSriastradh {
206986953edcSriastradh
207086953edcSriastradh switch (crd->crd_alg) {
207186953edcSriastradh case CRYPTO_AES_CBC:
207286953edcSriastradh return 16;
207386953edcSriastradh #ifdef CRYPTO_AES_XTS
207486953edcSriastradh case CRYPTO_AES_XTS:
207586953edcSriastradh return 16;
207686953edcSriastradh #endif
207786953edcSriastradh case CRYPTO_AES_CTR: /* XXX opencrypto quirk */
207886953edcSriastradh return 8;
207986953edcSriastradh #ifdef CRYPTO_DES_CBC
208086953edcSriastradh case CRYPTO_DES_CBC:
208186953edcSriastradh return 8;
208286953edcSriastradh #endif
208386953edcSriastradh case CRYPTO_3DES_CBC:
208486953edcSriastradh return 8;
208586953edcSriastradh case CRYPTO_MD5:
208686953edcSriastradh return 16;
208786953edcSriastradh #ifdef CRYPTO_SHA224
208886953edcSriastradh case CRYPTO_SHA224:
208986953edcSriastradh return 32;
209086953edcSriastradh #endif
209186953edcSriastradh #ifdef CRYPTO_SHA256
209286953edcSriastradh case CRYPTO_SHA256:
209386953edcSriastradh return 32;
209486953edcSriastradh #endif
209586953edcSriastradh case CRYPTO_SHA1_HMAC:
209686953edcSriastradh return 20;
209786953edcSriastradh case CRYPTO_SHA2_256_HMAC:
209886953edcSriastradh return 32;
209986953edcSriastradh default:
210086953edcSriastradh return 0;
210186953edcSriastradh }
210286953edcSriastradh }
210386953edcSriastradh
210486953edcSriastradh /*
210586953edcSriastradh * sun8i_crypto_process(cookie, crp, hint)
210686953edcSriastradh *
210786953edcSriastradh * Main opencrypto processing dispatch.
210886953edcSriastradh */
210986953edcSriastradh static int
sun8i_crypto_process(void * cookie,struct cryptop * crp,int hint)211086953edcSriastradh sun8i_crypto_process(void *cookie, struct cryptop *crp, int hint)
211186953edcSriastradh {
211286953edcSriastradh struct sun8i_crypto_softc *sc = cookie;
211386953edcSriastradh struct sun8i_crypto_task *task;
211486953edcSriastradh struct cryptodesc *crd = crp->crp_desc;
211586953edcSriastradh unsigned klen, ivlen;
211686953edcSriastradh uint32_t tdqc = 0, tdqs = 0;
211786953edcSriastradh uint32_t dir, method, mode = 0, ctrwidth = 0, aeskeysize = 0;
211886953edcSriastradh const uint32_t tdqa = 0;
211986953edcSriastradh int error;
212086953edcSriastradh
212186953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, entry, sc, crp, hint);
212286953edcSriastradh
212386953edcSriastradh /* Reject compositions -- we do not handle them. */
212486953edcSriastradh if (crd->crd_next != NULL) {
212586953edcSriastradh error = EOPNOTSUPP;
212686953edcSriastradh goto fail0;
212786953edcSriastradh }
212886953edcSriastradh
212986953edcSriastradh /* Reject transfers with nonsense skip. */
213086953edcSriastradh if (crd->crd_skip < 0) {
213186953edcSriastradh error = EINVAL;
213286953edcSriastradh goto fail0;
213386953edcSriastradh }
213486953edcSriastradh
213586953edcSriastradh /*
213686953edcSriastradh * Actually just reject any nonzero skip, because it requires
213786953edcSriastradh * DMA segment bookkeeping that we don't do yet.
213886953edcSriastradh */
213986953edcSriastradh if (crd->crd_skip) {
214086953edcSriastradh error = EOPNOTSUPP;
214186953edcSriastradh goto fail0;
214286953edcSriastradh }
214386953edcSriastradh
214486953edcSriastradh /* Reject large transfers. */
214586953edcSriastradh if (crd->crd_len > SUN8I_CRYPTO_MAXDMASIZE) {
214686953edcSriastradh error = EFBIG;
214786953edcSriastradh goto fail0;
214886953edcSriastradh }
214986953edcSriastradh
215086953edcSriastradh /* Reject nonsense, unaligned, or mismatched lengths. */
215186953edcSriastradh if (crd->crd_len < 0 ||
215286953edcSriastradh crd->crd_len % 4 ||
215386953edcSriastradh crd->crd_len != crp->crp_ilen) {
215486953edcSriastradh error = EINVAL;
215586953edcSriastradh goto fail0;
215686953edcSriastradh }
215786953edcSriastradh
215886953edcSriastradh /* Reject mismatched buffer lengths. */
215986953edcSriastradh /* XXX Handle crd_skip. */
216086953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) {
216186953edcSriastradh struct mbuf *m = crp->crp_buf;
216286953edcSriastradh uint32_t nbytes = 0;
216386953edcSriastradh while (m != NULL) {
216486953edcSriastradh KASSERT(m->m_len >= 0);
216586953edcSriastradh if (m->m_len > crd->crd_len ||
216686953edcSriastradh nbytes > crd->crd_len - m->m_len) {
216786953edcSriastradh error = EINVAL;
216886953edcSriastradh goto fail0;
216986953edcSriastradh }
217086953edcSriastradh nbytes += m->m_len;
217186953edcSriastradh m = m->m_next;
217286953edcSriastradh }
217386953edcSriastradh if (nbytes != crd->crd_len) {
217486953edcSriastradh error = EINVAL;
217586953edcSriastradh goto fail0;
217686953edcSriastradh }
217786953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) {
217886953edcSriastradh struct uio *uio = crp->crp_buf;
217986953edcSriastradh if (uio->uio_resid != crd->crd_len) {
218086953edcSriastradh error = EINVAL;
218186953edcSriastradh goto fail0;
218286953edcSriastradh }
218386953edcSriastradh }
218486953edcSriastradh
218586953edcSriastradh /* Get a task, or fail with ERESTART if we can't. */
218686953edcSriastradh task = sun8i_crypto_task_get(sc, &sun8i_crypto_callback, crp,
218786953edcSriastradh PR_NOWAIT);
218886953edcSriastradh if (task == NULL) {
218986953edcSriastradh /*
219086953edcSriastradh * Don't invoke crypto_done -- we are asking the
219186953edcSriastradh * opencrypto(9) machinery to queue the request and get
219286953edcSriastradh * back to us.
219386953edcSriastradh */
219486953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, busy, sc, crp, hint);
219586953edcSriastradh return ERESTART;
219686953edcSriastradh }
219786953edcSriastradh
219886953edcSriastradh /* Load key in, if relevant. */
219986953edcSriastradh klen = crd->crd_klen;
220086953edcSriastradh if (klen) {
220186953edcSriastradh if (crd->crd_alg == CRYPTO_AES_CTR)
220286953edcSriastradh /* AES-CTR is special -- see IV processing below. */
220386953edcSriastradh klen -= 32;
220486953edcSriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
220586953edcSriastradh crd->crd_key, klen/8, NULL, BUS_DMA_NOWAIT);
220686953edcSriastradh if (error)
220786953edcSriastradh goto fail1;
220886953edcSriastradh task->ct_flags |= TASK_KEY;
220986953edcSriastradh }
221086953edcSriastradh
221186953edcSriastradh /* Handle the IV, if relevant. */
221286953edcSriastradh ivlen = sun8i_crypto_ivlen(crd);
221386953edcSriastradh if (ivlen) {
221486953edcSriastradh void *iv;
221586953edcSriastradh
221686953edcSriastradh /*
221786953edcSriastradh * If there's an explicit IV, use it; otherwise
221886953edcSriastradh * randomly generate one.
221986953edcSriastradh */
222086953edcSriastradh if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
222186953edcSriastradh iv = crd->crd_iv;
222286953edcSriastradh } else {
222386953edcSriastradh cprng_fast(task->ct_iv, ivlen);
222486953edcSriastradh iv = task->ct_iv;
222586953edcSriastradh }
222686953edcSriastradh
222786953edcSriastradh /*
222886953edcSriastradh * If the IV is not already present in the user's
222986953edcSriastradh * buffer, copy it over.
223086953edcSriastradh */
223186953edcSriastradh if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
223286953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) {
223386953edcSriastradh m_copyback(crp->crp_buf, crd->crd_inject,
223486953edcSriastradh ivlen, iv);
223586953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) {
223686953edcSriastradh cuio_copyback(crp->crp_buf, crd->crd_inject,
223786953edcSriastradh ivlen, iv);
223886953edcSriastradh } else {
223986953edcSriastradh panic("invalid buffer type %x",
224086953edcSriastradh crp->crp_flags);
224186953edcSriastradh }
224286953edcSriastradh }
224386953edcSriastradh
224486953edcSriastradh /*
224586953edcSriastradh * opencrypto's idea of `AES-CTR' is special.
224686953edcSriastradh *
224786953edcSriastradh * - The low 4 bytes of the input block are drawn from
224886953edcSriastradh * an extra 4 bytes at the end of the key.
224986953edcSriastradh *
225086953edcSriastradh * - The next 8 bytes of the input block are drawn from
225186953edcSriastradh * the opencrypto iv.
225286953edcSriastradh *
225386953edcSriastradh * - The high 4 bytes are the big-endian block counter,
225486953edcSriastradh * which starts at 1 because why not.
225586953edcSriastradh */
225686953edcSriastradh if (crd->crd_alg == CRYPTO_AES_CTR) {
225786953edcSriastradh uint8_t block[16];
225886953edcSriastradh uint32_t blkno = 1;
225986953edcSriastradh
226086953edcSriastradh /* Format the initial input block. */
226186953edcSriastradh memcpy(block, crd->crd_key + klen/8, 4);
226286953edcSriastradh memcpy(block + 4, iv, 8);
226386953edcSriastradh be32enc(block + 12, blkno);
226486953edcSriastradh
226586953edcSriastradh /* Copy it into the DMA buffer. */
226686953edcSriastradh memcpy(task->ct_iv, block, 16);
226786953edcSriastradh iv = task->ct_iv;
226886953edcSriastradh ivlen = 16;
226986953edcSriastradh }
227086953edcSriastradh
227186953edcSriastradh /* Load the IV. */
227286953edcSriastradh error = bus_dmamap_load(sc->sc_dmat, task->ct_ivmap, iv, ivlen,
227386953edcSriastradh NULL, BUS_DMA_NOWAIT);
227486953edcSriastradh if (error)
227586953edcSriastradh goto fail1;
227686953edcSriastradh task->ct_flags |= TASK_IV;
227786953edcSriastradh }
227886953edcSriastradh
227986953edcSriastradh /* Load the src and dst. */
228086953edcSriastradh if (crp->crp_flags & CRYPTO_F_IMBUF) {
228186953edcSriastradh struct mbuf *m = crp->crp_buf;
228286953edcSriastradh
228386953edcSriastradh /* XXX Handle crd_skip. */
228486953edcSriastradh KASSERT(crd->crd_skip == 0);
228586953edcSriastradh error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_srcmap, m,
228686953edcSriastradh BUS_DMA_NOWAIT);
228786953edcSriastradh if (error)
228886953edcSriastradh goto fail1;
228986953edcSriastradh task->ct_flags |= TASK_SRC;
229086953edcSriastradh
229186953edcSriastradh /* XXX Handle crd_skip. */
229286953edcSriastradh KASSERT(crd->crd_skip == 0);
229386953edcSriastradh error = bus_dmamap_load_mbuf(sc->sc_dmat, task->ct_dstmap, m,
229486953edcSriastradh BUS_DMA_NOWAIT);
229586953edcSriastradh if (error)
229686953edcSriastradh goto fail1;
229786953edcSriastradh } else if (crp->crp_flags & CRYPTO_F_IOV) {
229886953edcSriastradh struct uio *uio = crp->crp_buf;
229986953edcSriastradh
230086953edcSriastradh /* XXX Handle crd_skip. */
230186953edcSriastradh KASSERT(crd->crd_skip == 0);
230286953edcSriastradh error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_srcmap, uio,
230386953edcSriastradh BUS_DMA_NOWAIT);
230486953edcSriastradh if (error)
230586953edcSriastradh goto fail1;
230686953edcSriastradh task->ct_flags |= TASK_SRC;
230786953edcSriastradh
230886953edcSriastradh /* XXX Handle crd_skip. */
230986953edcSriastradh KASSERT(crd->crd_skip == 0);
231086953edcSriastradh error = bus_dmamap_load_uio(sc->sc_dmat, task->ct_dstmap, uio,
231186953edcSriastradh BUS_DMA_NOWAIT);
231286953edcSriastradh if (error)
231386953edcSriastradh goto fail1;
231486953edcSriastradh } else {
231586953edcSriastradh panic("invalid buffer type %x", crp->crp_flags);
231686953edcSriastradh }
231786953edcSriastradh
231886953edcSriastradh /* Set the encryption direction. */
231986953edcSriastradh if (crd->crd_flags & CRD_F_ENCRYPT)
232086953edcSriastradh dir = SUN8I_CRYPTO_TDQC_OP_DIR_ENC;
232186953edcSriastradh else
232286953edcSriastradh dir = SUN8I_CRYPTO_TDQC_OP_DIR_DEC;
232386953edcSriastradh tdqc |= __SHIFTIN(dir, SUN8I_CRYPTO_TDQC_OP_DIR);
232486953edcSriastradh
232586953edcSriastradh /* Set the method. */
232686953edcSriastradh switch (crd->crd_alg) {
232786953edcSriastradh case CRYPTO_AES_CBC:
232886953edcSriastradh case CRYPTO_AES_CTR:
232986953edcSriastradh #ifdef CRYPTO_AES_ECB
233086953edcSriastradh case CRYPTO_AES_ECB:
233186953edcSriastradh #endif
233286953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_AES;
233386953edcSriastradh break;
233486953edcSriastradh #ifdef CRYPTO_AES_XTS
233586953edcSriastradh case CRYPTO_AES_XTS:
233686953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_AES;
233786953edcSriastradh break;
233886953edcSriastradh #endif
233986953edcSriastradh case CRYPTO_DES_CBC:
234086953edcSriastradh #ifdef CRYPTO_DES_ECB
234186953edcSriastradh case CRYPTO_DES_ECB:
234286953edcSriastradh #endif
234386953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_DES;
234486953edcSriastradh break;
234586953edcSriastradh case CRYPTO_3DES_CBC:
234686953edcSriastradh #ifdef CRYPTO_3DES_ECB
234786953edcSriastradh case CRYPTO_3DES_ECB:
234886953edcSriastradh #endif
234986953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_3DES;
235086953edcSriastradh break;
235186953edcSriastradh case CRYPTO_MD5:
235286953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_MD5;
235386953edcSriastradh break;
235486953edcSriastradh case CRYPTO_SHA1:
235586953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA1;
235686953edcSriastradh break;
235786953edcSriastradh #ifdef CRYPTO_SHA224
235886953edcSriastradh case CRYPTO_SHA224:
235986953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA224;
236086953edcSriastradh break;
236186953edcSriastradh #endif
236286953edcSriastradh #ifdef CRYPTO_SHA256
236386953edcSriastradh case CRYPTO_SHA256:
236486953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_SHA256;
236586953edcSriastradh break;
236686953edcSriastradh #endif
236786953edcSriastradh case CRYPTO_SHA1_HMAC:
236886953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA1;
236986953edcSriastradh break;
237086953edcSriastradh case CRYPTO_SHA2_256_HMAC:
237186953edcSriastradh method = SUN8I_CRYPTO_TDQC_METHOD_HMAC_SHA256;
237286953edcSriastradh break;
237386953edcSriastradh default:
237486953edcSriastradh panic("unknown algorithm %d", crd->crd_alg);
237586953edcSriastradh }
237686953edcSriastradh tdqc |= __SHIFTIN(method, SUN8I_CRYPTO_TDQC_METHOD);
237786953edcSriastradh
237886953edcSriastradh /* Set the key selector. No idea how to use the internal keys. */
237986953edcSriastradh tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
238086953edcSriastradh SUN8I_CRYPTO_TDQS_SKEY_SELECT);
238186953edcSriastradh
238286953edcSriastradh /* XXX Deal with AES_CTS_Last_Block_Flag. */
238386953edcSriastradh
238486953edcSriastradh /* Set the mode. */
238586953edcSriastradh switch (crd->crd_alg) {
238686953edcSriastradh #ifdef CRYPTO_AES_ECB
238786953edcSriastradh case CRYPTO_AES_ECB:
238886953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
238986953edcSriastradh break;
239086953edcSriastradh #endif
239186953edcSriastradh #ifdef CRYPTO_DES_ECB
239286953edcSriastradh case CRYPTO_DES_ECB:
239386953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
239486953edcSriastradh break;
239586953edcSriastradh #endif
239686953edcSriastradh #ifdef CRYPTO_3DES_ECB
239786953edcSriastradh case CRYPTO_3DES_ECB:
239886953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_ECB;
239986953edcSriastradh break;
240086953edcSriastradh #endif
240186953edcSriastradh case CRYPTO_AES_CBC:
240286953edcSriastradh case CRYPTO_DES_CBC:
240386953edcSriastradh case CRYPTO_3DES_CBC:
240486953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CBC;
240586953edcSriastradh break;
240686953edcSriastradh case CRYPTO_AES_CTR:
240786953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTR;
240886953edcSriastradh break;
240986953edcSriastradh #ifdef CRYPTO_AES_XTS
241086953edcSriastradh case CRYPTO_AES_XTS:
241186953edcSriastradh mode = SUN8I_CRYPTO_TDQS_OP_MODE_CTS;
241286953edcSriastradh break;
241386953edcSriastradh #endif
241486953edcSriastradh default:
241586953edcSriastradh panic("unknown algorithm %d", crd->crd_alg);
241686953edcSriastradh }
241786953edcSriastradh tdqs |= __SHIFTIN(mode, SUN8I_CRYPTO_TDQS_OP_MODE);
241886953edcSriastradh
241986953edcSriastradh /* Set the CTR width. */
242086953edcSriastradh switch (crd->crd_alg) {
242186953edcSriastradh case CRYPTO_AES_CTR:
242286953edcSriastradh ctrwidth = SUN8I_CRYPTO_TDQS_CTR_WIDTH_32;
242386953edcSriastradh break;
242486953edcSriastradh }
242586953edcSriastradh tdqs |= __SHIFTIN(ctrwidth, SUN8I_CRYPTO_TDQS_CTR_WIDTH);
242686953edcSriastradh
242786953edcSriastradh /* Set the AES key size. */
242886953edcSriastradh switch (crd->crd_alg) {
242986953edcSriastradh case CRYPTO_AES_CBC:
243086953edcSriastradh #ifdef CRYPTO_AES_ECB
243186953edcSriastradh case CRYPTO_AES_ECB:
243286953edcSriastradh #endif
243386953edcSriastradh switch (crd->crd_klen) {
243486953edcSriastradh case 128:
243586953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
243686953edcSriastradh break;
243786953edcSriastradh case 192:
243886953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
243986953edcSriastradh break;
244086953edcSriastradh case 256:
244186953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
244286953edcSriastradh break;
244386953edcSriastradh default:
244486953edcSriastradh panic("invalid AES key size in bits: %u",
244586953edcSriastradh crd->crd_klen);
244686953edcSriastradh }
244786953edcSriastradh break;
244886953edcSriastradh case CRYPTO_AES_CTR:
244986953edcSriastradh switch (crd->crd_klen) {
245086953edcSriastradh case 128 + 32:
245186953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
245286953edcSriastradh break;
245386953edcSriastradh case 192 + 32:
245486953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
245586953edcSriastradh break;
245686953edcSriastradh case 256 + 32:
245786953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
245886953edcSriastradh break;
245986953edcSriastradh default:
246086953edcSriastradh panic("invalid `AES-CTR' ` ``key'' size' in bits: %u",
246186953edcSriastradh crd->crd_klen);
246286953edcSriastradh }
246386953edcSriastradh break;
246486953edcSriastradh #ifdef CRYPTO_AES_XTS
246586953edcSriastradh case CRYPTO_AES_XTS:
246686953edcSriastradh switch (crd->crd_klen) {
246786953edcSriastradh case 256:
246886953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128;
246986953edcSriastradh break;
247086953edcSriastradh case 384:
247186953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192;
247286953edcSriastradh break;
247386953edcSriastradh case 512:
247486953edcSriastradh aeskeysize = SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256;
247586953edcSriastradh break;
247686953edcSriastradh default:
247786953edcSriastradh panic("invalid AES-XTS key size in bits: %u",
247886953edcSriastradh crd->crd_klen);
247986953edcSriastradh }
248086953edcSriastradh break;
248186953edcSriastradh #endif
248286953edcSriastradh }
248386953edcSriastradh tdqs |= __SHIFTIN(aeskeysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);
248486953edcSriastradh
248586953edcSriastradh /* Set up the task descriptor. */
248686953edcSriastradh error = sun8i_crypto_task_load(sc, task, crd->crd_len,
248786953edcSriastradh tdqc, tdqs, tdqa);
248886953edcSriastradh if (error)
248986953edcSriastradh goto fail2;
249086953edcSriastradh
249186953edcSriastradh /* Submit! */
249286953edcSriastradh error = sun8i_crypto_submit(sc, task);
249386953edcSriastradh if (error)
249486953edcSriastradh goto fail2;
249586953edcSriastradh
249686953edcSriastradh /* Success! */
249786953edcSriastradh SDT_PROBE4(sdt, sun8i_crypto, process, queued, sc, crp, hint, task);
249886953edcSriastradh return 0;
249986953edcSriastradh
250086953edcSriastradh fail2: bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
250186953edcSriastradh fail1: if (task->ct_flags & TASK_SRC)
250286953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
250386953edcSriastradh if (task->ct_flags & TASK_CTR)
250486953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
250586953edcSriastradh if (task->ct_flags & TASK_IV)
250686953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
250786953edcSriastradh if (task->ct_flags & TASK_KEY)
250886953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
250986953edcSriastradh sun8i_crypto_task_put(sc, task);
251086953edcSriastradh fail0: KASSERT(error);
251186953edcSriastradh KASSERT(error != ERESTART);
251286953edcSriastradh crp->crp_etype = error;
251386953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
251486953edcSriastradh crypto_done(crp);
251586953edcSriastradh return 0;
251686953edcSriastradh }
251786953edcSriastradh
251886953edcSriastradh /*
251986953edcSriastradh * sun8i_crypto_callback(sc, task, cookie, error)
252086953edcSriastradh *
252186953edcSriastradh * Completion callback for a task submitted via opencrypto.
252286953edcSriastradh * Release the task and pass the error on to opencrypto with
252386953edcSriastradh * crypto_done.
252486953edcSriastradh */
252586953edcSriastradh static void
sun8i_crypto_callback(struct sun8i_crypto_softc * sc,struct sun8i_crypto_task * task,void * cookie,int error)252686953edcSriastradh sun8i_crypto_callback(struct sun8i_crypto_softc *sc,
252786953edcSriastradh struct sun8i_crypto_task *task, void *cookie, int error)
252886953edcSriastradh {
252986953edcSriastradh struct cryptop *crp = cookie;
25305170c8cfSad struct cryptodesc *crd __diagused = crp->crp_desc;
253186953edcSriastradh
253286953edcSriastradh KASSERT(error != ERESTART);
253386953edcSriastradh KASSERT(crd != NULL);
253486953edcSriastradh KASSERT(crd->crd_next == NULL);
253586953edcSriastradh
253686953edcSriastradh /* Return the number of bytes processed. */
253786953edcSriastradh crp->crp_olen = error ? 0 : crp->crp_ilen;
253886953edcSriastradh
253986953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
254086953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
254186953edcSriastradh if (task->ct_flags & TASK_CTR)
254286953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ctrmap);
254386953edcSriastradh if (task->ct_flags & TASK_IV)
254486953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_ivmap);
254586953edcSriastradh if (task->ct_flags & TASK_KEY)
254686953edcSriastradh bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
254786953edcSriastradh sun8i_crypto_task_put(sc, task);
254886953edcSriastradh KASSERT(error != ERESTART);
254986953edcSriastradh crp->crp_etype = error;
255086953edcSriastradh SDT_PROBE3(sdt, sun8i_crypto, process, done, sc, crp, error);
255186953edcSriastradh crypto_done(crp);
255286953edcSriastradh }
2553